diff --git a/.github/workflows/add-issue-to-project.yml b/.github/workflows/add-issue-to-project.yml deleted file mode 100644 index 7b5d3473..00000000 --- a/.github/workflows/add-issue-to-project.yml +++ /dev/null @@ -1,16 +0,0 @@ -name: Add new issues to Verification Team project - -on: - issues: - types: - - opened - -jobs: - add-to-project: - name: Add issue to project - runs-on: ubuntu-latest - steps: - - uses: actions/add-to-project@main - with: - project-url: https://github.com/orgs/NOAA-GSL/projects/14 - github-token: ${{ secrets.ADD_TO_PROJECT_PAT }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..5fcc5399 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,146 @@ +name: "CI" +on: + push: + tags: + - "[0-9]+.[0-9]+.[0-9]+" + - "[0-9]+.[0-9]+.[0-9]+-rc[0-9]+" + branches: [main] + pull_request: + workflow_dispatch: # Manually +env: + REGISTRY: ghcr.io/noaa-gsl/vxingest + +jobs: + build-ingest: + name: Build Ingest image + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: actions/checkout@v4 + - name: Generate image metadata + uses: docker/metadata-action@v4 + id: meta + with: + images: | + ghcr.io/noaa-gsl/vxingest/ingest + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + labels: | + org.opencontainers.image.vendor=NOAA's Global Systems Laboratory + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build image and push + uses: docker/build-push-action@v5 + with: + file: docker/ingest/Dockerfile + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + push: true + # Use a cache to speed up builds + # Note: this may cause security issues if the apt updates are cached + # It may make more sense to build eccodes as its own image instead. + cache-from: type=registry,ref=ghcr.io/noaa-gsl/vxingest/cache/ingest:buildcache + cache-to: type=registry,ref=ghcr.io/noaa-gsl/vxingest/cache/ingest:buildcache,mode=max + build-import: + name: Build Import image + runs-on: ubuntu-latest + permissions: + packages: write + steps: + - uses: actions/checkout@v4 + - name: Generate image metadata + uses: docker/metadata-action@v4 + id: meta + with: + images: | + ghcr.io/noaa-gsl/vxingest/import + tags: | + type=ref,event=branch + type=ref,event=pr + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=sha + # set latest tag for default branch + type=raw,value=latest,enable={{is_default_branch}} + labels: | + org.opencontainers.image.vendor=NOAA's Global Systems Laboratory + - uses: docker/setup-qemu-action@v3 + - uses: docker/setup-buildx-action@v3 + - name: Login to GHCR + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build image and push + uses: docker/build-push-action@v5 + with: + file: docker/import/Dockerfile + context: . + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + platforms: linux/amd64,linux/arm64 + push: true + scan-ingest: + runs-on: ubuntu-latest + needs: build-ingest + steps: + - name: Extract SHORT_SHA + run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV" + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@0.16.0 + with: + image-ref: "ghcr.io/noaa-gsl/vxingest/ingest:sha-${{ env.SHORT_SHA }}" + format: "sarif" + output: "trivy-ingest-results.sarif" + ignore-unfixed: true + severity: "CRITICAL,HIGH" + limit-severities-for-sarif: true + exit-code: "1" + env: + TRIVY_USERNAME: ${{ github.actor }} + TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Trivy scan results to GitHub Security tab + if: always() + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-ingest-results.sarif" + scan-import: + runs-on: ubuntu-latest + needs: build-import + steps: + - name: Extract SHORT_SHA + run: echo "SHORT_SHA=${GITHUB_SHA::7}" >> "$GITHUB_ENV" + - name: Scan image with Trivy + uses: aquasecurity/trivy-action@0.16.0 + with: + image-ref: "ghcr.io/noaa-gsl/vxingest/import:sha-${{ env.SHORT_SHA }}" + format: "sarif" + output: "trivy-import-results.sarif" + ignore-unfixed: true + severity: "CRITICAL,HIGH" + limit-severities-for-sarif: true + # exit-code: "1" # FIXME: allow failures for now. Couchbase needs to update cbtools + env: + TRIVY_USERNAME: ${{ github.actor }} + TRIVY_PASSWORD: ${{ secrets.GITHUB_TOKEN }} + - name: Upload Trivy scan results to GitHub Security tab + if: always() + uses: github/codeql-action/upload-sarif@v2 + with: + sarif_file: "trivy-import-results.sarif" diff --git a/docker/import/Dockerfile b/docker/import/Dockerfile index e6cb83b1..c6a2fa81 100644 --- a/docker/import/Dockerfile +++ b/docker/import/Dockerfile @@ -30,9 +30,13 @@ RUN groupadd --gid ${ID} amb-verif && \ WORKDIR /app +# Install the couchbase tools +ENV CB_VERSION=7.2.2 +RUN curl -L https://packages.couchbase.com/releases/${CB_VERSION}/couchbase-server-tools_${CB_VERSION}-linux_$(uname -m).tar.gz | tar xz --directory /usr/local + # Copy the scripts and metadata dirs so the import script can run -COPY ./scripts/ /app/ -COPY ./mats_metadata_and_indexes /app/ +COPY ./scripts /app/scripts +COPY ./mats_metadata_and_indexes /app/mats_metadata_and_indexes # TODO - install the cbtools directly and remove from the git repo # See: https://docs.couchbase.com/cloud/reference/command-line-tools.html#download-and-install-the-couchbase-command-line-tools diff --git a/scripts/VXingest_utilities/import_docs.sh b/scripts/VXingest_utilities/import_docs.sh index b9341764..82905378 100755 --- a/scripts/VXingest_utilities/import_docs.sh +++ b/scripts/VXingest_utilities/import_docs.sh @@ -98,7 +98,7 @@ do_import() { sleep 10 cat ${file_list} | while read f; do echo "cbimport json --cluster couchbase://${host} --bucket ${bucket} --scope-collection-exp ${scope}.${collection} --username ${user} --password ${pwd} --format list --generate-key %id% --dataset file:///${f}" - ${curdir}/scripts/cbtools/bin/cbimport json --cluster couchbase://${host} --bucket ${bucket} --scope-collection-exp ${scope}.${collection} --username ${user} --password ${pwd} --format list --generate-key %id% --dataset file:///${f} + cbimport json --cluster couchbase://${host} --bucket ${bucket} --scope-collection-exp ${scope}.${collection} --username ${user} --password ${pwd} --format list --generate-key %id% --dataset file:///${f} done } diff --git a/scripts/VXingest_utilities/promql b/scripts/VXingest_utilities/promql deleted file mode 100755 index b9185834..00000000 Binary files a/scripts/VXingest_utilities/promql and /dev/null differ diff --git a/scripts/cbtools/LICENSE.txt b/scripts/cbtools/LICENSE.txt deleted file mode 100644 index e829534c..00000000 --- a/scripts/cbtools/LICENSE.txt +++ /dev/null @@ -1,799 +0,0 @@ -COUCHBASE, INC. LICENSE AGREEMENT -IMPORTANT--READ CAREFULLY: BY CLICKING THE "I ACCEPT" BOX OR INSTALLING, -DOWNLOADING OR OTHERWISE USING THE SOFTWARE AND ANY ASSOCIATED -DOCUMENTATION, YOU, ON BEHALF OF YOURSELF AND/OR AS AN AUTHORIZED -REPRESENTATIVE ON BEHALF OF AN ENTITY (COLLECTIVELY, "CUSTOMER"), -AGREE TO ALL THE TERMS OF THIS LICENSE AGREEMENT AND, IF APPLICABLE, -THE NON-GA OFFERING SUPPLEMENTAL TERMS (COLLECTIVELY, THE "AGREEMENT") -REGARDING CUSTOMER'S USE OF THE SOFTWARE. YOU REPRESENT AND WARRANT THAT -YOU HAVE FULL LEGAL AUTHORITY TO BIND CUSTOMER TO THIS AGREEMENT. IF -YOU DO NOT AGREE WITH ALL OF THESE TERMS, DO NOT SELECT THE "I ACCEPT" -BOX AND DO NOT INSTALL, DOWNLOAD OR OTHERWISE USE THE SOFTWARE. THE -EFFECTIVE DATE OF THIS AGREEMENT IS THE DATE ON WHICH YOU CLICK "I ACCEPT" -OR OTHERWISE INSTALL, DOWNLOAD OR USE THE SOFTWARE. - - -1. Definitions. - 1.1 Capitalized terms used herein shall have the following definitions: - "Commercial Details" means the identified products and/or services, - quantity (number of Licensed Nodes, Licensed Devices, and/or service - entitlements), price, server size, support level, subscription start - and end date, and Professional Services description. - "Core" means the virtual representation of one or more hardware - threads. A hardware thread can be either a physical core or a - hyper-threaded core. - "Cross-Data Center Replication" means an asynchronous data replication - system, involving replicating active data to multiple, geographically - diverse data centers or data repositories. - "Customer" means the person, legal entity or organization, as - applicable, accepting the terms and conditions hereunder and/or - entering into an Order that references this Agreement. - "Deployment" means all Licensed Nodes and Licensed Devices within - a particular workload or application. "Documentation" means the - technical user guides or manuals provided by Couchbase related to - the Software. - "Fees" means any sums, fees or charges specified on an Order or - otherwise payable under this Agreement. - "Licensed Device" means a unique device (such as a mobile device, - laptop, or IoT device) that stores data locally using the "Couchbase - Lite" product during a rolling thirty (30) day period. - "Licensed Node" means an instance of the Software running on a server, - including a physical server, server blade, virtual machine, software - container, or cloud server (whether in a testing, development, - production, or other environment). - "Order" means a transaction document (such as a signed sales quote or - statement of work) identifying the applicable Professional Services, - Software, number of Licensed Nodes and Licensed Devices, Fees and - Subscription Term. - "Privacy Notice" means the privacy notice located at - https://www.couchbase.com/privacy-policy (and any successor location - designated by Couchbase), as may be updated from time to time. - "Professional Services" means consulting services, training, - and other professional services as well as the resulting reports, - written materials, or other outputs provided to Customer as part of - an engagement as identified in the applicable Order. - "RAM" or Random Access Memory, means the main memory used to store - data for quick access by a computer's processor. - "Software" means the applicable Couchbase product either (i) - as reflected in an Order or (ii) as otherwise made available to - Customer under this Agreement. - "Subscription Term" means, as applicable: (i) the period stated - on an Order, if any, during which Customer is licensed to use the - Software and Documentation and receive the Professional Services (if - applicable) and Support; (ii) the first date an Enterprise License is - required (including before an Order is signed) until the date such - Enterprise License is either (a) memorialized in an Order (in which - case the definition of subsection (i) herein shall apply), or (b) - terminated in accordance with the terms herein; or (iii) the date - of download for a Free License until the date such Free License is - either (a) converted into an Enterprise License (in which case the - definitions of subsections (i) or (ii) shall apply, as applicable), - or (b) terminated in accordance with the terms herein. - "Support" means the technical support and Software maintenance - services (with the right to receive Software updates and upgrades made - generally available by Couchbase) as described in the then-current - Couchbase support services terms at www.couchbase.com/support-policy - (and any successor location designated by Couchbase), as may be - updated from time to time. - -The term "including" means including but not limited to. - -2. License Grants. - 2.1 License Grant as to Free Licenses. A "Free License" is allowed only - for development use and evaluation of the Software and is provided with - no Support or any other services. Couchbase is under no obligation to - provide or continue to provide the Free Licenses (including any update, - upgrade or new version of the Software licensed thereunder). During - the Subscription Term, and subject to Customer's compliance with the - terms and conditions of this Agreement, Couchbase grants to Customer a - revocable, unpaid, non-exclusive, non-transferable, non-sublicensable, - non-fee bearing download license to install and use the Software only - for Customer's own internal development use and evaluation of the - Software. Prior to using the Software for any other use (including - for testing, quality assurance, or in production), Customer agrees to - enter into an Order with Couchbase and pay the applicable Fees. If, - at any time, Customer uses the Software for any other use (including - for testing, quality assurance, or in production) without an active - Order, then (i) Customer acknowledges and agrees that its Free License - is automatically converted to an Enterprise License, (ii) Couchbase - shall have the right to audit and charge Customer for such use as set - forth in Section 8 herein, and (iii) Customer shall use best efforts to - enter into an Order with Couchbase for such use in order to remedy the - noncompliant use as soon as possible. Free Licenses granted to Customer - hereunder are revocable and terminable at any time in Couchbase's sole - and absolute discretion. - - 2.2 License Grant as to Enterprise Licenses. An "Enterprise License" - is required if Customer uses the Software (a) for testing, quality - assurance, or in production, (b) for any use other than solely - for internal development use or evaluation of the Software, or (c) - after Customer's initial request for Support. After Customer in the - first instance uses the Software for testing, quality assurance, - or in production, or converts to an Enterprise License otherwise, - then the Enterprise License terms of this Agreement (absent another - applicable enterprise agreement between the parties) shall apply. During - the Subscription Term, and subject to Customer's compliance with the - terms and conditions of this Agreement, Couchbase grants to Customer a - revocable, non-exclusive, non-transferable, non-sublicensable (except - as explicitly set forth herein), fee-bearing license to install and - use the Software and Documentation only for Customer's own internal - use and limited to the type, size, and number of Licensed Nodes and - Licensed Devices paid for by Customer and in accordance with any - additional license terms specified in the applicable Order, and for - no other purposes whatsoever. - - 2.3 Tools, Connectors, and Third Party Products. The Software may: - (i) include certain components developed and owned by Couchbase - ("Couchbase Components") which connect to products, components, files, - programs, databases, content sources or other materials developed and - owned by third parties ("Third Party Products"); and (ii) include or - be distributed with Third Party Products. Customer's use of Couchbase - Components and Third Party Products may be subject to additional terms - and conditions and third party license terms, including as set forth - in Exhibit A of this Agreement. Further, Third Party Products may be - subject to third party licenses provided by such third party or executed - between Customer and such third party (in each case, a "Third Party - License"). Couchbase has not developed or published and does not own, - control or license such Third Party Products. Customer shall be solely - responsible for (i) its compliance with any such Third Party License - in its use of any Third Party Products and (ii) the confidentiality, - security, and integrity of any content or data Customer chooses - to transfer to any Third Party Products. Couchbase shall have no - responsibility or liability whatsoever for the installation, operation, - or support functionality of, or otherwise related to or arising out - of any Third Party Products, including any loss or compromise of data. - -3. Customer Responsibilities. - 3.1 Customer shall not (and shall not allow any third party to): - (a) copy or use the Software, Documentation, or any related reports, - technologies or services in any manner except as expressly permitted - in this Agreement; - (b) for Enterprise Licenses, use or deploy the Software in excess of - the number of Licensed Nodes and Licensed Devices for which Customer - has paid the applicable Fees; - (c) for Enterprise Licenses, use or deploy the Software in excess - of the number of Cores and RAM for each Licensed Node for which - Customer has paid the applicable Fees; - (d) transfer, sell, rent, lease, lend, distribute, market, - commercialize or sublicense the Software, Documentation, or any - related reports, technologies or services to any third party, - provided that Customer may use the Software in connection with an - application made available to Customer's end users as long as they - cannot access the Software directly; - (e) use the Software for providing time-sharing services, service - bureau services or as part of an application services provider or - as a service offering primarily designed to offer the functionality - of the Software; - (f) reverse engineer, disassemble, or decompile the Software (except - to the extent such restrictions are prohibited); - (g) alter, modify, enhance or prepare any derivative work from or - of the Software and Documentation; - (h) alter or remove any proprietary notices in the Software, - Documentation or any related reports, technologies or services; - (i) for Enterprise Licenses, use or transfer Licensed Nodes and/or - Licensed Devices designated to a particular Deployment or project - to another or new Deployment or project; or - (j) use the Software, Documentation or any related reports, - technologies or services for the purposes of, or publicly display or - communicate the results of, benchmarking or competitive analysis of - the Software, or developing, using, providing, or supporting products - or services competitive to Couchbase. - - 3.2 Customer understands that the Software, Documentation, and any - related technology or services are subject to U.S. export control - laws and regulations. Customer shall comply with all applicable - laws in connection with its use of the Software, Documentation, - Professional Services and Support, including but not limited to - applicable international and domestic export controls, including - economic sanctions, laws, regulations, or orders that apply to Customer, - the Software the Documentation and any related reports, technologies - or services ("Export Laws"). In furtherance of this obligation, - Customer shall ensure that: (a) Customer does not use the Software, - technology or services in violation of any Export Laws; and (b) it - does not provide access to the Software, technology or services to - (i) persons on the U.S. Department of Commerce's Denied Persons List - or Entity List, or the U.S. Treasury Department's list of Specially - Designated Nationals, (ii) military end-users or for military end-use, - or (iii) parties engaged in activities directly or indirectly related - to the proliferation of weapons of mass destruction. - - 3.3 If Customer does not comply with the license terms or the foregoing - restrictions, Couchbase may (without refund or credit, if applicable, - and without prejudice to any other rights or remedies Couchbase - may have under this Agreement or at law), at its sole discretion: - (i) terminate this Agreement (including all active Orders hereunder) - thereby revoking Customer's Free and/or Enterprise License(s) to the - Software and Documentation, or (ii) suspend Customer's Free and/or - Enterprise License(s) to the Software and Documentation until Customer - comes into compliance with such terms and restrictions. - - 3.4 Customer acknowledges that a breach of its obligations to Couchbase - under this Agreement, other than the payment obligations, will result - in irreparable and continuing damage for which monetary damages may not - be sufficient, and agrees that Couchbase will be entitled to receive - in addition to its other rights and remedies hereunder or at law, - injunctive and/or other equitable relief. All remedies of Couchbase - set forth in this Agreement are cumulative and in addition to, and - not in lieu of any other remedy of Couchbase as law or in equity. - -4. Services. - 4.1 This entire Section 4 applies only to Enterprise Licenses fully - paid for by Customer and not to Free Licenses. - - 4.2 The parties may agree to have Couchbase provide Professional - Services to Customer, which shall be set forth on an Order signed - by both parties. Such Professional Services shall be governed by the - terms and conditions of this Agreement. - - 4.3 Professional Services and any related reports, technologies or - services are accepted when delivered unless otherwise set forth in an - Order. Couchbase may engage qualified subcontractors to provide the - Professional Services, provided that Couchbase remains responsible for - any subcontractor's compliance with this Agreement. Couchbase grants to - Customer a royalty-free, perpetual, non-exclusive, non-transferable, - non-sublicensable license to use and reproduce any reports for - Customer's internal business purposes. - - 4.4 Unless explicitly set forth in an Order, any Professional Services - purchased under this Agreement shall expire upon the end of the - Subscription Term (up to and including any expiration date) or after - twelve (12) months if no expiration date is specified (in each case, - the "Services Expiration Date"). Any unused Professional Services - after such Services Expiration Date shall expire without refund - of any prepaid Fees. For Professional Services that would have been - invoiceable in arrears, Couchbase will provide a final invoice for the - unused Professional Services in accordance with this Agreement. Unless - otherwise stated in an Order, Customer shall pay Couchbase's reasonable - travel and incidental expenses incurred in conducting (in relation to - the Professional Services or otherwise) on-site activities at Customer's - site upon receiving an invoice from Couchbase. - -5. Proprietary Rights. - 5.1 This Agreement does not transfer any right, title or interest - in any intellectual property to any party, except as expressly - set forth in this Agreement. The Software (including the source - code, any modifications, improvements, enhancements or derivatives - thereto), and all Documentation and outputs resulting from Professional - Services, are and shall remain the sole property of Couchbase and its - licensors. Except for the license rights granted under this Agreement, - Couchbase and its licensors retain all right, title and interest in and - to the Software, Documentation, and Professional Services (including - resulting reports or written materials), including all intellectual - property rights therein and thereto. - - 5.2 The Software may include open source software components and such - open source components shall be licensed to Customer under the terms of - the applicable open source license conditions and/or copyright notices - that can be found in the licenses files, Documentation or materials - accompanying the Software. - - 5.3 If Customer provides any suggestions or feedback regarding the - Software, Documentation, Support, or Professional Services, Couchbase - may use such information without obligation to Customer, and Customer - hereby irrevocably assigns to Couchbase all right, title, and interest - in that feedback or those suggestions. - - 5.4 If Customer is the United States Government or any contractor - thereof, all licenses granted hereunder are subject to the following: - (a) for acquisition by or on behalf of civil agencies, as necessary - to obtain protection as "commercial computer software" and related - documentation in accordance with the terms of this Agreement and as - specified in Subpart 12.1212 of the Federal Acquisition Regulation - (FAR), 48 C.F.R.12.1212, and its successors; and - (b) for acquisition by or on behalf of the Department of Defense - (DOD) and any agencies or units thereof, as necessary to obtain - protection as "commercial computer software" and related documentation - in accordance with the terms of this Agreement and as specified in - Subparts 227.7202-1 and 227.7202-3 of the DOD FAR Supplement, 48 - C.F.R.227.7202-1 and 227.7202-3, and its successors, manufacturer - is Couchbase, Inc. - -6. Support. - 6.1 This entire Section 6 applies only to Enterprise Licenses fully - paid for by Customer and not to Free Licenses. - - 6.2 Couchbase will provide Customer with the level of Support - indicated on the applicable Order and paid for by Customer. For all - Licensed Nodes and Licensed Devices within a specific Deployment, - all such nodes and instances must be at the same level of Support, - including any that are used for disaster recovery or backup that are - associated with the specific Deployment. For the avoidance of doubt, - each specific Deployment can have its own level of Support. - - 6.3 When using the Cross-Data Center Replication feature, Customer - must have all Licensed Nodes and Licensed Devices at the same level of - Support for all instances on all sides of the replication connection, - including if one side of the connection is only used for disaster - recovery or backup. - - 6.4 Couchbase may modify the terms of Support from time to time, - provided that such modifications do not materially and adversely affect - Customer's Support subscription. - -7. Fees. - 7.1 This Section 7 applies only to Enterprise Licenses fully paid for - by Customer and not to Free Licenses. - - 7.2 Customer will pay Couchbase the Fees in advance, within thirty (30) - days of the date of the invoice, unless otherwise explicitly indicated - in the applicable Order. If Customer purchases licenses or services - related to the Software through an approved third-party partner, - then the applicable agreement with that third-party will govern with - respect to Customer for the appropriate payment terms, provided that - such third-party agreement shall not override or attempt to override the - payment terms in place between Couchbase and such approved third-party - partner. All other terms and restrictions with respect to the use of - the Software shall continue to apply to Customer. All payments are - non-cancelable, not subject to the Limitation of Liability in Section - 12 below, and shall be made in the currency stated on the applicable - Order. Fees are non-refundable except to the extent expressly provided - for in this Agreement. Late payments will bear interest at the lesser - of one- and one-half percent (1 1/2 %) per month or the maximum rate - allowed by applicable law. Customer will reimburse Couchbase for all - reasonable costs and expenses incurred (including reasonable attorneys' - fees) in collecting any overdue amounts. - - 7.3 All Fees payable by Customer are exclusive of applicable taxes - and duties (such as, without limitation, VAT, Service Tax, GST, - excise taxes, sales and transactions taxes, and gross receipts tax - (collectively, the "Transaction Taxes"). If applicable, Couchbase may - charge and Customer shall pay all Transaction Taxes that Couchbase - is legally obligated or authorized to collect from Customer. Customer - will provide such information to Couchbase as reasonably required to - determine whether Couchbase is obligated to collect Transaction Taxes - from Customer. Couchbase will not collect, and Customer will not pay, - any Transaction Taxes for which Customer furnishes a properly completed - exemption certificate or a direct payment permit certificate for - which Couchbase may claim an available exemption from such Transaction - Taxes. All payments made by Customer to Couchbase under this Agreement - will be made free and clear of any deduction or withholding, as may - be required by law. If any such deduction or withholding (including - but not limited to cross-border withholding taxes) is required on any - payment, Customer will pay such additional amounts as are necessary so - that the net amount received by Couchbase is equal to the amount then - due and payable under this Agreement. Couchbase will provide Customer - with such tax forms as are reasonably requested in order to reduce - or eliminate the amount of any withholding or deduction for taxes in - respect of payments made under this Agreement. - - 7.4 Customer is obligated to pay all applicable Fees without any - requirement for Couchbase to provide a purchase order ("PO") number on - Couchbase's invoice (or otherwise). If Customer sends Couchbase a PO - in lieu of executing an Order, the PO will be deemed a binding contract - offer which Couchbase may accept by (i) signing the PO or (ii) sending - a written order acknowledgment of acceptance of the PO (thereby forming - a mutually agreed Order governed by this Agreement). In any event, - only the Commercial Details listed on PO shall be considered part of - the Order created (exclusive of any pre-printed terms on the PO). Any - other terms on the PO which either (i) conflict with the terms of this - Agreement, or (ii) are not agreed under this Agreement, will be void and - without effect, even if Couchbase signs the PO. All accepted POs will - automatically be governed by this Agreement (even if the PO does not - reference this Agreement). It is expressly agreed that Section 7 shall - apply in respect of any PO sent by Customer and accepted by Couchbase. - -8. Records Retention and Audit. - 8.1 During any Subscription Term and for at least one (1) year - thereafter, Customer shall maintain complete and accurate records to - permit Couchbase to verify Customer's compliance with this Agreement - (including the number of Licensed Nodes and Licensed Devices used - by Customer as well as Customer's compliance with its obligations - post-termination or expiration), and provide Couchbase with such - records within ten (10) days of request. - - 8.2 Every three (3) months within a Subscription Term (or on another - cadence as mutually agreed between the parties) or upon request by - Couchbase for any use requiring an Enterprise License where no Order - is in place, Customer shall self-certify to Couchbase the total number - of Licensed Nodes and Licensed Devices used in any production, test, - development, or other Deployment. Such self-certification shall - be in accordance with Couchbase's instruction and in the form of - either a (i) written report signed by an authorized representative of - Customer or (ii) copy of an automatically generated report created by - Customer. Couchbase will review such reports and determine if any true - up Order is required. Additional fees for any excess usage shall be - calculated based on the fees specified in the applicable Order for the - applicable Licensed Node or Licensed Device size and type, and prorated, - as applicable. If (i) the excess usage includes nodes or device sizes or - types for which fees are not specified in the applicable existing Order - or (ii) use requiring an Enterprise License occurred without an Order, - then the (additional) fees shall be calculated based on Couchbase's - list price in effect at the time and prorated, as applicable. - - 8.3 Upon at least thirty (30) days prior written notice, but no - more than once in any twelve (12) month period, Couchbase may audit - Customer's use of the Software solely as necessary to verify Customer's - compliance with the terms of this Agreement during the Subscription - Term and for one (1) year thereafter. Any such audit will be conducted - during regular business hours at Customer's facilities and will not - unreasonably interfere with Customer's business activities. Customer - will provide Couchbase with access to the relevant Customer records - and facilities. - -9. Confidentiality. - 9.1 Customer and Couchbase will maintain the confidentiality of - Confidential Information. "Confidential Information" means any - proprietary information received by the other party during, or prior to - entering into, this Agreement that a party should know is confidential - or proprietary based on the circumstances surrounding the disclosure, - including the Software and any non-public technical and business - information (including Commercial Details). Confidential Information - does not include information that (a) is or becomes generally known - to the public through no fault of or breach of this Agreement by the - receiving party; (b) is rightfully known by the receiving party at - the time of disclosure without an obligation of confidentiality to - the disclosing party; (c) is independently developed by the receiving - party without use of the disclosing party's Confidential Information; or - (d) the receiving party rightfully obtains from a third party without - restriction on use or disclosure. - - 9.2 The receiving party of any Confidential Information of the other - party agrees not to use such Confidential Information for any purpose - except as necessary to fulfill its obligations and exercise its rights - under this Agreement. The receiving party shall protect the secrecy - of and prevent any unauthorized disclosure or use of the disclosing - party's Confidential Information using the same degree of care that - it takes to protect its own confidential information and in no event - shall use less than reasonable care. - - 9.3 Upon termination of this Agreement, the receiving party will, at - the disclosing party's option, promptly return or destroy (and provide - written certification of such destruction) the disclosing party's - Confidential Information. A party may disclose the other party's - Confidential Information to the extent required by law or regulation. - -10. DISCLAIMER OF WARRANTY. -THE SOFTWARE, DOCUMENTATION AND ANY PROFESSIONAL SERVICES PROVIDED -HEREUNDER ARE PROVIDED "AS IS" WITHOUT WARRANTY OF ANY KIND INCLUDING -THAT THE SOFTWARE, DOCUMENTATION OR PROFESSIONAL SERVICES PROVIDED -HEREUNDER WILL MEET CUSTOMER'S REQUIREMENTS, OPERATE IN THE COMBINATIONS -CUSTOMER MAY SELECT FOR USE, WILL BE ERROR-FREE OR UNINTERRUPTED, OR -THAT ALL SOFTWARE ERRORS OR DEFECTS WILL BE CORRECTED. TO THE MAXIMUM -EXTENT PERMITTED BY APPLICABLE LAW, COUCHBASE, ANY OF ITS AFFILIATES OR -LICENSORS (COLLECTIVELY, THE "COUCHBASE PARTIES") HEREBY DISCLAIM ALL -WARRANTIES, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE -IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, -NON-INFRINGEMENT, TITLE, AND ANY WARRANTIES ARISING OUT OF COURSE OF -DEALING, USAGE OR TRADE. THE COUCHBASE PARTIES DO NOT WARRANT THAT THE -SOFTWARE IS DESIGNED, MANUFACTURED, OR INTENDED FOR USE IN HAZARDOUS -ENVIRONMENTS REQUIRING FAIL-SAFE PERFORMANCE WHERE THE FAILURE OF THE -PRODUCT COULD LEAD TO DEATH, PERSONAL INJURY, OR SIGNIFICANT PHYSICAL -OR ENVIRONMENTAL DAMAGE. - -11. Indemnification of Third-Party Claims. - 11.1 Couchbase Indemnification. Subject to the terms of this Section - 11, Couchbase will indemnify and defend Customer from and against - any damages finally awarded against Customer resulting from any - third-party claims that the non-open source software components of - the Software, Documentation or Professional Services infringe any - valid, enforceable United States patent, United States copyright, - or United States trademark; provided that: (a) Customer promptly - notifies Couchbase of the claim; (b) Customer gives Couchbase all - necessary information regarding the claim and reasonably cooperates - with Couchbase; (c) Customer allows Couchbase exclusive control of the - defense and all related settlement negotiations; and (d) Customer does - not admit fault or liability with respect to this Agreement, any Order, - Customer's actions or those of Couchbase. - - 11.2 Enjoinment. Without limiting the forgoing, and notwithstanding - anything to the contrary in this Agreement, if use of the Software - is enjoined, or Couchbase determines that such use may be enjoined, - Couchbase will, at its sole option and expense, (i) procure for - Customer the right to continue using the affected Software; (ii) - replace or modify the affected Software such that it does not infringe; - or (iii) if either option (i) or (ii) is not commercially feasible in - Couchbase's reasonable opinion, as applicable, terminate the affected - portions of an Order and refund Customer any prepaid but unused Fees - for the affected Software. - - 11.3 Customer Indemnification. Customer will indemnify and defend - the Couchbase Parties from and against any damages awarded against - Couchbase in connection with any third-party claim arising out of (a) - Customer's use, operation or combination of the Software, including - Customer's data or content, that infringes any United States patent, - United States copyright or United States trademark; (b) use of, - or inability to use the Software by Customer, or any third party - that receives or obtains access to or relies on the Software or any - component thereof from or through (directly or indirectly) Customer; - and (c) any breach by Customer of any provisions of this Agreement, - provided that: (i) Couchbase promptly notifies Customer of the claim; - (ii) Couchbase gives Customer all necessary information regarding the - claim and reasonably cooperates with Customer; (iii) Couchbase allows - Customer exclusive control of the defense and all related settlement - negotiations; and (iv) Couchbase does not admit fault or liability - with respect to this Agreement, any Order, Couchbase's actions or - those of Customer. - - 11.4 Exclusions. Couchbase will have no liability or obligation to - indemnify under Section 11.1 for any infringement claim arising out of - (i) modifications made by a party other than Couchbase, to the extent - a claim would not have occurred but for such modifications, (ii) the - use of any non-current version of the Software provided that Couchbase - has given reasonable written notice to Customer to migrate to the - then-current version of the Software (unless the infringing portion is - also in the then-current, unaltered release), (iii) the use, operation - or combination of the Software with non-Couchbase programs, data, or - equipment to the extent such infringement would have been avoided but - for such use, operation or combination, (iv) any open source software - components, (v) use in violation of this Agreement or in non-compliance - with the applicable Documentation, (vi) Customer's continuation of its - allegedly infringing activity after being notified thereof or after - being provided a replacement or modification by Couchbase that would - have avoided the alleged infringement; or (vii) Couchbase's compliance - with any materials, designs, specifications or instructions provided - by Customer. The Couchbase indemnification obligations will also not - apply to any claim to the extent it arises from any matter for which - Customer is obligated to indemnify Couchbase pursuant to Section 11.3. - - 11.5 Sole Remedy. THE TERMS OF THIS SECTION 11 CONSTITUTE THE ENTIRE - LIABILITY OF THE COUCHBASE PARTIES, AND CUSTOMER'S SOLE AND EXCLUSIVE - REMEDY WITH RESPECT TO ANY THIRD-PARTY CLAIMS OF INFRINGEMENT OR - MISAPPROPRIATION OF INTELLECTUAL PROPERTY RIGHTS OF ANY KIND. - - 11.6 Applicability. Section 11, excluding 11.3 above, applies only to - Enterprise Licenses fully paid for by Customer and not to Free Licenses. - -12. LIMITATION OF LIABILITY. -TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL THE -COUCHBASE PARTIES BE LIABLE TO CUSTOMER OR TO ANY THIRD PARTY FOR: (A) -ANY INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL OR EXEMPLARY DAMAGES; -OR (B) THE COST OF PROCURING SUBSTITUTE PRODUCTS OR PROFESSIONAL -SERVICES ARISING OUT OF OR IN ANY WAY RELATING TO THIS AGREEMENT, -OR THE USE OF OR INABILITY TO USE THE SOFTWARE, DOCUMENTATION OR THE -PROFESSIONAL SERVICES; OR (C) DAMAGES OR OTHER LOSSES FOR LOSS OF USE, -LOSS OF BUSINESS, LOSS OF GOODWILL, WORK STOPPAGE, LOST PROFITS, LOSS -OF DATA, COMPUTER FAILURE OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR -LOSSES EVEN IF ADVISED OF THE POSSIBILITY THEREOF AND REGARDLESS OF -THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT OR OTHERWISE) UPON WHICH -THE CLAIM IS BASED. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, -IN NO EVENT WILL THE COUCHBASE PARTIES' AGGREGATE LIABILITY TO CUSTOMER, -FROM ALL CAUSES OF ACTION AND UNDER ALL THEORIES OF LIABILITY, EXCEED THE -TOTAL AMOUNT OF FEES PAID OR DUE AND OWING UNDER THE APPLICABLE ORDER(S) -BY CUSTOMER TO COUCHBASE THAT ARE ATTRIBUTABLE TO THE ORDER GIVING RISE -TO THE LIABILITY IN THE TWELVE (12) MONTH PERIOD IMMEDIATELY PRECEDING THE -ACT OR OMISSION FIRST GIVING RISE TO THE LIABILITY. The parties expressly -acknowledge and agree that Couchbase has set its prices and entered into -this Agreement in reliance upon the limitations of liability specified -herein, which allocate the risk between Couchbase and Customer and form -a basis of the bargain between the parties. - -13. Term and Termination. - 13.1 This Agreement is effective as of the date of Customer's acceptance - of this Agreement (the "Effective Date") and will continue until: - (i) Customer ceases all use of the Software, or (ii) ninety (90) days - after all of Customer's active Orders hereunder have expired whether - either party gives notice or not, or (iii) either party terminates this - Agreement as set forth in this Section, provided that the parties' - rights and obligations under the provisions listed in Section 13.4 - shall survive any termination for all of the foregoing subsections - (i) through (iii). This Agreement shall govern: (i) Customer's Free - Licenses and/or Enterprise Licenses whether Customer has entered - into an Order or not, (ii) Orders that reference this Agreement, and - (iii) any POs submitted by Customer as set forth in Section 7.4. The - Subscription Term for each Order shall commence and have the duration - as set forth in the applicable Order. - - 13.2 Subject to Couchbase's rights under Section 3 above, either party - may terminate the applicable Order, if the other party materially - breaches its obligations thereunder and, where such breach is curable, - such breach remains uncured for thirty (30) days following written - notice of the breach. Customer's obligation to make a payment of - any outstanding, unpaid fees for any Enterprise License shall survive - termination or expiration of an Order or this Agreement. - - 13.3 Upon termination or expiration of all active Orders, any Enterprise - Licenses without an Order, and/or this Agreement, Customer will (i) - promptly pay any outstanding fees accrued for any Enterprise Licenses - or as determined under Section 8 hereof, (ii) promptly return or - destroy the Documentation (except as explicitly permitted to be - retained by Customer in an Order) and all copies and portions thereof, - in all forms and types of media, and (iii) uninstall the Software as - described in the then-current Couchbase uninstallation instructions - (located https://docs.couchbase.com/manual/uninstall/). Upon request, - Customer shall provide written certification signed by an officer - of Customer of cessation of use and destruction as set forth herein - within five (5) calendar days of request. - - 13.4 The following sections will survive termination or expiration - of any Order and/or this Agreement: Section 2.3 (Tools, Connectors, - and Third Party Products), Sections 3 (Customer Responsibilities), - 5 (Proprietary Rights), 7 (Fees), 8 (Records Retention and Audit), 9 - (Confidentiality), 10 (Disclaimer of Warranty), 11 (Indemnification - of Third-Party Claims), 12 (Limitation of Liability), 13 (Term and - Termination), and 14 (General). - -14. General. - 14.1 Neither party shall be liable for any delay or failure in - performance (except for any payment obligations by Customer) due to - causes beyond its reasonable control. - - 14.2 Customer agrees that Couchbase may include the Customer's name - and logo in client lists that Couchbase may publish for promotional - purposes from time to time and grants Couchbase a limited license to - its trademark solely for this purpose, provided that Couchbase complies - with Customer's branding guidelines. - - 14.3 Customer may not assign or transfer its rights or obligations under - this Agreement or an Order, in whole or in part, by operation of law or - otherwise, without Couchbase's prior written consent. Any attempt to - assign or otherwise transfer this Agreement or an Order without such - consent will be null and of no effect. Subject to the foregoing, - this Agreement will bind and inure to the benefit of each party's - successors and permitted assigns. - - 14.4 If for any reason a court of competent jurisdiction finds any - provision of this Agreement invalid or unenforceable, that provision - of this Agreement will be enforced to the maximum extent permissible - and the other provisions of this Agreement will remain in full force - and effect. The failure by either party to enforce any provision - of this Agreement will not constitute a waiver of future enforcement - of that or any other provision. All waivers must be in writing and - signed by both parties. - - 14.5 Any notice or communication provided by Couchbase under this - Agreement may be provided by posting a notice on the Couchbase website, - or by mail or email to the relevant address associated with Customer's - account, if available. Any notice or communication provided by Customer - to Couchbase under this Agreement shall be provided to Couchbase by - certified mail, return receipt requested, to Couchbase, Inc., Attn: - Legal Dept, 3250 Olcott Street, Santa Clara, CA 95054, United States, - with a copy emailed to legal@couchbase.com. - - 14.6 This Agreement shall be governed by the laws of the State of - California, U.S.A., excluding its conflicts of law rules. The parties - expressly agree that the UN Convention for the International Sale of - Goods will not apply. Any legal action or proceeding arising under - this Agreement will be brought exclusively in the federal or state - courts located in Santa Clara County, California and the parties - hereby irrevocably consent to the personal jurisdiction and venue - therein. Except as otherwise set forth in this Agreement, Couchbase - may modify this Agreement (including the Support terms) at any time - by (i) posting a revised version on the Couchbase website or (ii) - by otherwise notifying Customer in accordance with Section 14.5, - and by continuing to use the Software after the effective date of any - such modifications to this Agreement, Customer agrees to be bound by - this Agreement, as modified. The date Couchbase last modified this - Agreement is set forth at the end of this Agreement. Notwithstanding - the foregoing, any Orders placed under this version of the Agreement - may only be modified by a mutually signed amendment by the parties. - - 14.7 This Agreement incorporates the Privacy Notice by reference, - and together with each Order constitute the entire agreement and - understanding of the parties and supersedes all prior or contemporaneous - oral or written agreements regarding the subject matter hereof, - including any agreement on confidentiality previously executed by - the parties. Furthermore, no additional or conflicting terms set - forth on any other document shall have any force or effect and are - hereby rejected unless expressly agreed upon by the parties' duly - authorized representatives in writing. To the extent that any terms - and conditions set forth in an Order conflict with the terms of this - Agreement, the applicable terms of the Order shall prevail. - - 14.8 Except as expressly set forth in this Agreement, the exercise by - either party of any of its remedies under this Agreement will be without - prejudice to its other remedies under this Agreement or otherwise. - - 14.9 The parties to this Agreement are independent contractors and - this Agreement will not establish any relationship of partnership, - joint venture, employment, franchise, or agency between the parties. - - 14.10 Neither party will have the power to bind the other or incur - obligations on the other's behalf without the other's prior written - consent. - - 14.11 Customer has not relied on the availability of any future version - of the Software or any future product in making its decision to enter - into this Agreement. - - 14.12 This Agreement may be executed in any number of counterparts, - each of which shall be deemed an original, but all of which together - shall constitute one instrument. Signatures transmitted electronically - or by facsimile shall be deemed original signatures. - - 14.13 This Agreement is applicable both to use of the Software without - a signed Order, but also to use of the Software pursuant to any Order - signed by You or Customer. - ------------------------------------------ - -IMPORTANT CLARIFICATION: From time to time, Couchbase may release -versions of the Software and/or certain features that are not generally -available (the "Non-GA Offering"), including test, alpha, beta, trial, -pre-production, preview and/or developer preview versions or features. If -Customer uses any Non-GA Offering, then the same terms and conditions -of the "Free Licenses" under the Agreement above shall apply, subject -to the modifications below. - -Non-GA Offering Supplemental Terms - -For any Non-GA Offering, the definition of Software in Section 1 of -the Agreement shall be replaced by the following and all references to -"Software" in the applicable provisions shall refer to the below instead: - -"Software" means the applicable Couchbase product, including any Non-GA -Offering, either (i) as reflected in an Order or (ii) as otherwise made -available to Customer under this Agreement. - -Section 2.1 of the Agreement shall be replaced in its entirety by the -following: - -2.1. License Grant. Subject to Customer's compliance with the terms and -conditions of this Agreement and the Documentation, Couchbase hereby -grants to Customer a revocable, non-exclusive, non-transferable, -non-sublicensable, royalty-free, non-fee bearing limited license to -install and use the Non-GA Offering of the Software only for Customer's -own internal non-production use for the purpose of evaluation and -development, if applicable (the "Non-GA Offering License"). By accepting -an invitation to install or use, or by installing or using the Non-GA -Offering, Customer acknowledges and agrees that (i) it has relied upon -its own skill and judgment in electing to use such Non-GA Offering in its -sole discretion, (ii) the Non-GA Offering may not work correctly or in -the same way the final version may work and is offered exclusive of any -warranty (as described in Section 10 of the Agreement) or service level -agreement, (iii) the license granted to Customer with respect to the -Non-GA Offering is revocable and terminable at any time in Couchbase's -sole and absolute discretion, (iv) Couchbase may change, withdraw or -discontinue the Non-GA Offering at any time without notice and is under -no obligation to make such generally available, and (v) Couchbase is -not obligated to provide any services or support, including Support, -Professional Services, updates, patches, enhancements, or fixes (either -in the form of descriptions in the Documentation or on the Couchbase -website or otherwise). Couchbase may use any data, feedback or information -that Customer makes available to Couchbase or that Couchbase derives or -generates from Customer's use of the Non-GA Offering ("Feedback"), and -Customer hereby irrevocably assigns to Couchbase all right, title, and -interest in such Feedback. Customer agrees to the restrictions generally -applicable to the Free Licenses and Software, as applicable, under -this Agreement and any additional requirements set forth by Couchbase -in writing (whether in the Documentation or otherwise) regarding any -particular Non-GA Offering. For the avoidance of doubt, Customer shall -not use any Non-GA Offering in production under any circumstance. - -Section 12 of the Agreement shall be replaced by the following: - -12. Limitation of Liability. -TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL THE -COUCHBASE PARTIES BE LIABLE TO CUSTOMER OR TO ANY THIRD PARTY FOR: (A) -ANY INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL OR EXEMPLARY DAMAGES; -OR (B) THE COST OF PROCURING SUBSTITUTE PRODUCTS OR PROFESSIONAL -SERVICES ARISING OUT OF OR IN ANY WAY RELATING TO THIS AGREEMENT, -OR THE USE OF OR INABILITY TO USE THE SOFTWARE, DOCUMENTATION OR THE -PROFESSIONAL SERVICES; OR (C) DAMAGES OR OTHER LOSSES FOR LOSS OF USE, -LOSS OF BUSINESS, LOSS OF GOODWILL, WORK STOPPAGE, LOST PROFITS, LOSS -OF DATA, COMPUTER FAILURE OR ANY AND ALL OTHER COMMERCIAL DAMAGES OR -LOSSES EVEN IF ADVISED OF THE POSSIBILITY THEREOF AND REGARDLESS OF THE -LEGAL OR EQUITABLE THEORY (CONTRACT, TORT OR OTHERWISE) UPON WHICH THE -CLAIM IS BASED. TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN -NO EVENT WILL THE COUCHBASE PARTIES' AGGREGATE LIABILITY TO CUSTOMER, -FROM ALL CAUSES OF ACTION AND UNDER ALL THEORIES OF LIABILITY EXCEED -ONE HUNDRED DOLLARS (US $100). The parties expressly acknowledge and -agree that Couchbase has set its prices and entered into this Agreement -in reliance upon the limitations of liability specified herein, which -allocate the risk between Couchbase and Customer and form a basis of -the bargain between the parties. - -Section 13.1 of the Agreement shall be replaced by the following: - -13.1 This Agreement is effective as of the date of Customer's acceptance -of this Agreement and will continue until: (i) Customer ceases to use -the Non-GA Offering and/or the Non-GA Offering License, (ii) the Non-GA -Offering becomes generally available (in which case the terms of the -Free License or the Enterprise License (with or without an Order), as -applicable, shall automatically apply to Customer's use of the Software), -(iii) Couchbase discontinues and terminates the Non-GA Offering and/or -Non-GA Offering Licenses, in its sole discretion, or (iv) either party -terminates this Agreement as set forth in this Section, provided that -the parties' rights and obligations of the provisions listed in Section -13.4 shall survive any termination. - -If you have any questions regarding this Agreement, please contact us -at legal@couchbase.com. - -Exhibit A -Additional Terms and Conditions for Couchbase Components and Third -Party Products - -Couchbase Component: Connector to Tableau Software, LLC -Additional Terms and Conditions: www.couchbase.com/toollicenses/tableau - -Product License V11: 20230213 diff --git a/scripts/cbtools/README.txt b/scripts/cbtools/README.txt deleted file mode 100644 index 81bce05a..00000000 --- a/scripts/cbtools/README.txt +++ /dev/null @@ -1,1646 +0,0 @@ -IMPORTANT: -========== -Couchbase Capella users needing support should create a Support ticket in Capella. -Self-managed EE Couchbase Server users needing support should follow the support -for their EE Couchbase Server cluster. - -Tools for Couchbase Capella -=========================== -cbbackupmgr -- utility for backup and restore -cbexport -- utility for exporting data -cbimport -- utility for importing data - -Platform Note -============= -On Windows, you will need to have a recent Microsoft Visual C++ Redistributable -already installed. You can get them from here if necessary: -https://docs.microsoft.com/en-us/cpp/windows/latest-supported-vc-redist?view=msvc-170 - -Version -======= -To determine the version of the utility, use the --version option. - -Tools Documentation -=================== -See Couchbase Server documentation for the utilities: - -cbbackupmgr: - https://docs.couchbase.com/server/current/backup-restore/cbbackupmgr.html - -cbexport: - https://docs.couchbase.com/server/current/tools/cbexport.html - https://docs.couchbase.com/server/current/tools/cbexport-json.html - -cbimport: - https://docs.couchbase.com/server/current/tools/cbimport.html - https://docs.couchbase.com/server/current/tools/cbimport-csv.html - https://docs.couchbase.com/server/current/tools/cbimport-json.html - -For documentation on older versions, reference that version in the -documentation. To determine the version of the utility, use the --version -option. - -Note: -===== -When specifying the Couchbase Capella cluster for a command using the --c/--cluster flag, use the Wide Area Network URL (DNS SRV record) given in the -Couchbase Capella cluster connection info with the "couchbases://" prefix: - -Example: -c couchbases://cb.xxxxxxxx.cloud.couchbase.com - - -DOCUMENTATION - -The online documentation library is available at: -https://docs.couchbase.com/home/index.html - -THIRD-PARTY LICENSE INFORMATION - -Couchbase uses third-party libraries or other resources that may be -distributed under licenses different from the one that governs Couchbase -Server. You can find the licenses for each third-party component below, in the -accompanying license file(s), and/or at: -https://www.couchbase.com/legal/agreements - -Couchbase Server includes the libgcc, libstdc++, and libgomp runtime -libraries, which are made available under the GNU General Public License -v3.0 w/GCC Runtime Library exception. The source code for each is part -of the GNU Compiler Collection, or GCC, which is available for download -at: https://gcc.gnu.org/ - -Couchbase Server also includes libnuma1 which is made available under the GNU -Lesser General Public License v2.1. The source code for libnuma1 is part of -the numactl project, which is available here: -https://github.com/numactl/numactl - -Couchbase Server also includes OpenJDK which is made available under the GNU -General Public License, version 2, with the Classpath Exception. The source -code for OpenJDK is available here: https://github.com/openjdk/ - - -1. License Notice for libgcc, libstdc++, and libgomp libraries --------------------------------------------------------------- - -GNU GENERAL PUBLIC LICENSE -========================== - - -Version 3, 29 June 2007 - -Copyright (C) 2007 Free Software Foundation, Inc. - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - - -Preamble - -The GNU General Public License is a free, copyleft license for software and -other kinds of works. - -The licenses for most software and other practical works are designed to take -away your freedom to share and change the works. By contrast, the GNU General -Public License is intended to guarantee your freedom to share and change all -versions of a program--to make sure it remains free software for all its -users. We, the Free Software Foundation, use the GNU General Public License -for most of our software; it applies also to any other work released this way -by its authors. You can apply it to your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for them if you wish), that you -receive source code or can get it if you want it, that you can change the -software or use pieces of it in new free programs, and that you know you can -do these things. - -To protect your rights, we need to prevent others from denying you these -rights or asking you to surrender the rights. Therefore, you have certain -responsibilities if you distribute copies of the software, or if you modify -it: responsibilities to respect the freedom of others. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must pass on to the recipients the same freedoms that you received. -You must make sure that they, too, receive or can get the source code. And you -must show them these terms so they know their rights. - -Developers that use the GNU GPL protect your rights with two steps: (1) assert -copyright on the software, and (2) offer you this License giving you legal -permission to copy, distribute and/or modify it. - -For the developers' and authors' protection, the GPL clearly explains that -there is no warranty for this free software. For both users' and authors' -sake, the GPL requires that modified versions be marked as changed, so that -their problems will not be attributed erroneously to authors of previous -versions. - -Some devices are designed to deny users access to install or run modified -versions of the software inside them, although the manufacturer can do so. -This is fundamentally incompatible with the aim of protecting users' freedom -to change the software. The systematic pattern of such abuse occurs in the -area of products for individuals to use, which is precisely where it is most -unacceptable. Therefore, we have designed this version of the GPL to prohibit -the practice for those products. If such problems arise substantially in other -domains, we stand ready to extend this provision to those domains in future -versions of the GPL, as needed to protect the freedom of users. - -Finally, every program is threatened constantly by software patents. States -should not allow patents to restrict development and use of software on -general-purpose computers, but in those that do, we wish to avoid the special -danger that patents applied to a free program could make it effectively -proprietary. To prevent this, the GPL assures that patents cannot be used to -render the program non-free. - -The precise terms and conditions for copying, distribution and modification -follow. - - -TERMS AND CONDITIONS - -0. Definitions. - -“This License” refers to version 3 of the GNU General Public License. - -“Copyright” also means copyright-like laws that apply to other kinds of works, -such as semiconductor masks. - -“The Program” refers to any copyrightable work licensed under this License. -Each licensee is addressed as “you”. “Licensees” and “recipients” may be -individuals or organizations. - -To “modify” a work means to copy from or adapt all or part of the work in a -fashion requiring copyright permission, other than the making of an exact -copy. The resulting work is called a “modified version” of the earlier work or -a work“based on” the earlier work. - -A “covered work” means either the unmodified Program or a work based on the -Program. - -To “propagate” a work means to do anything with it that, without permission, -would make you directly or secondarily liable for infringement under -applicable copyright law, except executing it on a computer or modifying a -private copy. Propagation includes copying, distribution (with or without -modification), making available to the public, and in some countries other -activities as well. - -To “convey” a work means any kind of propagation that enables other parties to -make or receive copies. Mere interaction with a user through a computer -network, with no transfer of a copy, is not conveying. - -An interactive user interface displays “Appropriate Legal Notices” to the -extent that it includes a convenient and prominently visible feature that (1) -displays an appropriate copyright notice, and (2) tells the user that there is -no warranty for the work (except to the extent that warranties are provided), -that licensees may convey the work under this License, and how to view a copy -of this License. If the interface presents a list of user commands or options, -such as a menu, a prominent item in the list meets this criterion. - -1. Source Code. - -The “source code” for a work means the preferred form of the work for making -modifications to it. “Object code” means any non-source form of a work. - -A “Standard Interface” means an interface that either is an official standard -defined by a recognized standards body, or, in the case of interfaces -specified for a particular programming language, one that is widely used among -developers working in that language. - -The “System Libraries” of an executable work include anything, other than the -work as a whole, that (a) is included in the normal form of packaging a Major -Component, but which is not part of that Major Component, and (b) serves only -to enable use of the work with that Major Component, or to implement a -Standard Interface for which an implementation is available to the public in -source code form. A “Major Component”, in this context, means a major -essential component (kernel, window system, and so on) of the specific -operating system (if any) on which the executable work runs, or a compiler -used to produce the work, or an object code interpreter used to run it. - -The “Corresponding Source” for a work in object code form means all the source -code needed to generate, install, and (for an executable work) run the object -code and to modify the work, including scripts to control those activities. -However, it does not include the work's System Libraries, or general-purpose -tools or generally available free programs which are used unmodified in -performing those activities but which are not part of the work. For example, -Corresponding Source includes interface definition files associated with -source files for the work, and the source code for shared libraries and -dynamically linked subprograms that the work is specifically designed to -require, such as by intimate data communication or control flow between those -subprograms and other parts of the work. - -The Corresponding Source need not include anything that users can regenerate -automatically from other parts of the Corresponding Source. - -The Corresponding Source for a work in source code form is that same work. - -2. Basic Permissions. - -All rights granted under this License are granted for the term of copyright on -the Program, and are irrevocable provided the stated conditions are met. This -License explicitly affirms your unlimited permission to run the unmodified -Program. The output from running a covered work is covered by this License -only if the output, given its content, constitutes a covered work. This -License acknowledges your rights of fair use or other equivalent, as provided -by copyright law. - -You may make, run and propagate covered works that you do not convey, without -conditions so long as your license otherwise remains in force. You may convey -covered works to others for the sole purpose of having them make modifications -exclusively for you, or provide you with facilities for running those works, -provided that you comply with the terms of this License in conveying all -material for which you do not control copyright. Those thus making or running -the covered works for you must do so exclusively on your behalf, under your -direction and control, on terms that prohibit them from making any copies of -your copyrighted material outside their relationship with you. - -Conveying under any other circumstances is permitted solely under the -conditions stated below. Sublicensing is not allowed; section 10 makes it -unnecessary. - -3. Protecting Users' Legal Rights From Anti-Circumvention Law. - -No covered work shall be deemed part of an effective technological measure -under any applicable law fulfilling obligations under article 11 of the WIPO -copyright treaty adopted on 20 December 1996, or similar laws prohibiting or -restricting circumvention of such measures. - -When you convey a covered work, you waive any legal power to forbid -circumvention of technological measures to the extent such circumvention is -effected by exercising rights under this License with respect to the covered -work, and you disclaim any intention to limit operation or modification of the -work as a means of enforcing, against the work's users, your or third parties' -legal rights to forbid circumvention of technological measures. - -4. Conveying Verbatim Copies. - -You may convey verbatim copies of the Program's source code as you receive it, -in any medium, provided that you conspicuously and appropriately publish on -each copy an appropriate copyright notice; keep intact all notices stating -that this License and any non-permissive terms added in accord with section 7 -apply to the code; keep intact all notices of the absence of any warranty; and -give all recipients a copy of this License along with the Program. - -You may charge any price or no price for each copy that you convey, and you -may offer support or warranty protection for a fee. - -5. Conveying Modified Source Versions. - -You may convey a work based on the Program, or the modifications to produce it -from the Program, in the form of source code under the terms of section 4, -provided that you also meet all of these conditions: - - * a) The work must carry prominent notices stating that you modified it, and - giving a relevant date. - - * b) The work must carry prominent notices stating that it is released under - this License and any conditions added under section 7. This requirement - modifies the requirement in section 4 to “keep intact all notices”. - - * c) You must license the entire work, as a whole, under this License to - anyone who comes into possession of a copy. This License will therefore - apply, along with any applicable section 7 additional terms, to the whole - of the work, and all its parts, regardless of how they are packaged. This - License gives no permission to license the work in any other way, but it - does not invalidate such permission if you have separately received it. - - * d) If the work has interactive user interfaces, each must display - Appropriate Legal Notices; however, if the Program has interactive - interfaces that do not display Appropriate Legal Notices, your work need - not make them do so. - -A compilation of a covered work with other separate and independent works, -which are not by their nature extensions of the covered work, and which are -not combined with it such as to form a larger program, in or on a volume of a -storage or distribution medium, is called an “aggregate” if the compilation -and its resulting copyright are not used to limit the access or legal rights -of the compilation's users beyond what the individual works permit. Inclusion -of a covered work in an aggregate does not cause this License to apply to the -other parts of the aggregate. - -6. Conveying Non-Source Forms. - -You may convey a covered work in object code form under the terms of sections -4 and 5, provided that you also convey the machine-readable Corresponding -Source under the terms of this License, in one of these ways: - - * a) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by the - Corresponding Source fixed on a durable physical medium customarily used - for software interchange. - - * b) Convey the object code in, or embodied in, a physical product - (including a physical distribution medium), accompanied by a written - offer, valid for at least three years and valid for as long as you offer - spare parts or customer support for that product model, to give anyone who - possesses the object code either (1) a copy of the Corresponding Source - for all the software in the product that is covered by this License, on a - durable physical medium customarily used for software interchange, for a - price no more than your reasonable cost of physically performing this - conveying of source, or (2) access to copy the Corresponding Source from a - network server at no charge. - - * c) Convey individual copies of the object code with a copy of the written - offer to provide the Corresponding Source. This alternative is allowed - only occasionally and noncommercially, and only if you received the object - code with such an offer, in accord with subsection 6b. - - * d) Convey the object code by offering access from a designated place - (gratis or for a charge), and offer equivalent access to the Corresponding - Source in the same way through the same place at no further charge. You - need not require recipients to copy the Corresponding Source along with - the object code. If the place to copy the object code is a network server, - the Corresponding Source may be on a different server (operated by you or - a third party) that supports equivalent copying facilities, provided you - maintain clear directions next to the object code saying where to find the - Corresponding Source. Regardless of what server hosts the Corresponding - Source, you remain obligated to ensure that it is available for as long as - needed to satisfy these requirements. - - * e) Convey the object code using peer-to-peer transmission, provided you - inform other peers where the object code and Corresponding Source of the - work are being offered to the general public at no charge under subsection - 6d. - -A separable portion of the object code, whose source code is excluded from the -Corresponding Source as a System Library, need not be included in conveying -the object code work. - -A “User Product” is either (1) a “consumer product”, which means any tangible -personal property which is normally used for personal, family, or household -purposes, or (2) anything designed or sold for incorporation into a dwelling. -In determining whether a product is a consumer product, doubtful cases shall -be resolved in favor of coverage. For a particular product received by a -particular user, “normally used” refers to a typical or common use of that -class of product, regardless of the status of the particular user or of the -way in which the particular user actually uses, or expects or is expected to -use, the product. A product is a consumer product regardless of whether the -product has substantial commercial, industrial or non-consumer uses, unless -such uses represent the only significant mode of use of the product. - -“Installation Information” for a User Product means any methods, procedures, -authorization keys, or other information required to install and execute -modified versions of a covered work in that User Product from a modified -version of its Corresponding Source. The information must suffice to ensure -that the continued functioning of the modified object code is in no case -prevented or interfered with solely because modification has been made. - -If you convey an object code work under this section in, or with, or -specifically for use in, a User Product, and the conveying occurs as part of a -transaction in which the right of possession and use of the User Product is -transferred to the recipient in perpetuity or for a fixed term (regardless of -how the transaction is characterized), the Corresponding Source conveyed under -this section must be accompanied by the Installation Information. But this -requirement does not apply if neither you nor any third party retains the -ability to install modified object code on the User Product (for example, the -work has been installed in ROM). - -The requirement to provide Installation Information does not include a -requirement to continue to provide support service, warranty, or updates for a -work that has been modified or installed by the recipient, or for the User -Product in which it has been modified or installed. Access to a network may be -denied when the modification itself materially and adversely affects the -operation of the network or violates the rules and protocols for communication -across the network. - -Corresponding Source conveyed, and Installation Information provided, in -accord with this section must be in a format that is publicly documented (and -with an implementation available to the public in source code form), and must -require no special password or key for unpacking, reading or copying. - -7. Additional Terms. - -“Additional permissions” are terms that supplement the terms of this License -by making exceptions from one or more of its conditions. Additional -permissions that are applicable to the entire Program shall be treated as -though they were included in this License, to the extent that they are valid -under applicable law. If additional permissions apply only to part of the -Program, that part may be used separately under those permissions, but the -entire Program remains governed by this License without regard to the -additional permissions. - -When you convey a copy of a covered work, you may at your option remove any -additional permissions from that copy, or from any part of it. (Additional -permissions may be written to require their own removal in certain cases when -you modify the work.) You may place additional permissions on material, added -by you to a covered work, for which you have or can give appropriate copyright -permission. - -Notwithstanding any other provision of this License, for material you add to a -covered work, you may (if authorized by the copyright holders of that -material) supplement the terms of this License with terms: - - * a) Disclaiming warranty or limiting liability differently from the terms - of sections 15 and 16 of this License; or - - * b) Requiring preservation of specified reasonable legal notices or author - attributions in that material or in the Appropriate Legal Notices - displayed by works containing it; or - - * c) Prohibiting misrepresentation of the origin of that material, or - requiring that modified versions of such material be marked in reasonable - ways as different from the original version; or - - * d) Limiting the use for publicity purposes of names of licensors or - authors of the material; or - - * e) Declining to grant rights under trademark law for use of some trade - names, trademarks, or service marks; or - - * f) Requiring indemnification of licensors and authors of that material by - anyone who conveys the material (or modified versions of it) with - contractual assumptions of liability to the recipient, for any liability - that these contractual assumptions directly impose on those licensors and - authors. - -All other non-permissive additional terms are considered “further -restrictions”within the meaning of section 10. If the Program as you received -it, or any part of it, contains a notice stating that it is governed by this -License along with a term that is a further restriction, you may remove that -term. If a license document contains a further restriction but permits -relicensing or conveying under this License, you may add to a covered work -material governed by the terms of that license document, provided that the -further restriction does not survive such relicensing or conveying. - -If you add terms to a covered work in accord with this section, you must -place, in the relevant source files, a statement of the additional terms that -apply to those files, or a notice indicating where to find the applicable -terms. - -Additional terms, permissive or non-permissive, may be stated in the form of a -separately written license, or stated as exceptions; the above requirements -apply either way. - -8. Termination. - -You may not propagate or modify a covered work except as expressly provided -under this License. Any attempt otherwise to propagate or modify it is void, -and will automatically terminate your rights under this License (including any -patent licenses granted under the third paragraph of section 11). - -However, if you cease all violation of this License, then your license from a -particular copyright holder is reinstated (a) provisionally, unless and until -the copyright holder explicitly and finally terminates your license, and (b) -permanently, if the copyright holder fails to notify you of the violation by -some reasonable means prior to 60 days after the cessation. - -Moreover, your license from a particular copyright holder is reinstated -permanently if the copyright holder notifies you of the violation by some -reasonable means, this is the first time you have received notice of violation -of this License (for any work) from that copyright holder, and you cure the -violation prior to 30 days after your receipt of the notice. - -Termination of your rights under this section does not terminate the licenses -of parties who have received copies or rights from you under this License. If -your rights have been terminated and not permanently reinstated, you do not -qualify to receive new licenses for the same material under section 10. - -9. Acceptance Not Required for Having Copies. - -You are not required to accept this License in order to receive or run a copy -of the Program. Ancillary propagation of a covered work occurring solely as a -consequence of using peer-to-peer transmission to receive a copy likewise does -not require acceptance. However, nothing other than this License grants you -permission to propagate or modify any covered work. These actions infringe -copyright if you do not accept this License. Therefore, by modifying or -propagating a covered work, you indicate your acceptance of this License to do -so. - -10. Automatic Licensing of Downstream Recipients. - -Each time you convey a covered work, the recipient automatically receives a -license from the original licensors, to run, modify and propagate that work, -subject to this License. You are not responsible for enforcing compliance by -third parties with this License. - -An “entity transaction” is a transaction transferring control of an -organization, or substantially all assets of one, or subdividing an -organization, or merging organizations. If propagation of a covered work -results from an entity transaction, each party to that transaction who -receives a copy of the work also receives whatever licenses to the work the -party's predecessor in interest had or could give under the previous -paragraph, plus a right to possession of the Corresponding Source of the work -from the predecessor in interest, if the predecessor has it or can get it with -reasonable efforts. - -You may not impose any further restrictions on the exercise of the rights -granted or affirmed under this License. For example, you may not impose a -license fee, royalty, or other charge for exercise of rights granted under -this License, and you may not initiate litigation (including a cross-claim or -counterclaim in a lawsuit) alleging that any patent claim is infringed by -making, using, selling, offering for sale, or importing the Program or any -portion of it. - -11. Patents. - -A “contributor” is a copyright holder who authorizes use under this License of -the Program or a work on which the Program is based. The work thus licensed is -called the contributor's “contributor version”. - -A contributor's “essential patent claims” are all patent claims owned or -controlled by the contributor, whether already acquired or hereafter acquired, -that would be infringed by some manner, permitted by this License, of making, -using, or selling its contributor version, but do not include claims that -would be infringed only as a consequence of further modification of the -contributor version. For purposes of this definition, “control” includes the -right to grant patent sublicenses in a manner consistent with the requirements -of this License. - -Each contributor grants you a non-exclusive, worldwide, royalty-free patent -license under the contributor's essential patent claims, to make, use, sell, -offer for sale, import and otherwise run, modify and propagate the contents of -its contributor version. - -In the following three paragraphs, a “patent license” is any express agreement -or commitment, however denominated, not to enforce a patent (such as an -express permission to practice a patent or covenant not to sue for patent -infringement). To “grant” such a patent license to a party means to make such -an agreement or commitment not to enforce a patent against the party. - -If you convey a covered work, knowingly relying on a patent license, and the -Corresponding Source of the work is not available for anyone to copy, free of -charge and under the terms of this License, through a publicly available -network server or other readily accessible means, then you must either (1) -cause the Corresponding Source to be so available, or (2) arrange to deprive -yourself of the benefit of the patent license for this particular work, or (3) -arrange, in a manner consistent with the requirements of this License, to -extend the patent license to downstream recipients. “Knowingly relying” means -you have actual knowledge that, but for the patent license, your conveying the -covered work in a country, or your recipient's use of the covered work in a -country, would infringe one or more identifiable patents in that country that -you have reason to believe are valid. - -If, pursuant to or in connection with a single transaction or arrangement, you -convey, or propagate by procuring conveyance of, a covered work, and grant a -patent license to some of the parties receiving the covered work authorizing -them to use, propagate, modify or convey a specific copy of the covered work, -then the patent license you grant is automatically extended to all recipients -of the covered work and works based on it. - -A patent license is “discriminatory” if it does not include within the scope -of its coverage, prohibits the exercise of, or is conditioned on the -non-exercise of one or more of the rights that are specifically granted under -this License. You may not convey a covered work if you are a party to an -arrangement with a third party that is in the business of distributing -software, under which you make payment to the third party based on the extent -of your activity of conveying the work, and under which the third party -grants, to any of the parties who would receive the covered work from you, a -discriminatory patent license (a) in connection with copies of the covered -work conveyed by you (or copies made from those copies), or (b) primarily for -and in connection with specific products or compilations that contain the -covered work, unless you entered into that arrangement, or that patent license -was granted, prior to 28 March 2007. - -Nothing in this License shall be construed as excluding or limiting any -implied license or other defenses to infringement that may otherwise be -available to you under applicable patent law. - -12. No Surrender of Others' Freedom. - -If conditions are imposed on you (whether by court order, agreement or -otherwise) that contradict the conditions of this License, they do not excuse -you from the conditions of this License. If you cannot convey a covered work -so as to satisfy simultaneously your obligations under this License and any -other pertinent obligations, then as a consequence you may not convey it at -all. For example, if you agree to terms that obligate you to collect a royalty -for further conveying from those to whom you convey the Program, the only way -you could satisfy both those terms and this License would be to refrain -entirely from conveying the Program. - -13. Use with the GNU Affero General Public License. - -Notwithstanding any other provision of this License, you have permission to -link or combine any covered work with a work licensed under version 3 of the -GNU Affero General Public License into a single combined work, and to convey -the resulting work. The terms of this License will continue to apply to the -part which is the covered work, but the special requirements of the GNU Affero -General Public License, section 13, concerning interaction through a network -will apply to the combination as such. - -14. Revised Versions of this License. - -The Free Software Foundation may publish revised and/or new versions of the -GNU General Public License from time to time. Such new versions will be -similar in spirit to the present version, but may differ in detail to address -new problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies that a certain numbered version of the GNU General Public License -“or any later version” applies to it, you have the option of following the -terms and conditions either of that numbered version or of any later version -published by the Free Software Foundation. If the Program does not specify a -version number of the GNU General Public License, you may choose any version -ever published by the Free Software Foundation. - -If the Program specifies that a proxy can decide which future versions of the -GNU General Public License can be used, that proxy's public statement of -acceptance of a version permanently authorizes you to choose that version for -the Program. - -Later license versions may give you additional or different permissions. -However, no additional obligations are imposed on any author or copyright -holder as a result of your choosing to follow a later version. - -15. Disclaimer of Warranty. - -THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE -LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR -OTHER PARTIES PROVIDE THE PROGRAM “AS IS” WITHOUT WARRANTY OF ANY KIND, EITHER -EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF -MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO -THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM -PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR -CORRECTION. - -16. Limitation of Liability. - -IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY -COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM -AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, -SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR -INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR -DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR -A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH -HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -17. Interpretation of Sections 15 and 16. - -If the disclaimer of warranty and limitation of liability provided above -cannot be given local legal effect according to their terms, reviewing courts -shall apply local law that most closely approximates an absolute waiver of all -civil liability in connection with the Program, unless a warranty or -assumption of liability accompanies a copy of the Program in return for a fee. - -END OF TERMS AND CONDITIONS - - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively state the exclusion -of warranty; and each file should have at least the “copyright” line and a -pointer to where the full notice is found. - - - Copyright (C) - -This program is free software: you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation, either version 3 of the License, or (at your option) any later -version. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program. If not, see . - -Also add information on how to contact you by electronic and paper mail. - -If the program does terminal interaction, make it output a short notice like -this when it starts in an interactive mode: - - Copyright (C) - -This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'. -This is free software, and you are welcome to redistribute it under certain -conditions; type `show c' for details. - -The hypothetical commands `show w' and `show c' should show the appropriate -parts of the General Public License. Of course, your program's commands might -be different; for a GUI interface, you would use an “about box”. - -You should also get your employer (if you work as a programmer) or school, if -any, to sign a “copyright disclaimer” for the program, if necessary. For more -information on this, and how to apply and follow the GNU GPL, see -. - -The GNU General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Lesser General Public -License instead of this License. But first, please read -. - - -GCC RUNTIME LIBRARY EXCEPTION (for libgcc, libstdc++, and libgomp) -============================= - -Version 3.1, 31 March 2009 - -Copyright © 2009 Free Software Foundation, Inc. - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -This GCC Runtime Library Exception ("Exception") is an additional permission -under section 7 of the GNU General Public License, version 3 ("GPLv3"). It -applies to a given file (the "Runtime Library") that bears a notice placed by -the copyright holder of the file stating that the file is governed by GPLv3 -along with this Exception. - -When you use GCC to compile a program, GCC may combine portions of certain GCC -header files and runtime libraries with the compiled program. The purpose of -this Exception is to allow compilation of non-GPL (including proprietary) -programs to use, in this way, the header files and runtime libraries covered -by this Exception. - -0. Definitions. - -A file is an "Independent Module" if it either requires the Runtime -Library for execution after a Compilation Process, or makes use of an -interface provided by the Runtime Library, but is not otherwise based on -the Runtime Library. - -"GCC" means a version of the GNU Compiler Collection, with or without -modifications, governed by version 3 (or a specified later version) of -the GNU General Public License (GPL) with the option of using any -subsequent versions published by the FSF. - -"GPL-compatible Software" is software whose conditions of propagation, -modification and use would permit combination with GCC in accord with -the license of GCC. - -"Target Code" refers to output from any compiler for a real or virtual -target processor architecture, in executable form or suitable for input -to an assembler, loader, linker and/or execution phase. Notwithstanding -that, Target Code does not include data in any format that is used as a -compiler intermediate representation, or used for producing a compiler -intermediate representation. - -The "Compilation Process" transforms code entirely represented in -non-intermediate languages designed for human-written code, and/or in -Java Virtual Machine byte code, into Target Code. Thus, for example, use -of source code generators and preprocessors need not be considered part -of the Compilation Process, since the Compilation Process can be -understood as starting with the output of the generators or -preprocessors. - -A Compilation Process is "Eligible" if it is done using GCC, alone or -with other GPL-compatible software, or if it is done without using any -work based on GCC. For example, using non-GPL-compatible Software to -optimize any GCC intermediate representations would not qualify as an -Eligible Compilation Process. - -1. Grant of Additional Permission. - -You have permission to propagate a work of Target Code formed by -combining the Runtime Library with Independent Modules, even if such -propagation would otherwise violate the terms of GPLv3, provided that -all Target Code was generated by Eligible Compilation Processes. You may -then convey such a combination under terms of your choice, consistent -with the licensing of the Independent Modules. - -2. No Weakening of GCC Copyleft. - -The availability of this Exception does not imply any general -presumption that third-party software is unaffected by the copyleft -requirements of the license of GCC. - - -2. License notice for libnuma1 library --------------------------------------- - -GNU Lesser General Public License -================================= - -Version 2.1, February 1999 - - Copyright (C) 1991, 1999 Free Software Foundation, Inc. - - 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - - Everyone is permitted to copy and distribute verbatim copies of this - license document, but changing it is not allowed. - - [This is the first released version of the Lesser GPL. It also counts as - the successor of the GNU Library Public License, version 2, hence the - version number 2.1.] - - -Preamble --------- - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public Licenses are intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. - -This license, the Lesser General Public License, applies to some specially -designated software packages--typically libraries--of the Free Software -Foundation and other authors who decide to use it. You can use it too, but we -suggest you first think carefully about whether this license or the ordinary -General Public License is the better strategy to use in any particular case, -based on the explanations below. - -When we speak of free software, we are referring to freedom of use, not price. -Our General Public Licenses are designed to make sure that you have the -freedom to distribute copies of free software (and charge for this service if -you wish); that you receive source code or can get it if you want it; that you -can change the software and use pieces of it in new free programs; and that -you are informed that you can do these things. - -To protect your rights, we need to make restrictions that forbid distributors -to deny you these rights or to ask you to surrender these rights. These -restrictions translate to certain responsibilities for you if you distribute -copies of the library or if you modify it. - -For example, if you distribute copies of the library, whether gratis or for a -fee, you must give the recipients all the rights that we gave you. You must -make sure that they, too, receive or can get the source code. If you link -other code with the library, you must provide complete object files to the -recipients, so that they can relink them with the library after making changes -to the library and recompiling it. And you must show them these terms so they -know their rights. - -We protect your rights with a two-step method: (1) we copyright the library, -and (2) we offer you this license, which gives you legal permission to copy, -distribute and/or modify the library. - -To protect each distributor, we want to make it very clear that there is no -warranty for the free library. Also, if the library is modified by someone -else and passed on, the recipients should know that what they have is not the -original version, so that the original author's reputation will not be -affected by problems that might be introduced by others. - -Finally, software patents pose a constant threat to the existence of any free -program. We wish to make sure that a company cannot effectively restrict the -users of a free program by obtaining a restrictive license from a patent -holder. Therefore, we insist that any patent license obtained for a version of -the library must be consistent with the full freedom of use specified in this -license. - -Most GNU software, including some libraries, is covered by the ordinary GNU -General Public License. This license, the GNU Lesser General Public License, -applies to certain designated libraries, and is quite different from the -ordinary General Public License. We use this license for certain libraries in -order to permit linking those libraries into non-free programs. - -When a program is linked with a library, whether statically or using a shared -library, the combination of the two is legally speaking a combined work, a -derivative of the original library. The ordinary General Public License -therefore permits such linking only if the entire combination fits its -criteria of freedom. The Lesser General Public License permits more lax -criteria for linking other code with the library. - -We call this license the "Lesser" General Public License because it does Less -to protect the user's freedom than the ordinary General Public License. It -also provides other free software developers Less of an advantage over -competing non-free programs. These disadvantages are the reason we use the -ordinary General Public License for many libraries. However, the Lesser -license provides advantages in certain special circumstances. - -For example, on rare occasions, there may be a special need to encourage the -widest possible use of a certain library, so that it becomes a de-facto -standard. To achieve this, non-free programs must be allowed to use the -library. A more frequent case is that a free library does the same job as -widely used non-free libraries. In this case, there is little to gain by -limiting the free library to free software only, so we use the Lesser General -Public License. - -In other cases, permission to use a particular library in non-free programs -enables a greater number of people to use a large body of free software. For -example, permission to use the GNU C Library in non-free programs enables many -more people to use the whole GNU operating system, as well as its variant, the -GNU/Linux operating system. - -Although the Lesser General Public License is Less protective of the users' -freedom, it does ensure that the user of a program that is linked with the -Library has the freedom and the wherewithal to run that program using a -modified version of the Library. - -The precise terms and conditions for copying, distribution and modification -follow. Pay close attention to the difference between a "work based on the -library" and a "work that uses the library". The former contains code derived -from the library, whereas the latter must be combined with the library in -order to run. - - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION ---------------------------------------------------------------- - -0. This License Agreement applies to any software library or other program -which contains a notice placed by the copyright holder or other authorized -party saying it may be distributed under the terms of this Lesser General -Public License (also called "this License"). Each licensee is addressed as -"you". - -A "library" means a collection of software functions and/or data prepared so -as to be conveniently linked with application programs (which use some of -those functions and data) to form executables. - -The "Library", below, refers to any such software library or work which has -been distributed under these terms. A "work based on the Library" means either -the Library or any derivative work under copyright law: that is to say, a work -containing the Library or a portion of it, either verbatim or with -modifications and/or translated straightforwardly into another language. -(Hereinafter, translation is included without limitation in the term -"modification".) - -"Source code" for a work means the preferred form of the work for making -modifications to it. For a library, complete source code means all the source -code for all modules it contains, plus any associated interface definition -files, plus the scripts used to control compilation and installation of the -library. - -Activities other than copying, distribution and modification are not covered -by this License; they are outside its scope. The act of running a program -using the Library is not restricted, and output from such a program is covered -only if its contents constitute a work based on the Library (independent of -the use of the Library in a tool for writing it). Whether that is true depends -on what the Library does and what the program that uses the Library does. - -1. You may copy and distribute verbatim copies of the Library's complete -source code as you receive it, in any medium, provided that you conspicuously -and appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and distribute a copy of this License -along with the Library. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Library or any portion of it, -thus forming a work based on the Library, and copy and distribute such -modifications or work under the terms of Section 1 above, provided that you -also meet all of these conditions: - - a) The modified work must itself be a software library. - - b) You must cause the files modified to carry prominent notices stating - that you changed the files and the date of any change. - - c) You must cause the whole of the work to be licensed at no charge to - all third parties under the terms of this License. - - d) If a facility in the modified Library refers to a function or a table - of data to be supplied by an application program that uses the facility, - other than as an argument passed when the facility is invoked, then you - must make a good faith effort to ensure that, in the event an - application does not supply such function or table, the facility still - operates, and performs whatever part of its purpose remains meaningful. - - (For example, a function in a library to compute square roots has a - purpose that is entirely well-defined independent of the application. - Therefore, Subsection 2d requires that any application-supplied function - or table used by this function must be optional: if the application does - not supply it, the square root function must still compute square - roots.) - - These requirements apply to the modified work as a whole. If - identifiable sections of that work are not derived from the Library, and - can be reasonably considered independent and separate works in - themselves, then this License, and its terms, do not apply to those - sections when you distribute them as separate works. But when you - distribute the same sections as part of a whole which is a work based on - the Library, the distribution of the whole must be on the terms of this - License, whose permissions for other licensees extend to the entire - whole, and thus to each and every part regardless of who wrote it. - - Thus, it is not the intent of this section to claim rights or contest - your rights to work written entirely by you; rather, the intent is to - exercise the right to control the distribution of derivative or - collective works based on the Library. - - In addition, mere aggregation of another work not based on the Library - with the Library (or with a work based on the Library) on a volume of a - storage or distribution medium does not bring the other work under the - scope of this License. - -3. You may opt to apply the terms of the ordinary GNU General Public License -instead of this License to a given copy of the Library. To do this, you must -alter all the notices that refer to this License, so that they refer to the -ordinary GNU General Public License, version 2, instead of to this License. -(If a newer version than version 2 of the ordinary GNU General Public License -has appeared, then you can specify that version instead if you wish.) Do not -make any other change in these notices. - -Once this change is made in a given copy, it is irreversible for that copy, so -the ordinary GNU General Public License applies to all subsequent copies and -derivative works made from that copy. - -This option is useful when you wish to copy part of the code of the Library -into a program that is not a library. - -4. You may copy and distribute the Library (or a portion or derivative of it, -under Section 2) in object code or executable form under the terms of Sections -1 and 2 above provided that you accompany it with the complete corresponding -machine-readable source code, which must be distributed under the terms of -Sections 1 and 2 above on a medium customarily used for software interchange. - -If distribution of object code is made by offering access to copy from a -designated place, then offering equivalent access to copy the source code from -the same place satisfies the requirement to distribute the source code, even -though third parties are not compelled to copy the source along with the -object code. - -5. A program that contains no derivative of any portion of the Library, but is -designed to work with the Library by being compiled or linked with it, is -called a "work that uses the Library". Such a work, in isolation, is not a -derivative work of the Library, and therefore falls outside the scope of this -License. - -However, linking a "work that uses the Library" with the Library creates an -executable that is a derivative of the Library (because it contains portions -of the Library), rather than a "work that uses the library". The executable is -therefore covered by this License. Section 6 states terms for distribution of -such executables. - -When a "work that uses the Library" uses material from a header file that is -part of the Library, the object code for the work may be a derivative work ofapply -the Library even though the source code is not. Whether this is true is -especially significant if the work can be linked without the Library, or if -the work is itself a library. The threshold for this to be true is not -precisely defined by law. - -If such an object file uses only numerical parameters, data structure layouts -and accessors, and small macros and small inline functions (ten lines or less -in length), then the use of the object file is unrestricted, regardless of -whether it is legally a derivative work. (Executables containing this object -code plus portions of the Library will still fall under Section 6.) - -Otherwise, if the work is a derivative of the Library, you may distribute the -object code for the work under the terms of Section 6. Any executables -containing that work also fall under Section 6, whether or not they are linked -directly with the Library itself. - -6. As an exception to the Sections above, you may also combine or link a "work -that uses the Library" with the Library to produce a work containing portions -of the Library, and distribute that work under terms of your choice, provided -that the terms permit modification of the work for the customer's own use and -reverse engineering for debugging such modifications. - -You must give prominent notice with each copy of the work that the Library is -used in it and that the Library and its use are covered by this License. You -must supply a copy of this License. If the work during execution displays -copyright notices, you must include the copyright notice for the Library among -them, as well as a reference directing the user to the copy of this License. -Also, you must do one of these things: - - a) Accompany the work with the complete corresponding machine-readable - source code for the Library including whatever changes were used in the - work (which must be distributed under Sections 1 and 2 above); and, if - the work is an executable linked with the Library, with the complete - machine-readable "work that uses the Library", as object code and/or - source code, so that the user can modify the Library and then relink to - produce a modified executable containing the modified Library. (It is - understood that the user who changes the contents of definitions files - in the Library will not necessarily be able to recompile the application - to use the modified definitions.) - - b) Use a suitable shared library mechanism for linking with the Library. - A suitable mechanism is one that (1) uses at run time a copy of the - library already present on the user's computer system, rather than - copying library functions into the executable, and (2) will operate - properly with a modified version of the library, if the user installs - one, as long as the modified version is interface-compatible with the - version that the work was made with. - - c) Accompany the work with a written offer, valid for at least three - years, to give the same user the materials specified in Subsection 6a, - above, for a charge no more than the cost of performing this - distribution. - - d) If distribution of the work is made by offering access to copy from a - designated place, offer equivalent access to copy the above specified - materials from the same place. - - e) Verify that the user has already received a copy of these materials - or that you have already sent this user a copy. - -For an executable, the required form of the "work that uses the Library" must -include any data and utility programs needed for reproducing the executable -from it. However, as a special exception, the materials to be distributed need -not include anything that is normally distributed (in either source or binary -form) with the major components (compiler, kernel, and so on) of the operating -system on which the executable runs, unless that component itself accompanies -the executable. - -It may happen that this requirement contradicts the license restrictions of -other proprietary libraries that do not normally accompany the operating -system. Such a contradiction means you cannot use both them and the Library -together in an executable that you distribute. - -7. You may place library facilities that are a work based on the Library -side-by-side in a single library together with other library facilities not -covered by this License, and distribute such a combined library, provided that -the separate distribution of the work based on the Library and of the other -library facilities is otherwise permitted, and provided that you do these two -things: - - a) Accompany the combined library with a copy of the same work based on - the Library, uncombined with any other library facilities. This must be - distributed under the terms of the Sections above. - - b) Give prominent notice with the combined library of the fact that part - of it is a work based on the Library, and explaining where to find the - accompanying uncombined form of the same work. - -8. You may not copy, modify, sublicense, link with, or distribute the Library -except as expressly provided under this License. Any attempt otherwise to -copy, modify, sublicense, link with, or distribute the Library is void, and -will automatically terminate your rights under this License. However, parties -who have received copies, or rights, from you under this License will not have -their licenses terminated so long as such parties remain in full compliance. - -9. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the -Library or its derivative works. These actions are prohibited by law if you do -not accept this License. Therefore, by modifying or distributing the Library -(or any work based on the Library), you indicate your acceptance of this -License to do so, and all its terms and conditions for copying, distributing -or modifying the Library or works based on it. - -10. Each time you redistribute the Library (or any work based on the Library), -the recipient automatically receives a license from the original licensor to -copy, distribute, link with or modify the Library subject to these terms and -conditions. You may not impose any further restrictions on the recipients' -exercise of the rights granted herein. You are not responsible for enforcing -compliance by third parties with this License. - -11. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or otherwise) -that contradict the conditions of this License, they do not excuse you from -the conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Library at all. -For example, if a patent license would not permit royalty-free redistribution -of the Library by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Library. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply, and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -12. If the distribution and/or use of the Library is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Library under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -13. The Free Software Foundation may publish revised and/or new versions of -the Lesser General Public License from time to time. Such new versions will be -similar in spirit to the present version, but may differ in detail to address -new problems or concerns. - -Each version is given a distinguishing version number. If the Library -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software -Foundation. If the Library does not specify a license version number, you may -choose any version ever published by the Free Software Foundation. - -14. If you wish to incorporate parts of the Library into other free programs -whose distribution conditions are incompatible with these, write to the author -to ask for permission. For software which is copyrighted by the Free Software -Foundation, write to the Free Software Foundation; we sometimes make -exceptions for this. Our decision will be guided by the two goals of -preserving the free status of all derivatives of our free software and of -promoting the sharing and reuse of software generally. - -NO WARRANTY - -15. BECAUSE THE LIBRARY IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE LIBRARY, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -LIBRARY "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE LIBRARY IS WITH YOU. SHOULD THE LIBRARY PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -16. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE -THE LIBRARY AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE -OR INABILITY TO USE THE LIBRARY (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR -DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR -A FAILURE OF THE LIBRARY TO OPERATE WITH ANY OTHER SOFTWARE), EVEN IF SUCH -HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - - -END OF TERMS AND CONDITIONS - - -How to Apply These Terms to Your New Libraries ----------------------------------------------- - -If you develop a new library, and you want it to be of the greatest possible -use to the public, we recommend making it free software that everyone can -redistribute and change. You can do so by permitting redistribution under -these terms (or, alternatively, under the terms of the ordinary General Public -License). - -To apply these terms, attach the following notices to the library. It is -safest to attach them to the start of each source file to most effectively -convey the exclusion of warranty; and each file should have at least the -"copyright" line and a pointer to where the full notice is found. - - one line to give the library's name and an idea of what it does. - - Copyright (C) year name of author - - This library is free software; you can redistribute it and/or modify it - under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation; either version 2.1 of the License, or (at - your option) any later version. - - This library is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser - General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with this library; if not, write to the Free Software Foundation, - Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the library, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the library - `Frob' (a library for tweaking knobs) written by James Random Hacker. - - signature of Ty Coon, 1 April 1990 - - Ty Coon, President of Vice - -That's all there is to it! - - - -3. License notice for OpenJDK library -------------------------------------- - -The GNU General Public License (GPL) -==================================== - -Version 2, June 1991 - -Copyright (C) 1989, 1991 Free Software Foundation, Inc. 59 Temple Place, Suite -330, Boston, MA 02111-1307 USA - -Everyone is permitted to copy and distribute verbatim copies of this license -document, but changing it is not allowed. - -Preamble - -The licenses for most software are designed to take away your freedom to share -and change it. By contrast, the GNU General Public License is intended to -guarantee your freedom to share and change free software--to make sure the -software is free for all its users. This General Public License applies to -most of the Free Software Foundation's software and to any other program whose -authors commit to using it. (Some other Free Software Foundation software is -covered by the GNU Library General Public License instead.) You can apply it -to your programs, too. - -When we speak of free software, we are referring to freedom, not price. Our -General Public Licenses are designed to make sure that you have the freedom to -distribute copies of free software (and charge for this service if you wish), -that you receive source code or can get it if you want it, that you can change -the software or use pieces of it in new free programs; and that you know you -can do these things. - -To protect your rights, we need to make restrictions that forbid anyone to -deny you these rights or to ask you to surrender the rights. These -restrictions translate to certain responsibilities for you if you distribute -copies of the software, or if you modify it. - -For example, if you distribute copies of such a program, whether gratis or for -a fee, you must give the recipients all the rights that you have. You must -make sure that they, too, receive or can get the source code. And you must -show them these terms so they know their rights. - -We protect your rights with two steps: (1) copyright the software, and (2) -offer you this license which gives you legal permission to copy, distribute -and/or modify the software. - -Also, for each author's protection and ours, we want to make certain that -everyone understands that there is no warranty for this free software. If the -software is modified by someone else and passed on, we want its recipients to -know that what they have is not the original, so that any problems introduced -by others will not reflect on the original authors' reputations. - -Finally, any free program is threatened constantly by software patents. We -wish to avoid the danger that redistributors of a free program will -individually obtain patent licenses, in effect making the program proprietary. -To prevent this, we have made it clear that any patent must be licensed for -everyone's free use or not licensed at all. - -The precise terms and conditions for copying, distribution and modification -follow. - -TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION - -0. This License applies to any program or other work which contains a notice -placed by the copyright holder saying it may be distributed under the terms of -this General Public License. The "Program", below, refers to any such program -or work, and a "work based on the Program" means either the Program or any -derivative work under copyright law: that is to say, a work containing the -Program or a portion of it, either verbatim or with modifications and/or -translated into another language. (Hereinafter, translation is included -without limitation in the term "modification".) Each licensee is addressed as -"you". - -Activities other than copying, distribution and modification are not covered -by this License; they are outside its scope. The act of running the Program -is not restricted, and the output from the Program is covered only if its -contents constitute a work based on the Program (independent of having been -made by running the Program). Whether that is true depends on what the -Program does. - -1. You may copy and distribute verbatim copies of the Program's source code as -you receive it, in any medium, provided that you conspicuously and -appropriately publish on each copy an appropriate copyright notice and -disclaimer of warranty; keep intact all the notices that refer to this License -and to the absence of any warranty; and give any other recipients of the -Program a copy of this License along with the Program. - -You may charge a fee for the physical act of transferring a copy, and you may -at your option offer warranty protection in exchange for a fee. - -2. You may modify your copy or copies of the Program or any portion of it, -thus forming a work based on the Program, and copy and distribute such -modifications or work under the terms of Section 1 above, provided that you -also meet all of these conditions: - - a) You must cause the modified files to carry prominent notices stating - that you changed the files and the date of any change. - - b) You must cause any work that you distribute or publish, that in whole - or in part contains or is derived from the Program or any part thereof, to - be licensed as a whole at no charge to all third parties under the terms - of this License. - - c) If the modified program normally reads commands interactively when run, - you must cause it, when started running for such interactive use in the - most ordinary way, to print or display an announcement including an - appropriate copyright notice and a notice that there is no warranty (or - else, saying that you provide a warranty) and that users may redistribute - the program under these conditions, and telling the user how to view a - copy of this License. (Exception: if the Program itself is interactive - but does not normally print such an announcement, your work based on the - Program is not required to print an announcement.) - -These requirements apply to the modified work as a whole. If identifiable -sections of that work are not derived from the Program, and can be reasonably -considered independent and separate works in themselves, then this License, -and its terms, do not apply to those sections when you distribute them as -separate works. But when you distribute the same sections as part of a whole -which is a work based on the Program, the distribution of the whole must be on -the terms of this License, whose permissions for other licensees extend to the -entire whole, and thus to each and every part regardless of who wrote it. - -Thus, it is not the intent of this section to claim rights or contest your -rights to work written entirely by you; rather, the intent is to exercise the -right to control the distribution of derivative or collective works based on -the Program. - -In addition, mere aggregation of another work not based on the Program with -the Program (or with a work based on the Program) on a volume of a storage or -distribution medium does not bring the other work under the scope of this -License. - -3. You may copy and distribute the Program (or a work based on it, under -Section 2) in object code or executable form under the terms of Sections 1 and -2 above provided that you also do one of the following: - - a) Accompany it with the complete corresponding machine-readable source - code, which must be distributed under the terms of Sections 1 and 2 above - on a medium customarily used for software interchange; or, - - b) Accompany it with a written offer, valid for at least three years, to - give any third party, for a charge no more than your cost of physically - performing source distribution, a complete machine-readable copy of the - corresponding source code, to be distributed under the terms of Sections 1 - and 2 above on a medium customarily used for software interchange; or, - - c) Accompany it with the information you received as to the offer to - distribute corresponding source code. (This alternative is allowed only - for noncommercial distribution and only if you received the program in - object code or executable form with such an offer, in accord with - Subsection b above.) - -The source code for a work means the preferred form of the work for making -modifications to it. For an executable work, complete source code means all -the source code for all modules it contains, plus any associated interface -definition files, plus the scripts used to control compilation and -installation of the executable. However, as a special exception, the source -code distributed need not include anything that is normally distributed (in -either source or binary form) with the major components (compiler, kernel, and -so on) of the operating system on which the executable runs, unless that -component itself accompanies the executable. - -If distribution of executable or object code is made by offering access to -copy from a designated place, then offering equivalent access to copy the -source code from the same place counts as distribution of the source code, -even though third parties are not compelled to copy the source along with the -object code. - -4. You may not copy, modify, sublicense, or distribute the Program except as -expressly provided under this License. Any attempt otherwise to copy, modify, -sublicense or distribute the Program is void, and will automatically terminate -your rights under this License. However, parties who have received copies, or -rights, from you under this License will not have their licenses terminated so -long as such parties remain in full compliance. - -5. You are not required to accept this License, since you have not signed it. -However, nothing else grants you permission to modify or distribute the -Program or its derivative works. These actions are prohibited by law if you -do not accept this License. Therefore, by modifying or distributing the -Program (or any work based on the Program), you indicate your acceptance of -this License to do so, and all its terms and conditions for copying, -distributing or modifying the Program or works based on it. - -6. Each time you redistribute the Program (or any work based on the Program), -the recipient automatically receives a license from the original licensor to -copy, distribute or modify the Program subject to these terms and conditions. -You may not impose any further restrictions on the recipients' exercise of the -rights granted herein. You are not responsible for enforcing compliance by -third parties to this License. - -7. If, as a consequence of a court judgment or allegation of patent -infringement or for any other reason (not limited to patent issues), -conditions are imposed on you (whether by court order, agreement or otherwise) -that contradict the conditions of this License, they do not excuse you from -the conditions of this License. If you cannot distribute so as to satisfy -simultaneously your obligations under this License and any other pertinent -obligations, then as a consequence you may not distribute the Program at all. -For example, if a patent license would not permit royalty-free redistribution -of the Program by all those who receive copies directly or indirectly through -you, then the only way you could satisfy both it and this License would be to -refrain entirely from distribution of the Program. - -If any portion of this section is held invalid or unenforceable under any -particular circumstance, the balance of the section is intended to apply and -the section as a whole is intended to apply in other circumstances. - -It is not the purpose of this section to induce you to infringe any patents or -other property right claims or to contest validity of any such claims; this -section has the sole purpose of protecting the integrity of the free software -distribution system, which is implemented by public license practices. Many -people have made generous contributions to the wide range of software -distributed through that system in reliance on consistent application of that -system; it is up to the author/donor to decide if he or she is willing to -distribute software through any other system and a licensee cannot impose that -choice. - -This section is intended to make thoroughly clear what is believed to be a -consequence of the rest of this License. - -8. If the distribution and/or use of the Program is restricted in certain -countries either by patents or by copyrighted interfaces, the original -copyright holder who places the Program under this License may add an explicit -geographical distribution limitation excluding those countries, so that -distribution is permitted only in or among countries not thus excluded. In -such case, this License incorporates the limitation as if written in the body -of this License. - -9. The Free Software Foundation may publish revised and/or new versions of the -General Public License from time to time. Such new versions will be similar -in spirit to the present version, but may differ in detail to address new -problems or concerns. - -Each version is given a distinguishing version number. If the Program -specifies a version number of this License which applies to it and "any later -version", you have the option of following the terms and conditions either of -that version or of any later version published by the Free Software -Foundation. If the Program does not specify a version number of this License, -you may choose any version ever published by the Free Software Foundation. - -10. If you wish to incorporate parts of the Program into other free programs -whose distribution conditions are different, write to the author to ask for -permission. For software which is copyrighted by the Free Software -Foundation, write to the Free Software Foundation; we sometimes make -exceptions for this. Our decision will be guided by the two goals of -preserving the free status of all derivatives of our free software and of -promoting the sharing and reuse of software generally. - -NO WARRANTY - -11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR -THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE -STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE -PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND -PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, -YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. - -12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL -NY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE -PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY -GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE -OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR -DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR -A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH -HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. - -END OF TERMS AND CONDITIONS - -How to Apply These Terms to Your New Programs - -If you develop a new program, and you want it to be of the greatest possible -use to the public, the best way to achieve this is to make it free software -which everyone can redistribute and change under these terms. - -To do so, attach the following notices to the program. It is safest to attach -them to the start of each source file to most effectively convey the exclusion -of warranty; and each file should have at least the "copyright" line and a -pointer to where the full notice is found. - - One line to give the program's name and a brief idea of what it does. - - Copyright (C) - - This program is free software; you can redistribute it and/or modify it - under the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your option) - any later version. - - This program is distributed in the hope that it will be useful, but - WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY - or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - for more details. - - You should have received a copy of the GNU General Public License along - with this program; if not, write to the Free Software Foundation, Inc., 59 - Temple Place, Suite 330, Boston, MA 02111-1307 USA - -Also add information on how to contact you by electronic and paper mail. - -If the program is interactive, make it output a short notice like this when it -starts in an interactive mode: - - Gnomovision version 69, Copyright (C) year name of author Gnomovision - comes with ABSOLUTELY NO WARRANTY; for details type 'show w'. This is - free software, and you are welcome to redistribute it under certain - conditions; type 'show c' for details. - -The hypothetical commands 'show w' and 'show c' should show the appropriate -parts of the General Public License. Of course, the commands you use may be -called something other than 'show w' and 'show c'; they could even be -mouse-clicks or menu items--whatever suits your program. - -You should also get your employer (if you work as a programmer) or your -school, if any, to sign a "copyright disclaimer" for the program, if -necessary. Here is a sample; alter the names: - - Yoyodyne, Inc., hereby disclaims all copyright interest in the program - 'Gnomovision' (which makes passes at compilers) written by James Hacker. - - signature of Ty Coon, 1 April 1989 - - Ty Coon, President of Vice - -This General Public License does not permit incorporating your program into -proprietary programs. If your program is a subroutine library, you may -consider it more useful to permit linking proprietary applications with the -library. If this is what you want to do, use the GNU Library General Public -License instead of this License. - - -"CLASSPATH" EXCEPTION TO THE GPL - -Certain source files distributed by Oracle America and/or its affiliates are -subject to the following clarification and special exception to the GPL, but -only where Oracle has expressly included in the particular source file's -header the words "Oracle designates this particular file as subject to the -"Classpath" exception as provided by Oracle in the LICENSE file that -accompanied this code." - - Linking this library statically or dynamically with other modules is - making a combined work based on this library. Thus, the terms and - conditions of the GNU General Public License cover the whole combination. - - As a special exception, the copyright holders of this library give you - permission to link this library with independent modules to produce an - executable, regardless of the license terms of these independent modules, - and to copy and distribute the resulting executable under terms of your - choice, provided that you also meet, for each linked independent module, - the terms and conditions of the license of that module. An independent - module is a module which is not derived from or based on this library. If - you modify this library, you may extend this exception to your version of - the library, but you are not obligated to do so. If you do not wish to do - so, delete this exception statement from your version. \ No newline at end of file diff --git a/scripts/cbtools/bin/cbbackupmgr b/scripts/cbtools/bin/cbbackupmgr deleted file mode 100755 index 5b204cdf..00000000 Binary files a/scripts/cbtools/bin/cbbackupmgr and /dev/null differ diff --git a/scripts/cbtools/bin/cbexport b/scripts/cbtools/bin/cbexport deleted file mode 100755 index 756c9fc7..00000000 Binary files a/scripts/cbtools/bin/cbexport and /dev/null differ diff --git a/scripts/cbtools/bin/cbimport b/scripts/cbtools/bin/cbimport deleted file mode 100755 index 8a26fab8..00000000 Binary files a/scripts/cbtools/bin/cbimport and /dev/null differ diff --git a/scripts/cbtools/lib/libforestdb.so b/scripts/cbtools/lib/libforestdb.so deleted file mode 100755 index 8b431f75..00000000 Binary files a/scripts/cbtools/lib/libforestdb.so and /dev/null differ diff --git a/scripts/cbtools/lib/libgcc_s.so b/scripts/cbtools/lib/libgcc_s.so deleted file mode 100755 index c8e92242..00000000 --- a/scripts/cbtools/lib/libgcc_s.so +++ /dev/null @@ -1,4 +0,0 @@ -/* GNU ld script - Use the shared library, but some functions are only in - the static library. */ -GROUP ( libgcc_s.so.1 -lgcc ) diff --git a/scripts/cbtools/lib/libgcc_s.so.1 b/scripts/cbtools/lib/libgcc_s.so.1 deleted file mode 100755 index 37099061..00000000 Binary files a/scripts/cbtools/lib/libgcc_s.so.1 and /dev/null differ diff --git a/scripts/cbtools/lib/libjemalloc.so b/scripts/cbtools/lib/libjemalloc.so deleted file mode 120000 index 3d07d8a1..00000000 --- a/scripts/cbtools/lib/libjemalloc.so +++ /dev/null @@ -1 +0,0 @@ -libjemalloc.so.2 \ No newline at end of file diff --git a/scripts/cbtools/lib/libjemalloc.so.2 b/scripts/cbtools/lib/libjemalloc.so.2 deleted file mode 100644 index 709d06f7..00000000 Binary files a/scripts/cbtools/lib/libjemalloc.so.2 and /dev/null differ diff --git a/scripts/cbtools/lib/libsnappy.so b/scripts/cbtools/lib/libsnappy.so deleted file mode 120000 index 32c712f2..00000000 --- a/scripts/cbtools/lib/libsnappy.so +++ /dev/null @@ -1 +0,0 @@ -libsnappy.so.1 \ No newline at end of file diff --git a/scripts/cbtools/lib/libsnappy.so.1 b/scripts/cbtools/lib/libsnappy.so.1 deleted file mode 120000 index d286bf2e..00000000 --- a/scripts/cbtools/lib/libsnappy.so.1 +++ /dev/null @@ -1 +0,0 @@ -libsnappy.so.1.1.8 \ No newline at end of file diff --git a/scripts/cbtools/lib/libsnappy.so.1.1.8 b/scripts/cbtools/lib/libsnappy.so.1.1.8 deleted file mode 100644 index abc9bee5..00000000 Binary files a/scripts/cbtools/lib/libsnappy.so.1.1.8 and /dev/null differ diff --git a/scripts/cbtools/lib/libstdc++.so b/scripts/cbtools/lib/libstdc++.so deleted file mode 100755 index 02c48c0a..00000000 Binary files a/scripts/cbtools/lib/libstdc++.so and /dev/null differ diff --git a/scripts/cbtools/lib/libstdc++.so.6 b/scripts/cbtools/lib/libstdc++.so.6 deleted file mode 100755 index 02c48c0a..00000000 Binary files a/scripts/cbtools/lib/libstdc++.so.6 and /dev/null differ diff --git a/scripts/cbtools/lib/libstdc++.so.6.0.32 b/scripts/cbtools/lib/libstdc++.so.6.0.32 deleted file mode 100755 index 02c48c0a..00000000 Binary files a/scripts/cbtools/lib/libstdc++.so.6.0.32 and /dev/null differ diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-backup.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-backup.html deleted file mode 100644 index 8169f911..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-backup.html +++ /dev/null @@ -1,1626 +0,0 @@ - - - - - - - - - -cbbackupmgr-backup(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr backup [--archive <archive_dir>] [--repo <repo_name>]
-                   [--cluster <url>] [--username <username>]
-                   [--password <password>] [--client-cert <path>]
-                   [--client-cert-password <password>] [--client-key <path>]
-                   [--client-key-password <password>] [--resume] [--purge]
-                   [--threads <num>] [--cacert <file>] [--no-ssl-verify]
-                   [--value-compression <type>] [--no-progress-bar]
-                   [--skip-last-compaction] [--consistency-check <window>]
-                   [--full-backup] [--obj-access-key-id <access_key_id>]
-                   [--obj-cacert <cert_path>] [--obj-endpoint <endpoint>]
-                   [--obj-no-ssl-verify] [--obj-region <region>]
-                   [--obj-staging-dir <staging_dir>]
-                   [--obj-secret-access-key <secret_access_key>]
-                   [--s3-force-path-style] [--s3-log-level <level>]
-                   [--passphrase <passphrase>] [--km-key-url <url>]
-                   [--km-endpoint <endpoint>] [--km-region <region>]
-                   [--km-access-key-id <id>] [--km-secret-access-key <key>]
-                   [--km-auth-file <path>]
-
-
-
-
-

DESCRIPTION

-
-
-

Backs up a Couchbase cluster into the backup repository specified. Before -running the backup command, a backup repository must be created. See -cbbackupmgr-config(1) for more details on creating a backup -repository. The backup command uses information from the previous backup taken -in order to backup all new data on a Couchbase cluster. If no previous backup -exists then all data on the cluster is backed up. The backup is taken based on -the backup repository’s backup configuration. Each backup will create a new -folder in the backup repository. This folder will contain all data from the -backup and is named to reflect the time that the backup was started.

-
-
-

As the backup runs, it tracks its progress which allows failed backups to be -resumed from the point where they left off. If a backup fails before it is -complete it is considered a partial backup. To attempt to complete the backup -process, the backup may be resumed with the --resume flag. It may also be -deleted and resumed from the previous successful backup with the --purge flag.

-
-
-

The backup command is capable of backing up data when there is a cluster -rebalance operation in progress. During a rebalance, the backup command will -track data as it moves around the cluster and complete the backup. However, users -should use caution when running backups during a rebalance since both the -rebalance and backup operations can be resource intensive and may cause -temporary performance degradations in other parts of the cluster. See the ---threads flag for information on how to lower the impact of the backup command -on your Couchbase cluster.

-
-
-

The backup command is also capable of backing up data when there are server -failures in the target backup cluster. When a server failure occurs the backup -command will wait for 180 seconds for the failed server to come back online or -for the failed server to be failed over and removed from the cluster. If 180 -seconds passes without the failed server coming back online or being failed over -then the backup command will mark the data on that node as failed and attempt to -back up the rest of the data from the cluster. The backup will be marked as a -partial backup in the backup archive and will need to be either resumed or -purged when the backup command is invoked again.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of required and optional parameters for the backup command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The location of the backup archive directory. When backing up directly to S3 -prefix the archive path with s3://${BUCKET_NAME}/.

-
-
-r,--repo <repo_name>
-
-

The name of the backup repository to backup data into.

-
-
-c,--cluster <hostname>
-
-

The hostname of one of the nodes in the cluster to back up. See the -Host Formats section below for hostname specification details.

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
-
-
-
-

Optional

-
-
-
--resume
-
-

If the previous backup did not complete successfully it can be resumed from -where it left off by specifying this flag. Note that the resume and purge -flags may not be specified at the same time.

-
-
--purge
-
-

If the previous backup did not complete successfully the partial backup will -be removed and restarted from the point of the previous successful backup by -specifying this flag. Note that the purge and resume flags may not be -specified at the same time.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man-in-the-middle attack if -you use this flag. Either this flag or the --cacert flag must be specified -when using an SSL encrypted connection.

-
-
--cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag -must be specified when using an SSL encrypted connection.

-
-
--value-compression <compression_policy>
-
-

Specifies a compression policy for backed up values. When Couchbase sends -data to the backup client the data stream may contain all compressed values, -all uncompressed values, or a mix of compressed and uncompressed values. To -backup all data in the same form that the backup client receives it you can -specify "unchanged". If you wish for all values to be uncompressed then you -can specify "uncompressed". This policy will however uncompress any -compressed values received from Couchbase and may increase the backup file -size. To compress all values you can specify "compressed". This will compress -any uncompressed values before writing them to disk. The default value for -this option is "compressed".

-
-
-t,--threads <num>
-
-

Specifies the number of concurrent clients to use when taking a backup. -Fewer clients means backups will take longer, but there will be less cluster -resources used to complete the backup. More clients means faster backups, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the backup is -taking place.

-
-
--no-progress-bar
-
-

By default, a progress bar is printed to stdout so that the user can see how -long the backup is expected to take, the amount of data that is being -transferred per second, and the amount of data that has been backed up. -Specifying this flag disables the progress bar and is useful when running -automated jobs.

-
-
--consistency-check <window>
-
-

When a window larger than 1 is provided it will enable the consistency -checker. This will show a warning if the backup consistency window is larger -than the one provided in seconds. This is an Enterprise Edition feature and -is currently in developer preview. See DISCUSSION for more information.

-
-
--full-backup
-
-

Force a full backup in the same backup repository. This can be used to more -easily manage backups.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Backing up directly to object store is only supported for Couchbase Server -6.6.0 and above. It’s likely that backing up older clusters will result in -significantly higher memory consumption.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-

Encryption (Developer Preview)

-
-
-
--passphrase <passphrase>
-
-

Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing.

-
-
--km-key-url <url>
-
-

Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable CB_KM_KEY_URL. For more on how to authenticate using the different -providers see cbbackupmgr-encryption(7).

-
-

For AWS the expected key format is awskms://<KEY-ID|KEY-ALIAS>, for example -awskms://alias\keyAlias.

-
-
-

For GCP the expected key format is gcpkms://<KEY-RESOURCE-ID>, for example -gcpkms://projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key.

-
-
-

For Azure key vault the expected key format is azurekeyvault://<KEY-IDENTIFIER> -for example: -azurekeyvault://vault-name.vault.azure.net/object-type/object-name/object-version.

-
-
-

For HashiCorp Vault the expected format is hashivaults://<HOST>/<KEY-NAME> -for example: -hashivaults://127.0.0.1:8200/keyName.

-
-
-
--km-region <region>
-
-

Required when using AWS KMS, it allows you to set the key region.

-
-
--km-endpoint <endpoint>
-
-

The host or address to use as your KMS. It will override the default SDK one.

-
-
--km-access-key-id <id>
-
-

The user ID used to connect to the key management service. It can also be -provided via CB_KM_ACCESS_KEY_ID environmental variable. Please refer -to cbbackupmgr-encryption(7) for the required authentication for each -provider.

-
-
--km-secret-access-key <key>
-
-

The key used to connect to the key management service. It can also be -provided via the CB_KM_SECRET_ACCESS_KEY environmental variable. Please -refer to cbbackupmgr-encryption(7) for the required authentication for -each provider.

-
-
--km-auth-file <path>
-
-

The path to a file containing the authentication credentials for the key -management service. It can also be provided via the CB_KM_AUTH_FILE -environmental variable. Please refer to cbbackupmgr-encryption(7) for the -required authentication for each provider.

-
-
-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

RBAC

-
-
-

When performing a backup/restore with a user which is using RBAC, there are a -couple of things that should be taken into consideration each of which is -highlighted in this section.

-
-
-

Bucket Level

-
-

Bucket level data may be backed up/restored using the data_backup (Data -Backup & Restore) role.

-
-
-

The data_backup role does not have access to cluster level data such as:

-
-
-
    -
  • -

    Analytics Synonyms

    -
    • -
  • -

    Eventing Metadata

    -
    • -
  • -

    FTS Aliases

    -
    • -
-
-
-

Backing up/restoring cluster level data with the data_backup role will cause -permission errors like the one below.

-
-
-
-
Error backing up cluster: {"message":"Forbidden. User needs one of the following permissions","permissions":["cluster.fts!read"]}
-
-
-
-

When presented with an error message such as the one above, there’s two clear -options.

-
-
-

The first option is to provide the user with the required credentials using -either the cli, REST API or Couchbase Server WebUI. This can be done by editing -the user and adding the required role. See Cluster Level for more information -about the required roles.

-
-
-

Secondly, backing up/restoring the specific service can be disabled. For -backups this must be done when configuring the repository with the config -command using the --disable style flags. For restore, these flags may be used -directly to disable one or more services. See the backup/restore documentation -for more information.

-
-
-
-

Cluster Level

-
-

Backing up/restoring cluster level data requires additional RBAC roles, each of -which is highlighted below:

-
-
-
-
Analytics Synonyms
-
-

analytics_admin (Analytics Admin)

-
-
Eventing Metadata
-
-

eventing_admin (Eventing Full Admin)

-
-
FTS Aliases
-
-

fts_admin (Search Admin)

-
-
-
-
-

These additional roles are required since this is cluster level data which may -encompass multiple buckets.

-
-
-
-
-
-

EXAMPLES

-
-
-

The following command is used to take a backup of a Couchbase cluster.

-
-
-
-
$ cbbackupmgr config --archive /data/backups --repo example
-$ cbbackupmgr backup -a /data/backups -r example \
- -c couchbase://172.23.10.5 -u Administrator -p password
-
-
-
-

Once the backup has finished there will be a new directory in the specified -backup repository containing the backed up data. You can see this new directory -using the cbbackupmgr-info(1) command.

-
-
-
-
$ cbbackupmgr info -a /data/backups --all
-
-Name         | UUID                                 | Size     | # Repos  |
-example | 32c97d5f-821a-4284-840b-9ee7cf8733a3 | 55.56MB  | 1        |
-
-*  Name        | Size     | # Backups  |
-*  Manchester  | 55.56MB  | 1          |
-
-+    Backup                      | Size     | Type | Source               | Cluster UUID                     | Range | Events  | Aliases  | Complete  |
-+    2019-03-15T13_52_27.18301Z  | 55.56MB  | FULL | http://172.23.10.5   | c044f5eeb1dc16d0cd49dac29074b5f9 | N/A   | 0       | 1        | true      |
-
--      Bucket          | Size     | Items  | Mutations | Tombstones | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample     | 6.85MB   | 7303   | 7303      | 0          | 1      | 0    | 1        | 0     |
--      gamesim-sample  | 2.86MB   | 586    | 586       | 0          | 1      | 0    | 1        | 0     |
--      travel-sample   | 42.72MB  | 31591  | 31591     | 0          | 0      | 0    | 10       | 0     |
-
-
-
-

If a backup fails then it is considered a partial backup and the backup client -will not be able to back up any new data until the user decides whether to -resume or purge the partial backup. This decision is made by specifying either -the --resume or the --purge flag on the next invocation of the backup command. -Below is an example of how this process works if the user wants to resume a -backup.

-
-
-
-
$ cbbackupmgr config -a /data/backups -r example
-
-$ cbbackupmgr backup -a /data/backups -r example \
- -c 172.23.10.5 -u Administrator -p password
-
-Error backing up cluster: Not all data was backed up due to connectivity
-issues. Check to make sure there were no server side failures during
-backup. See backup logs for more details on what wasn't backed up.
-
-$ cbbackupmgr backup -a /data/backups -r example \
- -c 172.23.10.5 -u Administrator -p password
-
-Error backing up cluster: Partial backup error 2016-02-11T17:00:19.594970735-08:00
-
-$ cbbackupmgr backup -a /data/backups -r example -c 172.23.10.5 \
- -u Administrator -p password --resume
-
-Backup successfully completed
-
-
-
-

To backup a cluster with a different amount of concurrent clients and decrease -the backup time you can specify the --threads flag. Remember that specifying a -higher number of concurrent clients increases the amount of resources the -cluster uses to complete the backup. Below is an example of using 16 concurrent -clients.

-
-
-
-
$ cbbackupmgr config -a /data/backups -r example
-
-$ cbbackupmgr backup -a /data/backups -r example \
- -c 172.23.10.5 -u Administrator -p password -t 16
-
-
-
-

To force the creation of a full backup, use the --full-backup flag. This will -result in cbbackupmgr streaming all the available data again. It’s expected -that more disk space will be used and that the full backup will take longer -than performing an incremental backup (the default).

-
-
-

In the example below, the first backup is implicitly a full backup, the second -backup is an incremental (where no additional data needed to be backed up) and -the third is a forced full backup (note that we backed up the same amount of -items as the first backup).

-
-
-
-
$ cbbackupmgr backup -a ~/Projects/couchbase-archive -r repo -c 172.20.1.1 -u Administrator -p asdasd
-Backing up to '2021-02-09T13_35_15.426546996Z'
-Copied all data in 2.865028528s (Avg. 10.53MB/Sec)                                                                                        31591 items / 21.06MB
-[=====================================================================================================================================================] 100.00%
-Backup completed successfully
-Backed up bucket "travel-sample" succeeded
-Mutations backed up: 31591, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-
-$ cbbackupmgr backup -a ~/Projects/couchbase-archive -r repo -c 172.20.1.1 -u Administrator -p asdasd
-Backing up to '2021-02-09T13_35_21.580950579Z'
-Copied all data in 504.852625ms (Avg. 56.00KB/Sec)                                                                                            0 items / 56.00KB
-[=====================================================================================================================================================] 100.00%
-Backup completed successfully
-Backed up bucket "travel-sample" succeeded
-Mutations backed up: 0, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-Skipped due to purge number or conflict resolution: Mutations: 0 Deletions: 0
-
-$ cbbackupmgr backup -a ~/Projects/couchbase-archive -r repo -c 172.20.1.1 -u Administrator -p asdasd --fullbackup
-Backing up to '2021-02-09T13_35_24.18890408Z'
-Copied all data in 2.85286061s (Avg. 10.53MB/Sec)                                                                                         31591 items / 21.06MB
-[=====================================================================================================================================================] 100.00%
-Backup completed successfully
-Backed up bucket "travel-sample" succeeded
-Mutations backed up: 31591, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-Skipped due to purge number or conflict resolution: Mutations: 0 Deletions: 0
-
-
-
-
-
-

COLLECTION AWARE BACKUPS

-
-
-

The 7.0.0 release of Couchbase Server introduced collections supports. The -following section will briefly discuss how this affects backups created by -cbbackupmgr.

-
-
-

What makes a backup collection aware?

-
-

cbbackupmgr is not limited to only backing up the cluster version it was -released with; it may be used to backup previous versions of Couchbase -Server. This means that cbbackupmgr is able to create backups which are -collection aware or unaware.

-
-
-

The distinction between collection aware/unaware is simple; if you backup -a cluster prior to 7.0.0 your backup will be collection unaware. If you -backup a cluster version which supports collections (starting at 7.0.0) -your backup will be collection aware.

-
-
-

The default behavior of backup for the cluster versions which support -collections is to back up all available scopes/collections in a bucket.

-
-
-
-

What if I don’t want to use collections?

-
-

This is perfectly valid use case and is supported by Couchbase Server and -cbbackupmgr. When interacting with backups created of a collection aware -cluster, cbbackupmgr will only output collection aware information if the -backup contains a non-default collection manifest (which implies the use of -collections).

-
-
-

This means you may continue using cbbackupmgr without needing to interact with -collections. Note that you may still need to update/change the flags being -passed to some sub-commands. For example, when examining a backup, you will -still need to use the --collection-string flag. For example instead of ---bucket default you should supply --collection-string default.

-
-
-
-
-
-

DISCUSSION

-
-
-

This command always backs up data incrementally. By using the vbucket sequence -number that is associated with each item, the backup command is able to examine -previous backups in order to determine where the last backup finished.

-
-
-

When backing up a cluster, data for each bucket is backed up in the following -order:

-
-
-
    -
  • -

    Bucket Settings

    -
    • -
  • -

    View Definitions

    -
    • -
  • -

    Global Secondary Index (GSI) Definitions

    -
    • -
  • -

    Full-Text Index Definitions

    -
    • -
  • -

    Key-Value Data

    -
    • -
-
-
-

The backup command will store everything that is persisted to disk on the -Couchbase Server nodes at the time the backup is started. Couchbase server is -consistent at a vBucket level and not across a whole bucket. The tool tries -to provide a strong consistency window by opening all connection to every -node at the same time. Being a distributed system there are times when this is -not possible such as when the cluster is under-resourced or there are network -issues. These may affect the consistency of the backup across -the vBuckets. cbbackupmgr backup provides an Enterprise Edition, developer -preview feature that checks that the backup is inside a consistency window.

-
-
-
-
-

ROLLBACK

-
-
-

During a backup, it’s possible for cbbackupmgr to receive a rollback from the -cluster; this will be returned to you (the user) in the form of a message -similar to client received rollback, either purge this backup or create a new -backup repository.

-
-
-

There are two sensible actions which can be taken at this point (which are -briefly alluded in the above message):

-
-
-

1) You can rerun the backup using the --purge flag, this will remove the - failed backup creating a new incremental backup if possible, otherwise - falling back to creating a full backup. -2) You could create a new backup repository and begin creating backups there, - this method has the advantage of preserving any possible data which was - backed up prior to receiving the rollback.

-
-
-

In most cases, option one will be sufficient, however, if may be the case that -the partial backup taken by cbbackupmgr contains data which you don’t want to -delete. In this case, option two allows you to retain this data which could be -restored using the --restore-partial-backups flag.

-
-
-

Note that the --restore-partial-backups flag is only supported for local -backups; backups stored in object store which failed due to a rollback must be -purged using the --purge flag.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
CB_ENCRYPTION_PASSPHRASE
-
-

Specifies the passphrase used for encryption.

-
-
CB_KM_KEY_URL
-
-

Specifies the URL identifying the encryption key on the KMS. See ---km-key-url for the expected format and accepted KMSs.

-
-
CB_KM_ACCESS_ID
-
-

Specifies the key/user ID used to connect to the KMS.

-
-
CB_KM_SECRET_ACCESS_KEY
-
-

Specifies the secret key/token used to connect to the KMS.

-
-
CB_KM_AUTH_FILE
-
-

Specifies a path to a file containing the required credentials to connect to -the KMS.

-
-
-
-
-
-
-

FILES

-
-
-
-
restrictions.json
-
-

Keeps a list of restrictions used to ensure data is not restored to the -place.

-
-
bucket-config.json
-
-

Stores the bucket configuration settings for a bucket.

-
-
views.json
-
-

Stores the view definitions for a bucket.

-
-
gsi.json
-
-

Stores the global secondary index (GSI) definitions for a bucket.

-
-
full-text.json
-
-

Stores the full-text index definitions for a bucket.

-
-
shard-*.fdb
-
-

Stores the key-value data for a bucket bucket.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-config(1), -cbbackupmgr-info(1), -cbbackupmgr-strategies(1), -cbbackupmgr-cloud(7), -man:cbbackupmgr-encryption, -cbbackupmgr-network-filesystems(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-cloud.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-cloud.html deleted file mode 100644 index 9d6855da..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-cloud.html +++ /dev/null @@ -1,1342 +0,0 @@ - - - - - - - - - -cbbackupmgr-cloud(7) - - - - - -
-
-

DESCRIPTION

-
-
-

A document which should give you a basic understanding of how to utilize -cbbackupmgr’s Enterprise Edition feature, native cloud interactions.

-
-
-
-
-

TUTORIAL

-
-
-

CREDENTIALS

-
-

Backing up directly to an external cloud provider will mean that we require -permissions to access the given data store. Each provider has its own way -of authentication; see the sections below on how to authenticate for your -chosen cloud provider.

-
-
-

AWS

-
-

When using AWS S3, there are multiple different ways that you can supply -credentials to authorize yourself to AWS S3. Below is a list of the -supported techniques:

-
-
-
    -
  1. -

    When running in EC2, credentials may be obtained via the instance metadata service by setting/supplying:

    -
    -
      -
    • -

      CB_AWS_ENABLE_EC2_METADATA=true

      -
      • -
    • -

      --obj-auth-by-instance-metadata

      -
      • -
    -
    -
    2. -
  2. -

    Providing a set of environment variables including:

    -
    -
      -
    • -

      CB_OBJSTORE_REGION

      -
      • -
    • -

      CB_OBJSTORE_ACCESS_KEY_ID

      -
      • -
    • -

      CB_OBJSTORE_SECRET_ACCESS_KEY

      -
      • -
    -
    -
    3. -
  3. -

    Loading credentials from the shared config files located at:

    -
    -
      -
    • -

      $HOME/.aws/config

      -
      • -
    • -

      $HOME/.aws/credentials

      -
      • -
    -
    -
    4. -
  4. -

    Providing static config/credentials using the cli flags:

    -
    -
      -
    • -

      --obj-access-key-id

      -
      • -
    • -

      --obj-region

      -
      • -
    • -

      --obj-secret-access-key

      -
      • -
    -
    -
    5. -
-
-
-

Setting up cbbackupmgr to interact with AWS should be a very similar process to -setting up the aws-cli. The steps to configure the aws-cli can be found at -https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html#cli-quick-configuration.

-
-
-

For more information about authenticating using the EC2 instance metadata -service, please see the Cloud Provider Specific Features section.

-
-
-
-

Azure (7.1.2+)

-
-

As with AWS, there are multiple accepted methods for authorizing against Azure. -Below is a list of the supported techniques:

-
-
-
    -
  1. -

    Providing static credentials using the cli flags:

    -
    -
      -
    • -

      --obj-access-key-id

      -
      • -
    • -

      --obj-secret-access-key

      -
      • -
    -
    -
    2. -
  2. -

    Providing a set of environment variables including:

    -
    -
      -
    • -

      AZURE_STORAGE_ACCOUNT

      -
      • -
    • -

      AZURE_STORAGE_KEY

      -
      • -
    -
    -
    3. -
  3. -

    Providing a connection string via an environment variable:

    -
    -
      -
    • -

      AZURE_STORAGE_CONNECTION_STRING

      -
      • -
    -
    -
    4. -
  4. -

    Obtaining a service principle token through the environment, using:

    -
    -
      -
    1. -

      Client credentials

      -
      -
        -
      • -

        AZURE_CLIENT_ID

        -
        • -
      • -

        AZURE_TENANT_ID

        -
        • -
      • -

        AZURE_CLIENT_SECRET

        -
        • -
      -
      -
      2. -
    2. -

      A client certificate

      -
      -
        -
      • -

        AZURE_CLIENT_ID

        -
        • -
      • -

        AZURE_TENANT_ID

        -
        • -
      • -

        AZURE_CERTIFICATE_PATH

        -
        • -
      -
      -
      3. -
    3. -

      A username/password

      -
      -
        -
      • -

        AZURE_CLIENT_ID

        -
        • -
      • -

        AZURE_TENANT_ID

        -
        • -
      • -

        AZURE_USERNAME

        -
        • -
      • -

        AZURE_PASSWORD

        -
        • -
      -
      -
      4. -
    4. -

      A managed identity

      -
      5. -
    -
    -
    5. -
-
-
-

When using Azure Active Directory to authenticate, cbbackupmgr may not be able to determine the storage account name -and, as a consequence, the URL that it should use to connect to the Azure Blob Storage service. In this case it will -return an error such as the one below.

-
-
-
-
failed to determine account name, checked '--obj-access-key-id', 'AZURE_STORAGE_ACCOUNT' and
-'AZURE_STORAGE_CONNECTION_STRING'
-
-
-
-

The account name will need to be supplied by using one of the following options: -- The --obj-access-key-key flag -- The AZURE_STORAGE_ACCOUNT environment variable -- The AZURE_STORAGE_CONNECTION_STRING environment variable (by specifically setting the AccountName key)

-
-
-
-

GCP

-
-

As with AWS/Azure, GCP supports multiple methods of authenticating. Below is a -list of the supported techniques:

-
-
-
    -
  1. -

    Providing static credentials using the cli flags:

    -
    -
      -
    • -

      --obj-access-key-id

      -
      • -
    • -

      --obj-secret-access-key

      -
      • -
    • -

      --obj-refresh-token

      -
      • -
    -
    -
    2. -
  2. -

    Providing an SDK style auth file

    -
    -
      -
    1. -

      Using a static flag

      -
      -
        -
      • -

        --obj-auth-file

        -
        • -
      -
      -
      2. -
    2. -

      Using an environment variable

      -
      -
        -
      • -

        GOOGLE_APPLICATION_CREDENTIALS

        -
        • -
      -
      -
      3. -
    -
    -
    3. -
  3. -

    Using a "well known" credentials file

    -
    -
      -
    • -

      $HOME/.config/gcloud/application_default_credentials.json

      -
      • -
    -
    -
    4. -
  4. -

    Using the first generation app engine runtime

    -
    5. -
  5. -

    Using the metadata service when running in Google Compute Engine

    -
    6. -
-
-
-
-
-

THE STAGING DIRECTORY

-
-

One of the most important concepts behind how backup to object store works is -the staging directory. The staging directory is a location on disk where -temporary data is stored during the execution of a sub-command. For a -backup/restore this will be DCP metadata and storage indexes.

-
-
-

When creating an archive to store in a cloud provider you are required to -provide a location for the obj-staging-dir. This is a local location where -archive meta data will be stored. During a backup, files will be stored here -before they are uploaded to the cloud. Note that cbbackupmgr doesn’t store any -document values in the staging directory; they are streamed directly to the -cloud.

-
-
-

Each cloud archive must have a unique staging directory i.e. they can’t be -shared. cbbackupmgr will detect cases where the staging directory is being -reused across archives.

-
-
-

Any modifications to the cloud archive (using the web-ui or cli tools) and not -cbbackupmgr are not supported whilst using the same staging directory. If a -cloud archive has been modified, the staging directory should be removed and -recreated before using cbbackupmgr to interact with the archive again.

-
-
-

The staging directory is only used during operations e.g. backup/restore and -can be safely deleted once an operation completes; this is because all the files -will have been uploaded to the cloud.

-
-
-

The staging directory can become quite large during a normal backup depending -on the number of documents being backed up, and the size of their keys. See the -Disk requirements section for more information about how to provision the -staging directory.

-
-
-
-

CONFIGURING CLOUD BACKUPS

-
-

The first step is to create a backup archive in object store. This can be done with -the config command and only needs to be done once. All other commands will -automatically download the archive meta data in the directory provided via the obj-staging-dir -argument prior to performing any operations; this is done regardless of whether the archive -exists locally because we must ensure the archive in the staging directory is up to date. -Below is an example of how you would configure an archive in AWS S3.

-
-
-
-
$ cbbackupmgr config -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging
-
-Backup repository `repo` created successfully in archive `s3://bucket/archive`
-
-
-
-

Assuming your credentials are correct, the archive should now reside directly in the -provided S3 bucket. To verify, you could use the aws-cli to list the contents off the -bucket and they should be identical to that which would exist for a local backup.

-
-
-

Although it’s possible to have cbbackupmgr coexist in the same S3 bucket as other general -purpose storage, we recommend using a bucket which cbbackupmgr has exclusive access too.

-
-
-
-

BACKING UP A CLUSTER

-
-

Once an archive is configured performing a backup works in a similar fashion to performing a local -backup. It’s important to note that when backing up directly to S3 a certain amount of disk -space will be used to stage local meta data files and storage indexes. See the -Staging Directory section for more information. Below is an example of -doing a backup and storing directly in AWS S3.

-
-
-
-
$ cbbackupmgr backup -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging \
-  -c http://10.101.101.112:8091 -u Administrator -p password
-
-Copied all data in 1m2.525397237s (Avg. 51.61KB/Sec)      7303 items / 3.12MB
-beer-sample             [===========================================] 100.00%
-
-Backup successfully completed
-Backed up bucket "beer-sample" succeeded
-Mutations backedup; 7303, Mutations failed to backup: 0
-Deletions backedup: 0, Deletions failed to backup: 0
-
-
-
-

Performing incremental backups works exactly as it would if you were performing an incremental -locally; simply rerun the command above and an incremental backup would be created.

-
-
-

When choosing the amount of threads to use it’s important to consider that when backing up to the -cloud, cbbackupmgr buffers data in memory before uploading it. This means that choosing an extremely -large amount of threads when using a poor internet connection could lead to a scenario where your -machine runs out of memory.

-
-
-

To learn more about backup options see cbbackupmgr-backup(1).

-
-
-
-

RESTORING A BACKUP / MULTIPLE INCREMENTAL BACKUPS

-
-

Once you have created a backup, restoring it works in a similar way to restoring a local -backup. It’s worth noting that restoring a backup to a cluster that’s hosted outside of AWS -is likely to be significantly more expensive than performing a backup (depending on the size of -your dataset). See Costing for more information.

-
-
-

Below is an example of restoring a backup that is store in AWS S3.

-
-
-
-
$ cbbackupmgr restore -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging \
-  -c http://10.101.101.112:8091 -u Administrator -p password
-
-(1/1) Restoring backup 2020-03-19T15_35_00.467487218Z
-Copied all data in 28.048019272s (Avg. 103.21KB/Sec)     7303 items / 2.82MB
-beer-sample             [==========================================] 100.00%
-
-Restore bucket 'beer-sample' succeeded
-Mutations restored: 7303, Mutations failed to restore: 0
-Deletions restored: 0, Deletions failed to restore: 0
-Skipped due to purge number or conflict resolution: Mutations: 0 Deletions: 0
-Restore completed successfully
-
-
-
-

DISK REQUIREMENTS

-
-

As discussed in Staging Directory, you will be required to provision enough -disk space to store all the keys for your dataset on disk during a backup. We -would recommend doing a simple calculation to determine the approximate size of -the staging directory.

-
-
-

Using the formula below, you can calculate the approximate size of the staging -directory in Gigabytes:

-
-
-
-
(${NUMBER_OF_ITEMS} * (${AVERAGE_KEY_SIZE_IN_BYTES} + 30)) / (1024 ^ 3)
-
-
-
-

Note that this is a rough estimate which doesn’t account for factors such as -fragmentation, however, it should be a good starting point. Using this formula -and given a dataset with 50 Million keys with an average size of 75 bytes, we’d -expect to need to provision about 5GiB of disk space.

-
-
-

When approximating the size of the staging directory, we don’t need to account -for the size of the document values because they are never stored on disk; they -are uploaded directly to object store.

-
-
-
-

COSTING

-
-

Before using any cbbackupmgr sub-commands, it’s worth ensuring that you -understand the costing related to using your chosen cloud provider; often the -pattern being that it’s cheap to upload/store data, but (comparatively) -expensive to access/download (to the wider internet). We recommend using one of -the following calculators.

-
-
-
-
AWS
-
-

https://calculator.s3.amazonaws.com/index.html

-
-
Azure
-
-

https://azure.microsoft.com/en-gb/pricing/calculator

-
-
GCP
-
-

https://cloud.google.com/products/calculator

-
-
-
-
-
BACKUP
-
-

Backing up data from outside/inside AWS S3 is cheap; this is because at the time of writing, -it doesn’t cost anything to transfer data into S3 (you only pay for the storage/requests).

-
-
-
-
RESTORE
-
-

Restoring data is another matter, AWS S3 charges users for pulling data from AWS onto the -internet. This means that restoring large datasets can become quite costly if your cluster -is not in AWS. Before performing a restore, use info (as described below in Interrogating -backups) to determine the size of your backup. You can then use this to calculate how much -it will cost to restore your backup.

-
-
-

At the time of writing, restoring a backup to a cluster stored inside AWS S3 will not be -significantly costly since AWS do not charge for the bandwidth inside AWS. No matter -whether your cluster is hosted in/outside AWS it’s worth calculating the costs before -performing a restore.

-
-
-
-
-
-

MERGING

-
-

One of the main reasons for merging incremental backups is to save disk space. In AWS S3 -space is cheap and bandwidth (to the broader internet) is expensive. This means that there -isn’t a financially viable reason for merging cloud backups. For this reason merging incremental -backups stored in the cloud is not supported.

-
-
-

Restoring will continue to support applying incremental backups in chronological order in the -same fashion that it’s would when merging e.g. you will end up with the same data in your -Couchbase cluster.

-
-
-
-

INTERROGATING BACKUPS

-
-

Several tools have been made available for use with archives stored directly in the -cloud, currently these are:

-
-
-
-
examine
-
-

cbbackupmgr-examine(1)

-
-
info
-
-

cbbackupmgr-info(1)

-
-
-
-
-

EXAMINE

-
-

Examine can be used to query whether a document with the given key exists in given -collection (possibly across multiple backups), the examine tools supports directly -querying the data in S3.

-
-
-
-
$ cbbackupmgr examine -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging \
- --collection-string beer-sample --key '21st_amendment_brewery_cafe'
-
-   Key: 21st_amendment_brewery_cafe
-   SeqNo: 5
-   Backup: 2020-01-08T17_21_20.232087665Z
-   Deleted: false
-   Size: 27B (key), 29B (meta), 666B (value)
-   Meta: {"flags":33554432,"cas":1578502228728479744,"revseqno":1,"datatype":1}
-   Value: {"address":["563 Second Street"],"city":"San Francisco","code":"94107","country":"United States","description":"The 21st Amendment Brewery offers a variety of award winning house made brews and American grilled cuisine in a comfortable loft like setting. Join us before and after Giants baseball games in our outdoor beer garden. A great location for functions and parties in our semi-private Brewers Loft. See you soon at the 21A!","geo":{"accuracy":"ROOFTOP","lat":37.7825,"lon":-122.393},"name":"21st Amendment Brewery Cafe","phone":"1-415-369-0900","state":"California","type":"brewery","updated":"2010-10-24 13:54:07","website":"http://www.21st-amendment.com/"}
-
-
-
-

To learn more about examine options see cbbackupmgr-examine(1).

-
-
-
-

INFO

-
-

The info command can be used to query a broader archive to understand its structure and to gain -an understanding of what data is backed up and where.

-
-
-
-
$ cbbackupmgr info -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging
-
-Name  | Size  | # Backups  |
-rift  | 796B  | 1          |
-
-  +  Backup                          | Size  | Type  | Source                  | Range  | Events  | Aliases  | Complete  |
-  +  2020-01-08T17_21_20.232087665Z  | 796B  | FULL  | http://172.20.1.1:8091  | N/A    | 0       | 0        | true      |
-
-    -    Bucket       | Size  | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
-    -    beer-sample  | 790B  | 7303   | 7303       | 0           | 1      | 0    | 0        | 0     |
-
-
-
-

To learn more about info options see cbbackupmgr-info(1).

-
-
-
-
-

ARCHIVE LOCKING

-
-

It’s important that only one instance of cbbackupmgr has access to the archive at a time; this is -enforced using a lockfile meaning most of the time you shouldn’t need to worry about this. However, -there are some situations where cbbackupmgr may fail to ensure exclusive access to the archive:

-
-
-
    -
  1. -

    Another process (on another machine, or the local machine) already has an active lockfile.

    -
    2. -
  2. -

    A stale lockfile exists which belongs to a system with a different hostname.

    -
    3. -
-
-
-

In cases where cbbackupmgr fails to lock an archive a few simple steps can be taken:

-
-
-
    -
  1. -

    Manually ensure that nobody else is using the archive

    -
    2. -
  2. -

    If you are certain nobody else is using the archive, locate the lockfile in S3 (it has the -format lock-${UUID}.lk and is stored in the top-level of the archive).

    -
    3. -
  3. -

    Remove the lockfile and try to continue using the archive with your own instance of cbbackupmgr.

    -
    4. -
-
-
-

It’s extremely important that you only manually remove the lockfile if you a certain that there -isn’t another instance of cbbackupmgr using the archive. Having two instances of cbbackupmgr running -against the same archive could cause data loss through overlapping key prefixes.

-
-
-

Below is an example of an archive which contains a lockfile from a system that crashed where the -lockfile was never cleaned up.

-
-
-
-
$ aws s3 ls s3://backups --recursive
-2020-04-27 09:34:10        120 archive/.backup
-2020-04-27 09:34:23         34 archive/lock-14eb923b-60a7-480a-849e-8af48e47f9ea.lk
-2020-04-27 09:34:10        520 archive/logs/backup-0.log
-2020-04-27 09:34:10        651 archive/repo/backup-meta.json
-
-
-
-

If we attempt to use cbbackupmgr to create a backup, we should see a message similar to the one -below:

-
-
-
-
$ cbbackupmgr backup -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging \
-  -c 172.20.1.1:8091 -u admin -p password
-Error backing up cluster: the process '{PID}' running on '{HOSTNAME}' already holds the lock
-
-
-
-

In this case, cbbackupmgr will not remove the lock automatically since it cannot safely determine -whether the other process is active or not. We can use the information about which machine the other -instance of cbbackupmgr is running on to check whether it is active. If this machine has crashed and -that instance of cbbackupmgr is no longer using the archive, we can manually remove the lockfile.

-
-
-
-
$ aws s3 rm s3://backups/archive/lock-14eb923b-60a7-480a-849e-8af48e47f9ea.lk
-delete: s3://backups/archive/lock-14eb923b-60a7-480a-849e-8af48e47f9ea.lk
-
-
-
-

If we attempt to perform the backup once again we will see that it continues successfully; in the -case that the other machine failed during a backup you may be asked to purge the previous backup -using the --purge flag before you can create a new backup:

-
-
-
-
$ cbbackupmgr backup -a s3://bucket/archive -r repo --obj-staging-dir /mnt/staging \
-  -c 172.20.1.1:8091 -u admin -p password
-
-Copied all data in 1m2.525397237s (Avg. 51.61KB/Sec)      7303 items / 3.12MB
-beer-sample             [===========================================] 100.00%
-
-Backup successfully completed
-Backed up bucket "beer-sample" succeeded
-Mutations backedup; 7303, Mutations failed to backup: 0
-Deletions backedup: 0, Deletions failed to backup: 0
-
-
-
-
-
-
-

COMPATIBLE OBJECT STORES

-
-
-

cbbackupmgr is tested against the cloud providers that are supported, however, in some cases it will -work with compatible object stores e.g. Localstack/Scality. It’s important to note that experience -may be different when interacting with compatible object stores because some have slightly different -behaviors which cbbackupmgr may not explicitly handle.

-
-
-

AWS

-
-

It should be possible to use cbbackupmgr with S3 compatible object stores, however, there are some -things that need to be taken into consideration. First and foremost is the features that cbbackupmgr -leverages. Below is a list of S3 API features that cbbackupmgr uses but not all compatible object -stores support:

-
-
- -
-
-

It’s important that you check whether these features are implemented on your S3 compatible object -store because without them cbbackupmgr will not work as expected.

-
-
-

AWS also has a slightly newer virtual addressing style the documentation for which can be found at -https://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html. Not all S3 compatible object -stores support this style of addressing. The errors that are returned by the SDK (and therefore -cbbackupmgr) in these cases are not always clear. Before raising a support ticket about cbbackupmgr -not working with an S3 compatible object store you should first try using the ---s3-force-path-style argument. This will force cbbackupmgr to use the old path style addressing. -From our testing with S3 compatible object stores it’s very common for this flag to be required.

-
-
-
-

Azure/GCP

-
-

As with AWS S3 support, Azure/GCP should work compatible object storage -solutions. You also shouldn’t need to provide any specific flags when using -compatible storage solutions.

-
-
-

It should however, be noted that Azure/GCP will only work when the underlying -compatible object storage solution implements all the required features and it -should be noted that this behavior is not exhaustively validated by Couchbase.

-
-
-
-
-
-

CLOUD PROVIDER SPECIFIC FEATURES

-
-
-

As stated above in the 'Compatible Object Stores' section it’s possible to use cbbackupmgr with -other providers which expose an S3 compatible API. It’s important to note that some -features may only be accessible to those using the AWS.

-
-
-

AWS

-
-

When running cbbackupmgr in an AWS instance, it may use the EC2 instance metadata to get -credentials. This is disabled by default, however, may be enabled by either supplying the ---obj-auth-by-instance-metadata flag or setting the CB_AWS_ENABLE_EC2_METADATA environment -variable to true.

-
-
-

For example, if we wanted to use cbbackupmgr with the EC2 instance metadata we would: -. Create a role with a policy which allows S3 data manipulation (e.g. S3 Full Admin) -. Attach that role to the instance -.. Run export CB_AWS_ENABLE_EC2_METADATA=true to enable fetching EC2 instance metadata -.. Add the --obj-auth-by-instance-metadata flag to your cbbackupmgr command. -. Run cbbackupmgr as described elsewhere in this tutorial

-
-
-
-

Azure (7.1.2+)

-
-

When running in an Azure VM, cbbackupmgr will attempt to fetch a service -principle token from the environment which will be used to authenticate against -blob storage. In this case, the correct RBAC permissions should be provided to -allow cbbackupmgr access to the chosen storage account. As apposed to AWS, -this behavior is enabled by default.

-
-
-
-

GCP

-
-

When running in Google Compute, cbbackupmgr will using the instance metadata -server to fetch credentials which may be used to authenticate against Google -Storage. As with AWS/Azure, the correct RBAC permissions should be provided to -allow cbbackupmgr access to the given bucket. As apposed to AWS, this -behavior is enabled by default.

-
-
-
-
-
-

RBAC

-
-
-

It’s quite common to run cbbackupmgr with an account with limited permissions, this section covers -any cloud provider specific permissions which are required. Please note that any permissions listed -in the following sections are subject to change between releases.

-
-
-

AWS

-
-

The following is a list of the actions required by cbbackupmgr when interacting with a remote -archive in AWS.

-
-
-
    -
  • -

    AbortMultipartUpload

    -
    • -
  • -

    CompleteMultipartUpload

    -
    • -
  • -

    CreateMultipartUpload

    -
    • -
  • -

    DeleteObject

    -
    • -
  • -

    DeleteObjects

    -
    • -
  • -

    GetObject

    -
    • -
  • -

    HeadObject

    -
    • -
  • -

    ListObjectsV2

    -
    • -
  • -

    ListObjects

    -
    • -
  • -

    ListParts

    -
    • -
  • -

    PutObject

    -
    • -
-
-
-
-

Azure

-
-

The following are the required RBAC roles required by cbbackupmgr when -interacting with a remote archive in Azure.

-
-
-
    -
  • -

    Storage Blob Data Owner or Storage Blob Data Contributor

    -
    • -
-
-
-
-

GCP

-
-

The following are the required RBAC roles required by cbbackupmgr when -interacting with a remote archive in GCP.

-
-
-
    -
  • -

    Storage Object Admin

    -
    • -
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-config(1), -cbbackupmgr-backup(1), -cbbackupmgr-restore(1), -cbbackupmgr-merge(1), -cbbackupmgr-examine(1) -cbbackupmgr-info(1)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-collect-logs.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-collect-logs.html deleted file mode 100644 index 0c48915c..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-collect-logs.html +++ /dev/null @@ -1,821 +0,0 @@ - - - - - - - - - -cbbackupmgr-collect-logs(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr collect-logs [--archive <archive_dir>] [--output-dir <dir>]
-                         [--redact] [--salt <salt>] [--upload-host <url>]
-                         [--customer <customer-name>] [--upload-proxy <proxy>]
-                         [--ticket <ticket-number>] [--obj-access-key-id <access_key_id>]
-                         [--obj-cacert <cert_path>] [--obj-endpoint <endpoint>]
-                         [--obj-no-ssl-verify] [--obj-region <region>]
-                         [--obj-staging-dir <staging_dir>] [--obj-secret-access-key <secret_access_key>]
-                         [--s3-force-path-style]
-
-
-
-
-

DESCRIPTION

-
-
-

Collects debugging information to make it easier to debug user issues. This -command collects the logs, the output of the info command, the range.json files -from each backup, the seqnos and failover logs from each backup, and the -backup_meta.json files from each repo. This data is put into a zip file that -contains the date that the collect-info command was run. This command will only -collect meta from the backup archive.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of parameters for the compact command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The location of the archive directory.

-
-
-o,--output-dir <dir>
-
-

Specifies a path where the zip file that this command writes should be -written. If this flag is not used then the zip file is written to the logs -directory of the archive the command is run against.

-
-
-
-
-
-

Optional

-
-
-
--redact
-
-

When the redact flag is specified a redacted copy of the logs will be -produced, in the same directory as the unredacted version. The redacted -zip directory will be prefixed by `redacted-`.

-
-
--salt <salt>
-
-

Specifies the salt to used to hash the data. If possible the salt should be -the same used by Couchbase Server to allow the logs to be easily cross -referenced. The Couchbase Server salt is at the top of the couchbase.log in -the zip file generated by cbcollect_info. If no salt is specified it will be -auto-generated.

-
-
--upload-host <url>
-
-

Specifies the fully-qualified domain name of the host you want the logs -uploaded to. The protocol prefix of the domain name, https://, is optional. -It is the default-only supported protocol.

-
-
--customer <customer-name>
-
-

Specifies the customer name. This value must be a string whose maximum length -is 50 characters. Only the following characters can be used: [A-Za-z0-9_.-]. -If any other characters are included, the request is rejected.

-
-
--upload-proxy <upload-proxy>
-
-

Specifies a proxy for upload.

-
-
--ticket <ticket-number>
-
-

Specifies the Couchbase Support ticket-number. The value must be a string -with a maximum length of 7 characters, containing only digits in the range of -0-9.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-
-
-

EXAMPLES

-
-
-

To generate debugging information for an archive dir "/data/backup" and output -the file to "/data/output/logs" run the following command.

-
-
-
-
$ cbbackupmgr collect-logs -a /data/backup
-
-
-
-

To generate debugging information for an archive dir "/data/backup" and output -the file to "/tmp" run the following command.

-
-
-
-
$ cbbackupmgr collect-logs -a /data/backup -o /tmp
-
-
-
-

To upload the logs directly to support use:

-
-
-
-
$ cbbackupmgr collect-logs -a /data/backup -o /tmp \
-  --upload-host uploads.couchbase.com \
---customer CUSTOMER_NAME --ticket 378211
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-info(1),

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-compact.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-compact.html deleted file mode 100644 index cbbbee67..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-compact.html +++ /dev/null @@ -1,611 +0,0 @@ - - - - - - - - - -cbbackupmgr-compact(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr compact [--archive <archive_dir>] [--repo <repo_name>]
-                    [--backup <backup_name>]
-
-
-
-
-

DESCRIPTION

-
-
-

Compacting backups created with a 6.6.0+ version of cbbackupmgr is no longer -supported/required due to changes in the storage format which significantly -reduced backup fragmentation.

-
-
-

Compacts a backup in a backup repository in order to free up disk space. Should -be run on each backup after it completes. The amount of freed disk space is -displayed when the compaction completes.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of parameters for the compact command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The location of the archive directory.

-
-
-r,--repo <repo_name>
-
-

The name of the backup repository where the backup to be compacted is -located.

-
-
--backup <backup_name>
-
-

The name of the backup in the backup repository to compact or an index -value which references an incremental backup to compact. Valid index values -are any positive integer, "oldest", and "latest". If a positive integer is -used then it should reference the index of the incremental backup to compact -starting from the oldest to the most recent backup. For example, "1" -corresponds to the oldest backup, "2" corresponds to the second oldest backup, -and so on. Specifying "oldest" means that the index of the oldest backup -should be used and specifying "latest" means the index of the most recent -backup should be used.

-
-
-t,--threads <num>
-
-

Specify the number of files to be compacted in parallel. By default it is set -to 1, but it is recommended to set to match the number of CPUs in the system -although it can vary depending on the available RAM.

-
-
-
-
-
-
-
-

EXAMPLES

-
-
-

The compact command is used to compact backups in order to free disk space and -should be run after each backup. Below is an example of how to run the -compaction command. The /data/backup directory is used as the archive directory -and the backup repository is named "example".

-
-
-
-
$ cbbackupmgr config -a /data/backup -r example
-
-
-
-
-
$ cbbackupmgr backup -a /data/backup -r example \
- -c 127.0.0.1:8091 -u Administrator -p password
-
-
-
-
-
$ cbbackupmgr compact -a /data/backup -r example \
- --backup 2016-02-12T12_45_33.329408761-08_00
-
-
-
-
-
Compaction succeeded, 58589184 bytes freed
-
-
-
-
-
-

DISCUSSION

-
-
-

The compact command can be run after a backup to reclaim any fragmented disk -space in the backup data files. It should be noted however that compaction may -be run during the backup itself if the fragmentation in the data file begins to -grow too big. Although the compaction can run during the backup it is not -guaranteed to be run once the backup has completed, hence the reason for the -compact command.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
-
-
-
-
-

FILES

-
-
-
-
shard-*.fdb
-
-

The compact command will compact all shard-*.fdb files in the backup -specified by the compact command.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-backup(1), -cbbackupmgr-info(1), -cbbackupmgr-strategies(1)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-config.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-config.html deleted file mode 100644 index bd431d20..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-config.html +++ /dev/null @@ -1,1064 +0,0 @@ - - - - - - - - - -cbbackupmgr-config(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr config [--archive <archive_dir>] [--repo <repo_name>]
-                   [--include-data <collection_string_list>]
-                   [--exclude-data <collection_string_list>]
-                   [--disable-bucket-config] [--disable-views] [--disable-eventing]
-                   [--disable-gsi-indexes] [--disable-ft-indexes]
-                   [--disable-data] [--vbucket-filter <integer_list>]
-                   [--disable-cluster-analytics] [--disable-analytics] [--disable-ft-alias]
-                   [--disable-bucket-query] [--disable-cluster-query]
-                   [--obj-access-key-id <access_key_id>] [--obj-cacert <cert_path>]
-                   [--obj-endpoint <endpoint>] [--obj-no-ssl-verify]
-                   [--obj-region <region>] [--obj-staging-dir <staging_dir>]
-                   [--obj-secret-access-key <secret_access_key>]
-                   [--s3-force-path-style] [--s3-log-level <level>]
-                   [--readme-author <name>] [--point-in-time]
-                   [--encrypted] [--passphrase <passphrase>]
-                   [--encryption-algo <algo>] [--derivation-algo <algo>]
-                   [--km-key-url <url>] [--km-endpoint <endpoint>]
-                   [--km-region <region>] [--km-access-key-id <id>]
-                   [--km-secret-access-key <key>] [--km-auth-file <path>]
-
-
-
-
-

DESCRIPTION

-
-
-

Creates a new backup repository with a new backup configuration. The -configuration created is used for all backups in this backup repository.

-
-
-

By default a backup configuration is created that will backup all buckets in the -cluster. Each bucket will have its bucket configuration, views definitions, gsi -index definitions, full-text index definitions, and data backed up. Specifying -various flags, the config command can modify the configuration to backup a -subset of the data.

-
-
-

Once a backup repository is created its configuration cannot be changed.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of required and optional parameters for the config command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The directory where the new backup repository will be created. If it does -not already exist, an attempt will be made to create it. When configuring -an archive in S3 you must prefix the archive path with s3://${BUCKET_NAME}/.

-
-
-r,--repo <repo_name>
-
-

The name of the new backup repository.

-
-
-
-
-
-

Optional

-
-
-
--include-data <collection_string_list>
-
-

Modifies the repository configuration to backup only the data specified in -the <collection_string_list>. This flag takes a comma separated list of -collection strings and can’t be specified at the same time as ---exclude-data. Note that including data at the scope/collection level is -an Enterprise Edition feature.

-
-
--exclude-data <collection_string_list>
-
-

Modifies the repository configuration to skip restoring the data specified in -the <collection_string_list>. This flag takes a comma separated list of -collection strings and can’t be specified at the same time as ---include-data. Note that excluding data at the scope/collection level is -an Enterprise Edition feature.

-
-
--disable-views
-
-

Modifies the repository configuration to disable backing up view definitions -for all buckets.

-
-
--disable-gsi-indexes
-
-

Modifies the repository configuration to disable backing up gsi index -definitions for all buckets.

-
-
--disable-ft-indexes
-
-

Modifies the repository configuration to disable backing up full-text index -definitions for all buckets.

-
-
--disable-ft-alias
-
-

Modifies the repository configuration to disable backing up full-text alias -definitions.

-
-
--disable-data
-
-

Modifies the repository configuration to disable backing up all key-value -data for all buckets.

-
-
--disable-cluster-analytics
-
-

Modifies the repository configuration to disable backing up the cluster level -analytics data. For example, the Synonyms are backed up at the cluster level.

-
-
--disable-analytics
-
-

Modifies the repository configuration to disable backing up analytics data.

-
-
--disable-eventing
-
-

Modifies the repository configuration to disable backing up the Eventing -service metadata.

-
-
--disable-bucket-query
-
-

Modifies the repository configuration to disable backing up Query Service -metadata for all buckets.

-
-
--disable-cluster-query
-
-

Modifies the repository configuration to disable backing up the cluster level -Query Service metadata.

-
-
--vbucket-filter <list>
-
-

Specifies a list of VBuckets that should be backed up in the repo being -created. VBuckets are specified as a comma separated list of integers. If -this parameter is not set then all vbuckets are backed up. The comma -separated list can also take sequences denoted as e.g: 0-5 this is equivalent -to 0,1,2,3,4,5.

-
-
--readme-author <name>
-
-

Specifies the name of the creator of the repository. This information -will be recorded in a README file at the top-level of the repository.

-
-
--point-in-time
-
-

(BETA) This enables Point in Time feature. Which is currently in Beta and -is not supported, this should only be used in test environments. When it’s -enabled cbbackupmgr will get historical data from the cluster. This will -allow restoring to a single point in time. This will increase the amount of -space needed to store the backup. The cluster also has to have Point In -Time configured to use this feature.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-

Encryption (Developer Preview)

-
-
-
--encrypted
-
-

Enables configuring an encrypted repository.

-
-
--encryption-algo <algo>
-
-

The encryption algorithm to use. Currently supported are -[AES256CBC, AES256GCM]. Defaults to AES256GCM.

-
-
--derivation-algo <algo>
-
-

The derivation algorithm used when running with passphrase that creates -the repository encryption key. Currently supported [ARGON2ID, PBKDF2]. -Defaults to ARGON2ID.

-
-
--passphrase <passphrase>
-
-

Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing.

-
-
--km-key-url <url>
-
-

Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable CB_KM_KEY_URL. For more on how to authenticate using the different -providers see cbbackupmgr-encryption(7).

-
-

For AWS the expected key format is awskms://<KEY-ID|KEY-ALIAS>, for example -awskms://alias\keyAlias.

-
-
-

For GCP the expected key format is gcpkms://<KEY-RESOURCE-ID>, for example -gcpkms://projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key.

-
-
-

For Azure key vault the expected key format is azurekeyvault://<KEY-IDENTIFIER> -for example: -azurekeyvault://vault-name.vault.azure.net/object-type/object-name/object-version.

-
-
-

For HashiCorp Vault the expected format is hashivaults://<HOST>/<KEY-NAME> -for example: -hashivaults://127.0.0.1:8200/keyName.

-
-
-
--km-region <region>
-
-

Required when using AWS KMS, it allows you to set the key region.

-
-
--km-endpoint <endpoint>
-
-

The host or address to use as your KMS. It will override the default SDK one.

-
-
--km-access-key-id <id>
-
-

The user ID used to connect to the key management service. It can also be -provided via CB_KM_ACCESS_KEY_ID environmental variable. Please refer -to cbbackupmgr-encryption(7) for the required authentication for each -provider.

-
-
--km-secret-access-key <key>
-
-

The key used to connect to the key management service. It can also be -provided via the CB_KM_SECRET_ACCESS_KEY environmental variable. Please -refer to cbbackupmgr-encryption(7) for the required authentication for -each provider.

-
-
--km-auth-file <path>
-
-

The path to a file containing the authentication credentials for the key - management service. It can also be provided via the CB_KM_AUTH_FILE - environmental variable. Please refer to cbbackupmgr-encryption(7) for the - required authentication for each provider. -:!config:

-
-
-
-
-
-
-
-

EXAMPLES

-
-
-

The config command is used to create a backup repository and define the -repositories backup configuration. In the examples below, the backup archive is -located at /data/backups. Since this is the first backup repository we are -creating in a new backup archive, we need to ensure that /data/backups is an -empty directory. Archives are created automatically if an archive doesn’t -already exist at the archive path, but are only created if the directory at that -path is empty. In order to create a backup repository called "example" with the -default configuration use the following command:

-
-
-
-
$ cbbackupmgr config -a /data/backups -r example
-
-
-
-

Upon creation of a new backup repository there will be a new directory in the -backup archive containing a backup configuration. You can see this new directory -using the cbbackupmgr-info[1] command.

-
-
-
-
$ cbbackupmgr info -a ~/data/backups
-Name     | UUID                                  | Size  | # Repos  |
-backups  | 0db3337c-96b0-4b3a-a7fb-bbfd53790e5f  | 0B    | 1        |
-
-*  Name     | Size  | # Backups  |
-*  example  | 0B    | 0          |
-
-
-
-

Using the optional parameters of the create command, you can modify the backup -configuration settings. To create a backup repository with a backup -configuration that only backs up the buckets "airline_data" and "ticket_prices" -and does not back up bucket configuration data, you can run the following:

-
-
-
-
$ cbbackupmgr config -a /data/backups -r example \
- --include-data airline_data,ticket_prices --disable-bucket-config
-
-
-
-

To create a backup repository with a backup configuration that backs up only -data on all buckets, you can run the following command:

-
-
-
-
$ cbbackupmgr config -a /data/backups -r example --disable-bucket-config \
- --disable-views --disable-gsi-indexes --disable-ft-indexes
-
-
-
-

When creating an archive in an object store such as S3 due care must be taken -to ensure that the provided directory is empty and can be managed by cbbackupmgr.

-
-
-
-
$ cbbackupmgr config -a s3://bucket/archive -r repo --obj-staging-dir /staging
-
-
-
-
-
-

DISCUSSION

-
-
-

Though not required, it is recommended that there is a single backup repository -per cluster. Backup repositories are managed so that all backups can be taken -incrementally and merged together as backup data ages. Backing up in this manner -allows backups to transfer the least amount of data necessary which reduces back -up time and cluster resource usage. For more details on backup strategies see -cbbackupmgr-strategies[7].

-
-
-

When a backup repository is created, it should only be modified by the -cbbackupmgr utility. Any modifications done outside of this utility can cause -corruption of backup files.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
CB_ENCRYPTION_PASSPHRASE
-
-

Specifies the passphrase used for encryption.

-
-
CB_KM_KEY_URL
-
-

Specifies the URL identifying the encryption key on the KMS. See ---km-key-url for the expected format and accepted KMSs.

-
-
CB_KM_ACCESS_ID
-
-

Specifies the key/user ID used to connect to the KMS.

-
-
CB_KM_SECRET_ACCESS_KEY
-
-

Specifies the secret key/token used to connect to the KMS.

-
-
CB_KM_AUTH_FILE
-
-

Specifies a path to a file containing the required credentials to connect to -the KMS.

-
-
-
-
-
-
-

FILES

-
-
-
-
backup-meta.json
-
-

The config command creates a backup configuration file in the backup -repository called backup-meta.json. This file contains the backup -configuration for all backups run in the backup repository it was created -in. It should never be modified and treated as a read-only file.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-info(1), -cbbackupmgr-strategies(1), -cbbackupmgr-cloud(7), -cbbackupmgr-encryption(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-encryption.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-encryption.html deleted file mode 100644 index fcc0ae8e..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-encryption.html +++ /dev/null @@ -1,900 +0,0 @@ - - - - - - - - - -cbbackupmgr-encryption(7) - - - - - -
-
-

DESCRIPTION

-
-
-

A tutorial that explains how to use the encryption feature of cbbackupmgr -Enterprise Edition.

-
-
-
-
-

TUTORIAL

-
-
-

In this tutorial we will cover how to create encrypted backup repositories as -well as how to do backups, merges and restores in said repositories. To follow -this tutorial you will need an Enterprise Edition version of cbbackupmgr and -Couchbase Server. For later steps we will explain how to use external Key -Management Solutions (KMS), so you may need access to one of those.

-
-
-

Configuring a backup

-
-

To start using encrypted backups you will need to create a new encrypted backup -repository. This can be achieved with the config command, it is worth noting -the are two separate modes of configuration:

-
-
-

1. Passphrase mode (Development Only)

-
-

In this mode the command takes a passphrase via the command line argument ---passphrase or via the CB_ENCRYPTION_PASSPHRASE environmental variable.

-
-
-

An example of how to use this new mode is shown below:

-
-
-
-
$ cbbackupmgr config -a /backups/encrypted -r passphrase_repo --encrypted --passphrase couchbase
-Backup repository `passphrase` created successfully in archive `/backups/encrypted`
-
-
-
-

The passphrase mode uses the given passphrase to derive an encryption key using -ARGONID by default. The derivation algorithm can be changed using the ---derivation-algo flag. The resulting key is then used to -encrypt the auto-generated repository key which is used to encrypt the backup -data. Due to the inherent insecurity of human friendly passphrases using -passphrase mode in production is strongly discouraged.

-
-
-
-

2. KMS mode

-
-

In this mode the command will attempt to communicate with an external KMS to -encrypt the repository key. Note that the external key will not directly -encrypt the data but instead envelopes the encryption key used to encrypt the -data. This minimizes the number of request done to the external KMS as well as -the system latency.

-
-
-

The currently supported external KMS are:

-
-
-
    -
  1. -

    Amazon Web Services Key Management Service (AWS KMS)

    -
    2. -
  2. -

    Google Cloud Key Management Service (GCP KMS)

    -
    3. -
  3. -

    Azure Key Vault

    -
    4. -
  4. -

    HashiCorp Vault Transit Secret Engine

    -
    5. -
-
-
-

Depending on the KMS you choose the setup will look slightly different but -vaguely all four options have the ability to pick up the needed credentials -from the environment. In the next sections we will cover each KMS specifics and -give examples of how to set up. For all the KMSs you will have to provide -a URL to the key to use for encryption. This is done via the --km-key-url -or the CB_KM_KEY_URL environmental variable. The schema varies between -provider but will be covered in the following sections.

-
-
-

By default, when configuring backups the encryption algorithm will be set to -AES256GCM. It can be changed to AES256CBC by using the option ---encryption-algo AES256CBC.

-
-
-
AWS KMS
-
-

To use AWS KMS you provide the URL to the -Key Identifier -and prefix it with awskms://, for AWS you must also provide the region -via the --km-region environmental variable.

-
-
-

The supported authorization techniques are:

-
-
-
    -
  1. -

    When running in EC2, credentials may be obtained via the instance metadata -service by setting/supplying:

    -
    -
      -
    • -

      CB_AWS_ENABLE_EC2_METADATA=true

      -
      • -
    -
    -
    2. -
  2. -

    Providing a set of environment variables including:

    -
    -
      -
    • -

      CB_KM_ACCESS_KEY_ID

      -
      • -
    • -

      CB_KM_SECRET_ACCESS_KEY

      -
      • -
    -
    -
    3. -
  3. -

    Loading credentials from the shared config files located at:

    -
    -
      -
    • -

      $HOME/.aws/config

      -
      • -
    • -

      $HOME/.aws/credentials

      -
      • -
    -
    -
    4. -
  4. -

    Providing static config/credentials using the cli flags:

    -
    -
      -
    • -

      --km-access-key-id

      -
      • -
    • -

      --km-secret-access-key

      -
      • -
    -
    -
    5. -
-
-
-

Below is an example of configuring an encrypted repository using AWS:

-
-
-
-
$ cbbackupmgr config -a /backups/encrypted -r aws_repo --encrypted \
-  --km-key-url 'awskms://alias/cbbackupmgrEncrypt' --km-region 'us-east-1'
-
-
-
-

Note that if you want to use a private cloud compatible with AWS KMS you -can provide --km-endpoint to override the AWS endpoint.

-
-
-
-
GCP KMS
-
-

To use GCP KMS you have to provide the -Key Resource ID -prefixed with gcpkms:// either via the environment, or the CLI flag

-
-
-

The supported authorization techniques are:

-
-
-
    -
  1. -

    When running in Google Compute Engine it can be retrieved from the -environment automatically.

    -
    2. -
  2. -

    A service account can be pass via the GOOGLE_APPLICATION_CREDENTIALS -environmental variable.

    -
    3. -
  3. -

    Explicitly passing the path to a service auth file using either:

    -
    -
      -
    • -

      --km-auth-file CLI argument

      -
      • -
    • -

      CB_KM_AUTH_FILE environmental variable

      -
      • -
    -
    -
    4. -
-
-
-

For more information refer to -GCPs documentation.

-
-
-

An example of configuring an encrypted repository using GCP can be seen below.

-
-
-
-
$ cbbackupmgr config -a /backups/encrypted -r gcp_repo --encrypted \
-  --km-key-url gcpkms://projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key
-
-
-
-
-
Azure Key Vault
-
-

To use Azure Key Vault you have to provide the URL to the -Key ID -prefixed with azurekeyvault:// via either the CLI or environmental variables.

-
-
-

The supported authorization techniques are:

-
-
-
    -
  1. -

    Via the Azure environmental variables as described by the -Azure documentation.

    -
    2. -
  2. -

    Using the AZURE_AUTH_LOCATION environmental variable to point to an Azure -auth setting file.

    -
    3. -
-
-
-

An example of configuring an encrypted repository using Azure can be seen below.

-
-
-
-
$ cbbackupmgr config -a /backups/encrypted -r azure_repo --encrypted \
-  --km-key-url azurekeyvault://{vault-name}.vault.azure.net/{object-type}/{object-name}/{object-version}
-
-
-
-
-
HashiCorp Vault Transit Engine
-
-

To use HashiCorp Vault Transit engine you have to provide the URL to the -key ID in the form [hashivault|hashivaults]://{host}/{key name}. If -hashivaults used then HTTPS will be used to connect to the transit engine.

-
-
-

For HashiCorp vault the only accepted authorization method is TOKEN auth. -This can be provided either via: -* --km-secret-access-key CLI argument -* CB_KM_SECRET_ACCESS_KEY` environmental variable

-
-
-

An example of configuring an encrypted repository using HashiCorp Vault Transit -Engine can be seen below.

-
-
-
-
$ cbbackupmgr config -a /backups/encrypted -r hashicorp_repo --encrypted \
-  --km-key-url hashivault://127.0.0.1:8200/key --km-secret-access-key my_token
-
-
-
-
-
-
-

Running encrypted backups

-
-

For the examples below we will use 'passphrase mode', but they will work the -same with the 'KMS mode' if you replace the --passphrase flag with the -corresponding KMS flags.

-
-
-

To run an encrypted backup you have to first configure you repository as -described in the previous section. Once your encrypted repository is configured -you can run the backup as follows:

-
-
-
-
$ cbbackupmgr backup -c https://localhost:18091 -u Administrator -p asdasd \
-  -a /backups/encrypted -r passphrase_repo --passphrase couchbase
-Backing up to '2021-06-17T16_35_45.08865997+01_00'
-Copied all data in 16.746s (Avg. 2.84MiB/Sec)                                                                                                                                                                                           63288 items / 45.50MiB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 2.84MiB           | Thu, 17 Jun 2021 16:35:45 +0100/s | Thu, 17 Jun 2021 16:36:01 +0100 | 16.788s  |
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 45.50MiB    | 2.84MiB/s         | Thu, 17 Jun 2021 16:35:45 +0100 | Thu, 17 Jun 2021 16:36:01 +0100 | 16.427s  |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 63288    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-Backup completed successfully
-
-
-
-

When doing encrypted backups you will notice that it is slightly slower and -uses slightly more space. This is due to the overhead caused by encrypting -the data, and it is to be expected.

-
-
-

If the passphrase or the KMS details are missing when attempting to do a -backup cbbackupmgr will fail and return an error asking for the correct -credentials.

-
-
-

Note that when using encrypted backups you should always use https as -encrypted backups work locally. Thus, if http is used the data will be -transmitted over the network unencrypted and then encrypted once it reaches the -backup client.

-
-
-

Encrypted restores

-
-

To restore an encrypted backup use the following command:

-
-
-
-
$ cbbackupmgr restore -c https://localhost:18091 -u Administrator -p asdasd \
-  -a /backups/encrypted -r passphrase_repo --passphrase couchbase
-(1/1) Restoring backup '2021-06-17T16_35_45.08865997+01_00'
-Copied all data in 2.03s (Avg. 20.75MiB/Sec)                                                                                                                                                                                            63288 items / 41.49MiB
-[===================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 20.75MiB          | Thu, 17 Jun 2021 16:54:32 +0100/s | Thu, 17 Jun 2021 16:54:34 +0100 | 2.061s   |
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 41.49MiB    | 41.49MiB/s        | Thu, 17 Jun 2021 16:54:32 +0100 | Thu, 17 Jun 2021 16:54:34 +0100 | 1.952s   |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 63288    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-Restore completed successfully
-
-
-
-

Similar to the backup process, restore will also be slightly slower than its -plain text counter-part, and you should use the https endpoints to make sure -that the process is encrypted end-to-end.

-
-
-
-

Info for encrypted repositories

-
-

The info command does not require any changes to work with encrypted backups. -This is because info does not access any of the user data directly it only -checks for size and count.

-
-
-

An info of an encrypted repository would look as follows:

-
-
-
-
$ cbbackupmgr info -a /backups/encrypted -r passphrase_repo --depth 4
-Name            | Size      | # Backups | Encrypted |
-passphrase_repo | 238.15MiB | 2         | true      |
-
-+  Backup                             | Size      | Type | Source         | Cluster UUID                     | Range | Events | Aliases | Complete |
-+  2021-06-17T16_35_45.08865997+01_00 | 238.15MiB | FULL | localhost:9000 | 03e85fa504cb5f50fc5ff1722f052754 | N/A   | 0      | 0       | true     |
-
--    Bucket        | Size      | Items | Mutations | Tombstones | Views | FTS | Indexes | CBAS |
--    travel-sample | 238.15MiB | 63288 | 63288     | 0          | 0     | 0   | 0       | 0    |
-
-~      Scope           | Scope ID | Mutations | Tombstones |
-~      _default        | 0        | 31591     | 0          |
-~      tenant_agent_04 | 8        | 40        | 0          |
-~      tenant_agent_03 | 9        | 33        | 0          |
-~      tenant_agent_02 | 10       | 20        | 0          |
-~      tenant_agent_01 | 11       | 11        | 0          |
-~      tenant_agent_00 | 12       | 2         | 0          |
-~      inventory       | 13       | 31591     | 0          |
-
-
-
-
-
-
-
-

DISCUSSION

-
-
-

In this section we will briefly cover how the encryption model for cbbackupmgr works. -Whenever an encrypted repository is created the tool will auto-generate an encryption -key that we then will encrypt and persist using the external KMS or passphrase provided -by the user.

-
-
-

This means that when using an external KMS the tool will only use 1 API call per -operation. The only exception is info which requires 0.

-
-
-

To avoid issues with repeated nonces we create derived keys per backup, bucket and -vBucket, this means that whilst using AES_256_GCM we can safely encrypt up to 4.3e12 -(4.3 trillion) documents per bucket in each individual backup. After that the chances -of repeated nonces increase which weakens the encryption.

-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-config(1), -cbbackupmgr-backup(1), -cbbackupmgr-restore(1), -cbbackupmgr-merge(1), -cbbackupmgr-examine(1) -cbbackupmgr-info(1)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-examine.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-examine.html deleted file mode 100644 index c7c7a222..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-examine.html +++ /dev/null @@ -1,1332 +0,0 @@ - - - - - - - - - -cbbackupmgr-examine(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr examine [--archive <archive_dir>] [--repo <repo_name>]
-                    [--collection-string <collection_string>] [--key <key>]
-                    [--start start] [--end end] [--search-partial-backups]
-                    [--json] [--no-meta] [--no-xattrs] [--no-value]
-                    [--hex-meta] [--hex-xattrs] [--hex-value]
-                    [--obj-access-key-id <access_key_id>]
-                    [--obj-cacert <cert_path>] [--obj-endpoint <endpoint>]
-                    [--obj-no-ssl-verify] [--obj-region <region>]
-                    [--obj-staging-dir <staging_dir>]
-                    [--obj-secret-access-key <secret_access_key>]
-                    [--s3-force-path-style] [--s3-log-level <level>]
-                    [--passphrase <passphrase>] [--km-key-url <url>]
-                    [--km-endpoint <endpoint>] [--km-region <region>]
-                    [--km-access-key-id <id>] [--km-secret-access-key <key>]
-                    [--km-auth-file <path>] [--point-in-time <time>]
-
-
-
-
-

DESCRIPTION

-
-
-

Examine is an Enterprise Edition command used to find information about a given -key. Examine will produce a timeline highlighting events related to the key -during the backups e.g. first appearance, mutation and deletion.

-
-
-

You may optionally use the --start and --end flags to limit the range of -backups for which an event timeline is produced. See START AND END for more -information.

-
-
-

Examine also supports outputting the event timeline in multiple formats, -including being able to hexdump/include/omit portions of a document.

-
-
-
-
-

OPTIONS

-
-
-

Below is a list of parameters for the examine command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The archive directory to examine. When using examine against an archive stored -in S3 prefix the archive path with s3://${BUCKET_NAME}/

-
-
-r,--repository <repo_name>
-
-

The name of the backup repository to examine.

-
-
--collection-string <collection_string>
-
-

A dot separated collection string representing the bucket/collection which -contains the key you are trying to examine. When interacting with a -collection unaware backup the collection string must only contain a bucket -e.g. --collection-string default. When interacting with a collection aware -backup the collection string must contain the path to a collection e.g. ---collection-string default.scope.collection. Note that you may omit the -scope/collection for collection aware backups which only contain the -_default scope and collection e.g. --collection-string default is -internally translated to --collection-string default._default._default. For -more information see COLLECTION AWARE BACKUPS in cbbackupmgr-backup(1).

-
-
-k,--key
-
-

The name of the key you are trying to find.

-
-
-
-
-
-

Optional

-
-
-
--start <start>
-
-

The first backup to examine. See START AND END for information on what values -are accepted.

-
-
--end <end>
-
-

The final backup to examine. See START AND END for information on what values -are accepted.

-
-
--search-partial-backups
-
-

By default examine will skip backups which are incomplete. This flag may be -supplied to attempt to search incomplete backups, however, the resulting -search may fail, or not find the key you’re looking for.

-
-
-j,--json
-
-

Return any output as a JSON structure.

-
-
--no-meta
-
-

Do not return any of the documents metadata. Mutually exclusive with the ---hex-meta flag.

-
-
--no-xattrs
-
-

Do not return any of the documents extended attributes. Mutually exclusive -with the --hex-xattrs flag.

-
-
--no-value
-
-

Do not return the documents value. Mutually exclusive with the --hex-value -flag.

-
-
--hex-meta
-
-

Return the metadata encoded in hex. Mutually exclusive with the --no-meta -flag.

-
-
--hex-xattrs
-
-

Return the extended attributes encoded in hex. Mutually exclusive with the ---no-xattrs flag.

-
-
--hex-value
-
-

Return the value encoded in hex. Mutually exclusive with the --no-value -flag.

-
-
--point-in-time <time>
-
-

(Beta) Only examine documents up to the given point in time. The value -accepted is ISO8601 date time format (YYYY-MM-DDTHH:MM:SS). This feature is -currently in Beta and is not supported, this should only be used in test -environments.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-

Encryption (Developer Preview)

-
-
-
--passphrase <passphrase>
-
-

Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing.

-
-
--km-key-url <url>
-
-

Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable CB_KM_KEY_URL. For more on how to authenticate using the different -providers see cbbackupmgr-encryption(7).

-
-

For AWS the expected key format is awskms://<KEY-ID|KEY-ALIAS>, for example -awskms://alias\keyAlias.

-
-
-

For GCP the expected key format is gcpkms://<KEY-RESOURCE-ID>, for example -gcpkms://projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key.

-
-
-

For Azure key vault the expected key format is azurekeyvault://<KEY-IDENTIFIER> -for example: -azurekeyvault://vault-name.vault.azure.net/object-type/object-name/object-version.

-
-
-

For HashiCorp Vault the expected format is hashivaults://<HOST>/<KEY-NAME> -for example: -hashivaults://127.0.0.1:8200/keyName.

-
-
-
--km-region <region>
-
-

Required when using AWS KMS, it allows you to set the key region.

-
-
--km-endpoint <endpoint>
-
-

The host or address to use as your KMS. It will override the default SDK one.

-
-
--km-access-key-id <id>
-
-

The user ID used to connect to the key management service. It can also be -provided via CB_KM_ACCESS_KEY_ID environmental variable. Please refer -to cbbackupmgr-encryption(7) for the required authentication for each -provider.

-
-
--km-secret-access-key <key>
-
-

The key used to connect to the key management service. It can also be -provided via the CB_KM_SECRET_ACCESS_KEY environmental variable. Please -refer to cbbackupmgr-encryption(7) for the required authentication for -each provider.

-
-
--km-auth-file <path>
-
-

The path to a file containing the authentication credentials for the key -management service. It can also be provided via the CB_KM_AUTH_FILE -environmental variable. Please refer to cbbackupmgr-encryption(7) for the -required authentication for each provider.

-
-
-
-
-
-
-
-

START AND END

-
-
-

This sub-command accepts a --start and --end flag. These flags accept -multiple values to allow you to flexibly operate on a range of backups.

-
-
-

Indexes

-
-

Indexes may be supplied to operate on a range of backups, for example ---start 1 --end 2 will include start at the first backup and will finish with -the second backup. Note that the first backup is 1 and not 0 and that the ---end flag is inclusive.

-
-
-
-

Short Dates

-
-

Short dates may be supplied in the format day-month-year. For example ---start 01-08-2020 --end 31-08-2020 will operate on all the backups which -were taken during August of 2020. Note that the end date is inclusive.

-
-
-

When supplying short dates, you may supply start or oldest as a placeholder -for the date on which the first backup in this repository was taken. The -keywords end or latest may be used as a placeholder for the date last -backup in the repository was taken.

-
-
-
-

Backup Names

-
-

Backup names may be supplied as they exist on disk. For example ---start 2020-08-13T20_01_08.894226137+01_00 --end 2020-08-13T20_01_12.348300092+01_00 -will cause the sub-command to operate on all the backups which inclusively fall -between these two backups.

-
-
-

When supplying backup names, you may supply start or oldest as a -placeholder for the first backup in the repository. The keywords end or -latest may be used as a placeholder for the final backup in the repository.

-
-
-
-
-
-

SEARCH SPACE

-
-
-

The examine command supports searching though multiple backups, this section -explains how to use examine to search multiple backups and how it behaves when -a repository contains backups from an older version or mixed collection -aware/unaware backups.

-
-
-

BACKUPS FROM OLDER VERSIONS OF CBBACKUPMGR

-
-

The examine command only supports searching backups created using a version of -cbbackupmgr greater than or equal to 6.5.0.

-
-
-

When examine encounters a range of backups where no backups can be searched an -error will be returned to you.

-
-
-

When examine encounters a mixed range of backups where one or more backups can -be searched examine will skip the old backups and only search the supported -backups. Note that a warning will be displayed indicating how many backups were -skipped.

-
-
-
-

SEARCHING COLLECTION AWARE/UNAWARE BACKUPS

-
-

When searching a collection unaware backup, the --collection-string argument -expects only a bucket and will throw an error if provided with a scope or -collection.

-
-
-

When searching a collection aware backup, the --collection-string argument -expects a collection in the format bucket.scope.collection and will throw an -error if provided with a bucket or scope.

-
-
-

See the EXAMPLES section for examples of examining collection aware/unaware -backups.

-
-
-
-

SEARCHING MIXED COLLECTION AWARE/UNAWARE BACKUPS

-
-

When examining a range of backups which transition from collection unaware to -collection aware (for example after upgrading the target cluster to 7.0.0) -examine will stop search upon hitting the first collection aware backup.

-
-
-

Examine will print a warning indicating that it stopped searching prematurely, -this warning will contain the name of the backup which can be passed into the ---start flag to allow you to examine only the collection aware backups.

-
-
-
-
$ cbbackupmgr examine -a $PWD/archive -r repo --collection-string 'travel-sample' --key 'airline_10'
-Warning: Finished examining collection unaware backups, 1 collection aware backup(s) remain(s); use the --start/--end flags to examine the collection aware backup(s) starting at '2021-01-27T13_35_53.284474589Z'
--- Backup 2021-01-27T13_33_08.986105547Z --
-Description: First occurrence of key 'airline_10' in this timeline, document created
-Cluster UUID: 59845c6cdce57b81bd6f450e5043bee7
-Bucket: travel-sample (61ca19c7819478f27e03b398c6a540e7)
-Document:
-  Key: airline_10
-  Sequence Number: 1
-  Deleted: false
-  Size: 159B (Key: 10B) (Meta: 29B) (Value: 120B) (XATTRS: 0B)
-  Meta:
-    Flags: 33554432
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611754171980709888
-    Revseqno: 1
-    Datatype: 3 (snappy,json)
-  Value:
-    {
-        "callsign": "MILE-AIR",
-        "country": "United States",
-        "iata": "Q5",
-        "icao": "MLA",
-        "id": 10,
-        "name": "40-Mile Air",
-        "type": "airline"
-    }
-
-
-
-
-
-
-

EXAMPLES

-
-
-

The examine command can be used to determine whether a collection unaware backup -contains a specific document. See COLLECTION AWARE BACKUPS in -cbbackupmgr-backup(1) for information on what makes a backup collection -aware/unaware.

-
-
-
-
$ cbbackupmgr examine -a $PWD/archive -r repo --collection-string 'travel-sample' --key 'airline_10'
--- Backup 2021-01-27T12_36_59.013718359Z --
-Description: First occurrence of key 'airline_10' in this timeline, document created
-Cluster UUID: d22cb6cbb72f9d6a86b75317478f60b0
-Bucket: travel-sample (2de1c5edcd2daca4a08360c130ea6b19)
-Document:
-  Key: airline_10
-  Sequence Number: 1
-  Deleted: false
-  Size: 159B (Key: 10B) (Meta: 29B) (Value: 120B) (XATTRS: 0B)
-  Meta:
-    Flags: 33554432
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611750671350693888
-    Revseqno: 1
-    Datatype: 3 (snappy,json)
-  Value:
-    {
-        "id": 10,
-        "type": "airline",
-        "name": "40-Mile Air",
-        "iata": "Q5",
-        "icao": "MLA",
-        "callsign": "MILE-AIR",
-        "country": "United States"
-    }
-
-
-
-

The same principle applies when searching a collection aware backup, the only -difference being that the --collection-string argument requires a collection.

-
-
-
-
$ cbbackupmgr examine -a $PWD/archive -r repo --collection-string 'travel-sample.inventory.airline' --key 'airline_10'
--- Backup 2021-01-27T12_38_21.55739452Z --
-Description: First occurrence of key 'airline_10' in this timeline, document created
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-Scope: inventory (8)
-Collection: airline (10)
-Document:
-  Key: airline_10
-  Sequence Number: 44
-  Deleted: false
-  Size: 159B (Key: 10B) (Meta: 29B) (Value: 120B) (XATTRS: 0B)
-  Meta:
-    Flags: 0
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611751097702088704
-    Revseqno: 1
-    Datatype: 3 (snappy,json)
-  Value:
-    {
-        "id": 10,
-        "type": "airline",
-        "name": "40-Mile Air",
-        "iata": "Q5",
-        "icao": "MLA",
-        "callsign": "MILE-AIR",
-        "country": "United States"
-    }
-
-
-
-

When more than one backup exists in a repository, examine will continue -searching and output information for each backup it searches.

-
-
-
-
$ cbbackupmgr examine -a ~/Projects/couchbase-archive -r repo --collection-string 'travel-sample.inventory.airline' --key 'airline_10'
--- Backup 2021-01-27T12_38_21.55739452Z --
-Description: First occurrence of key 'airline_10' in this timeline, document created
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-Scope: inventory (8)
-Collection: airline (10)
-Document:
-  Key: airline_10
-  Sequence Number: 44
-  Deleted: false
-  Size: 159B (Key: 10B) (Meta: 29B) (Value: 120B) (XATTRS: 0B)
-  Meta:
-    Flags: 0
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611751097702088704
-    Revseqno: 1
-    Datatype: 3 (snappy,json)
-  Value:
-    {
-        "id": 10,
-        "type": "airline",
-        "name": "40-Mile Air",
-        "iata": "Q5",
-        "icao": "MLA",
-        "callsign": "MILE-AIR",
-        "country": "United States"
-    }
-
--- Backup 2021-01-27T12_41_13.009708335Z --
-Description: Mutation for document with key 'airline_10'
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-Scope: inventory (8)
-Collection: airline (10)
-Document:
-  Key: airline_10
-  Sequence Number: 75
-  Deleted: false
-  Size: 191B (Key: 10B) (Meta: 29B) (Value: 152B) (XATTRS: 0B)
-  Meta:
-    Flags: 33554438
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611751269058281472
-    Revseqno: 2
-    Datatype: 3 (snappy,json)
-  Value:
-    {
-        "id": 10,
-        "type": "airline",
-        "name": "40-Mile Air",
-        "iata": "Q5",
-        "icao": "MLA",
-        "callsign": "MILE-AIR",
-        "country": "United States",
-        "new_field": "new_value"
-    }
-
--- Backup 2021-01-27T12_42_07.578485175Z --
-Description: Deletion for document with key 'airline_10'
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-Scope: inventory (8)
-Collection: airline (10)
-Document:
-  Key: airline_10
-  Sequence Number: 76
-  Deleted: true
-  Size: 39B (Key: 10B) (Meta: 29B) (Value: 0B) (XATTRS: 0B)
-  Meta:
-    Flags: 0
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611751325090316288
-    Revseqno: 3
-    Datatype: 0 (binary)
-
--- Backup 2021-01-27T12_42_18.322132022Z --
-Description: Collection 'airline' dropped
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-Scope: inventory (8)
-
--- Backup 2021-01-27T12_42_23.72226735Z --
-Description: Scope 'inventory' dropped
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-
--- Backup 2021-01-27T12_42_34.08410595Z --
-Description: Bucket 'travel-sample' dropped
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-
--- Backup 2021-01-27T12_42_35.395081659Z --
-Description: Bucket 'travel-sample' not found
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-
-
-
-

When searching are large range of backups, the --start and --end flags (see -START AND END for more information) may be used to filter which backups are -searched.

-
-
-
-
$ cbbackupmgr examine -a ~/Projects/couchbase-archive -r repo --collection-string 'travel-sample.inventory.airline' --key 'airline_10' --start 2021-01-27T12_42_18.322132022Z --end 2021-01-27T12_42_35.395081659Z
--- Backup 2021-01-27T12_42_18.322132022Z --
-Description: Collection 'airline' not found
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-Scope: inventory (8)
-
--- Backup 2021-01-27T12_42_23.72226735Z --
-Description: Scope 'inventory' dropped
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-Bucket: travel-sample (13c11d3be4bfba95f9e4da73bf8d1960)
-
--- Backup 2021-01-27T12_42_34.08410595Z --
-Description: Bucket 'travel-sample' dropped
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-
--- Backup 2021-01-27T12_42_35.395081659Z --
-Description: Bucket 'travel-sample' not found
-Cluster UUID: 4fb26863aa5224ff97c9f17415c7bd92
-
-
-
-

By default, the examine will output binary documents as a hexdump (for the human -readable format) and as a hexstring (for the JSON format).

-
-
-
-
$ cbbackupmgr examine -a $PWD/archive -r repo --collection-string 'default' --key 'pymc0'
--- Backup 2021-01-27T14_00_38.445404421Z --
-Description: First occurrence of key 'pymc0' in this timeline, document created
-Cluster UUID: 59845c6cdce57b81bd6f450e5043bee7
-Bucket: default (7c8cb0f0bb9e2835d79e8dd91a5b0d67)
-Document:
-  Key: pymc0
-  Sequence Number: 1
-  Deleted: false
-  Size: 84B (Key: 5B) (Meta: 29B) (Value: 50B) (XATTRS: 0B)
-  Meta:
-    Flags: 0
-    Expiry: 0
-    Locktime: 0
-    CAS: 1611756033524170752
-    Revseqno: 1
-    Datatype: 0 (binary)
-  Value:
-    00000000  4d 59 4e 42 49 51 50 4d  5a 4a 50 4c 53 47 51 45  |MYNBIQPMZJPLSGQE|
-    00000010  4a 45 59 44 54 5a 49 52  57 5a 54 45 4a 44 58 43  |JEYDTZIRWZTEJDXC|
-    00000020  56 4b 50 52 44 4c 4e 4b  54 55 47 52 50 4f 51 49  |VKPRDLNKTUGRPOQI|
-    00000030  42 5a                                             |BZ|
-
-$ cbbackupmgr examine -a $PWD/archive -r repo --collection-string 'default' --key 'pymc0' --json | jq
-[
-  {
-    "backup": "2021-01-27T14_00_38.445404421Z",
-    "event_type": 1,
-    "event_description": "First occurrence of key 'pymc0' in this timeline, document created",
-    "cluster_uuid": "59845c6cdce57b81bd6f450e5043bee7",
-    "bucket_uuid": "7c8cb0f0bb9e2835d79e8dd91a5b0d67",
-    "scope_id": 0,
-    "collection_id": 0,
-    "document": {
-      "key": "pymc0",
-      "sequence_number": 1,
-      "value": "4d594e424951504d5a4a504c534751454a455944545a4952575a54454a445843564b5052444c4e4b54554752504f5149425a",
-      "metadata": {
-        "flags": 0,
-        "expiry": 0,
-        "locktime": 0,
-        "cas": 1611756033524170800,
-        "revseqno": 1,
-        "datatype": 0
-      },
-      "deleted": false
-    }
-  }
-]
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
CB_ENCRYPTION_PASSPHRASE
-
-

Specifies the passphrase used for encryption.

-
-
CB_KM_KEY_URL
-
-

Specifies the URL identifying the encryption key on the KMS. See ---km-key-url for the expected format and accepted KMSs.

-
-
CB_KM_ACCESS_ID
-
-

Specifies the key/user ID used to connect to the KMS.

-
-
CB_KM_SECRET_ACCESS_KEY
-
-

Specifies the secret key/token used to connect to the KMS.

-
-
CB_KM_AUTH_FILE
-
-

Specifies a path to a file containing the required credentials to connect to -the KMS.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-cloud(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-generate.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-generate.html deleted file mode 100644 index 43f0bf0b..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-generate.html +++ /dev/null @@ -1,968 +0,0 @@ - - - - - - - - - -cbbackupmgr-generate(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr generate [--num-documents <document_count>] [--archive <archive_dir>]
-                     [--cluster <cluster_hostname>] [--username <username>]
-                     [--password <password>] [--client-cert <path>]
-                     [--client-cert-password <password>] [--client-key <path>]
-                     [--client-key-password <password>] [--bucket <bucket_name>]
-                     [--cacert <certfile_path>] [--no-ssl-verify] [--repo <repository_name>]
-                     [--sink <sink_name>] [--prefix <prefix>] [--size <size>] [--json]
-                     [--json-fields <field_count>] [--json-field-size <field_size>]
-                     [--low-compression] [--xattrs] [--xattr-fields <field_count>]
-                     [--xattr-field-size <field_size>] [--compress]
-                     [--threads <thread_count>] [--purge] [--resume]
-                     [--storage <storage_type>] [--no-progress-bar]
-
-
-
-
-

DESCRIPTION

-
-
-

The generate command is primarily meant for testing the internal pipeline for -backup/restore. It allows you to very quickly generate documents and send them -to a desired 'sink'.

-
-
-
-
-

Required

-
-
-
-
-n,--num-documents <document_count>
-
-

The number of documents to generate.

-
-
-b,--bucket <bucket_name>
-
-

The bucket to generate documents for.

-
-
-
-
-
-
-

Optional

-
-
-
-
-a,--archive <archive_dir>
-
-

The archive in which to store generated documents (optionally used for logging when 'sink' is Couchbase).

-
-
-c,--cluster <cluster_hostname>
-
-

The hostname of the test cluster (only required when 'sink' is couchbase).

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. -Specifying this flag will allow a connection using SSL encryption, but will not verify the identity of the server you connect to. -You are vulnerable to a man-in-the-middle attack if you use this flag. -Either this flag or the --cacert flag must be specified when using an SSL encrypted connection.

-
-
--cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the server being connecting to. -Either this flag or the --no-ssl-verify flag must be specified when using an SSL encrypted connection.

-
-
-r,--repo <repo>
-
-

The name of a backup repository in an archive (only required when 'sink' is archive).

-
-
-s,--sink <sink>
-
-

The sink to send generated documents to (valid options are 'archive', 'blackhole' and 'couchbase')

-
-
-P,--prefix <prefix>
-
-

An optional prefix that will be prepended to generated keys.

-
-
-s,--size <size>
-
-

The size of generated binary documents in bytes.

-
-
-j,--json
-
-

Generated JSON formatted documents instead of binary documents.

-
-
--json-fields <num_fields>
-
-

The number of JSON fields to generate.

-
-
--json-value-size <value_size>
-
-

The size in bytes of the generated JSON values.

-
-
--low-compression
-
-

Generate document values/xattr value which are difficult to compress.

-
-
--xattrs
-
-

Generate extended attributes.

-
-
--xattr-fields <num_fields>
-
-

The number of fields to generate in the xattrs.

-
-
--xattr-value-size <value_size>
-
-

The size in bytes of the generated xattr values (hard coded limit of 2 bytes).

-
-
--compress
-
-

Force compression at the "gocbcore" level (default is uncompressed).

-
-
-t,--threads <thread_count>
-
-

The amount of worker threads to created for document generation.

-
-
--resume
-
-

If you previously canceled a backup/generate test there will be a backup -directory remaining; continue generating to the same backup.

-
-
--purge
-
-

If you previously canceled a backup/generate test there will be a backup -directory remaining; remove it before continuing.

-
-
--shard-count <shard_count>
-
-

The number of shards to create when generating to an archive. This option is -silently ignored when generating to the 'couchbase' or 'blackhole' sinks.

-
-
--no-progress-bar
-
-

By default, a progress bar is printed to stdout so that the user can see how -long the generation is expected to take, the amount of data that is being -transferred per second, and the amount of data that has been generated. -Specifying this flag disables the progress bar and is useful when running -automated jobs.

-
-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

RBAC

-
-
-

When performing a backup/restore with a user which is using RBAC, there are a -couple of things that should be taken into consideration each of which is -highlighted in this section.

-
-
-

Bucket Level

-
-

Bucket level data may be backed up/restored using the data_backup (Data -Backup & Restore) role.

-
-
-

The data_backup role does not have access to cluster level data such as:

-
-
-
    -
  • -

    Analytics Synonyms

    -
    • -
  • -

    Eventing Metadata

    -
    • -
  • -

    FTS Aliases

    -
    • -
-
-
-

Backing up/restoring cluster level data with the data_backup role will cause -permission errors like the one below.

-
-
-
-
Error backing up cluster: {"message":"Forbidden. User needs one of the following permissions","permissions":["cluster.fts!read"]}
-
-
-
-

When presented with an error message such as the one above, there’s two clear -options.

-
-
-

The first option is to provide the user with the required credentials using -either the cli, REST API or Couchbase Server WebUI. This can be done by editing -the user and adding the required role. See Cluster Level for more information -about the required roles.

-
-
-

Secondly, backing up/restoring the specific service can be disabled. For -backups this must be done when configuring the repository with the config -command using the --disable style flags. For restore, these flags may be used -directly to disable one or more services. See the backup/restore documentation -for more information.

-
-
-
-

Cluster Level

-
-

Backing up/restoring cluster level data requires additional RBAC roles, each of -which is highlighted below:

-
-
-
-
Analytics Synonyms
-
-

analytics_admin (Analytics Admin)

-
-
Eventing Metadata
-
-

eventing_admin (Eventing Full Admin)

-
-
FTS Aliases
-
-

fts_admin (Search Admin)

-
-
-
-
-

These additional roles are required since this is cluster level data which may -encompass multiple buckets.

-
-
-
-
-
-

EXAMPLES

-
-
-

cbbackupmgr generate is used to generate documents and send them through the -cbbackupmgr pipeline; this command is primarily intended for testing.

-
-
-
-
$ cbbackupmgr generate -a /data/backups -c localhost -u admin -p password -n 1000 -s 1024
-1000/1000 - 100% - 999D/s - 1.03MB/s
-Transfer completed in 1.000174463s
-
-
-
-

By supplying the optional 'sink' argument you can choose which sink to send the -generate documents to; to test the speed to the document generate itself, specify -the sink as 'blackhole'.

-
-
-
-
$ cbbackupmgr generate -a /data/backups -n 100000 -s 1048576 --sink 'blackhole'
-100000/100000 - 100% - 9084D/s - 8.87GB/s
-Transfer completed in 11.007203014s
-
-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-help.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-help.html deleted file mode 100644 index 08282cfc..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-help.html +++ /dev/null @@ -1,520 +0,0 @@ - - - - - - - - - -cbbackupmgr-help(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr help <page>
-
-
-
-
-

DESCRIPTION

-
-
-

Show documentation about cbbackupmgr. You can look up documentation by command -name or page name.

-
-
-
-
-

EXAMPLES

-
-
-

The help command can be used to show various kinds of documentation. Below are -some examples of how to view documentation for subcommands, tutorials, and other -pages. To show the help page for the cbbackupmgr-config(1) command you -would run the following command:

-
-
-
-
$ cbbackupmgr help config
-
-
-
-

To show the cbbackupmgr-tutorial page you would run the following command:

-
-
-
-
$ cbbackupmgr help tutorial
-
-
-
-

To show the cbbackupmgr-strategies page you would run the following command:

-
-
-
-
$ cbbackupmgr help strategies
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-config(1), -cbbackupmgr-tutorial(7), -cbbackupmgr-strategies(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-info.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-info.html deleted file mode 100644 index f2e51620..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-info.html +++ /dev/null @@ -1,1136 +0,0 @@ - - - - - - - - - -cbbackupmgr-info(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr info [--archive <archive_dir>] [--repo <repo_name>]
-                 [--backup <bucket_list>]
-                 [--collection-string <collection_string>]
-                 [--json] [--all] [--depth <depth>]
-                 [--obj-access-key-id <access_key_id>]
-                 [--obj-cacert <cert_path>] [--obj-endpoint <endpoint>]
-                 [--obj-no-ssl-verify] [--obj-region <region>]
-                 [--obj-staging-dir <staging_dir>]
-                 [--obj-secret-access-key <secret_access_key>]
-                 [--s3-force-path-style] [--s3-log-level <level>]
-
-
-
-
-

DESCRIPTION

-
-
-

This command provides information about the given path, be it the whole -archive, repository, backup or bucket. The information can be displayed in a -tabular format or JSON. By default only two levels of the data hierarchy will -be displayed in tabular format, to see it all use the --all flag. For the -exact format of the output please see the examples below. Unlike the list -command this command will not list individual files.

-
-
-
-
-

OPTIONS

-
-
-

Below are the required and optional parameters for the info command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The location of the backup archive to display information about. When using -info against an archive stored in S3 you must prefix the archive path with -s3://${BUCKET_NAME}/.

-
-
-
-
-
-

Optional

-
-
-
-r,--repo <repo_name>
-
-

If specified, the info command will only display information for this backup -repository.

-
-
--backup <backup>
-
-

If specified, the info command will only display information for this backup. -The --repo flag must also be specified if this flag is used.

-
-
--collection-string <collection_string>
-
-

A dot separated collection string representing the bucket/scope/collection to -get info for. The --backup flag must be provided if this flag is used.

-
-
--start <start>
-
-

Only backups which fall between the range specified by --start and --end -will be included in the output. --start and --end must be supplied -together and are only valid when getting info for a repository. See START AND -END for information on what values are accepted.

-
-
--end <end>
-
-

Only backups which fall between the range specified by --start and --end -will be included in the output. --start and --end must be supplied -together and are only valid when getting info for a repository. See START AND -END for information on what values are accepted.

-
-
--json
-
-

If specified the output will be a json object.

-
-
--all
-
-

Show all levels of the data hierarchy. Note that for json output all levels -are always returned. This flag will override any value supplied to --depth.

-
-
--depth <depth>
-
-

Show depth layers of additional information when printing non json output -to stdout. By default two layers of information will be displayed. Accepts -any non-zero positive integer.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-
-
-

START AND END

-
-
-

This sub-command accepts a --start and --end flag. These flags accept -multiple values to allow you to flexibly operate on a range of backups.

-
-
-

Indexes

-
-

Indexes may be supplied to operate on a range of backups, for example ---start 1 --end 2 will include start at the first backup and will finish with -the second backup. Note that the first backup is 1 and not 0 and that the ---end flag is inclusive.

-
-
-
-

Short Dates

-
-

Short dates may be supplied in the format day-month-year. For example ---start 01-08-2020 --end 31-08-2020 will operate on all the backups which -were taken during August of 2020. Note that the end date is inclusive.

-
-
-

When supplying short dates, you may supply start or oldest as a placeholder -for the date on which the first backup in this repository was taken. The -keywords end or latest may be used as a placeholder for the date last -backup in the repository was taken.

-
-
-
-

Backup Names

-
-

Backup names may be supplied as they exist on disk. For example ---start 2020-08-13T20_01_08.894226137+01_00 --end 2020-08-13T20_01_12.348300092+01_00 -will cause the sub-command to operate on all the backups which inclusively fall -between these two backups.

-
-
-

When supplying backup names, you may supply start or oldest as a -placeholder for the first backup in the repository. The keywords end or -latest may be used as a placeholder for the final backup in the repository.

-
-
-
-
-
-

EXAMPLES

-
-
-

The info command will display information of the backup archive in a user -friendly manner. Let’s imagine we have a backup archive /backup_archive and we want -to see information about it we would run the command:

-
-
-
-
$ cbbackupmgr info -a /backup_archive
-
-Name             | Size      | # Repos  |
-backup_archive   | 111.00MB  | 2        |
-
-*    Name        | Size     | # Backups  |
-*    Manchester  | 55.56MB  | 1          |
-*    London      | 55.44MB  | 2          |
-
-
-
-

This shows us that the archive backup_archive has two repositories and has a -total size of 110MB. It will also provide one level more into the hierarchy and -show information of the two repositories Manchester and London and we can -see their respective sizes as well as the number of backups each contains.

-
-
-

If we wanted to get more in depth information of the repository we would -provide the repository name as follows:

-
-
-
-
$cbbackupmgr info -a ~/backup_archive -r London
-
-Name    | Size     | # Backups  |
-London  | 55.44MB  | 2          |
-
-+  Backup                       | Size     | Type  | Source                 | Range  | Events  | Aliases  | Complete  |
-+  2019-03-15T13_19_54.826458Z  | 3.02MB   | INCR  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-+  2019-03-15T12_18_00.514284Z  | 52.42MB  | FULL  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-
-
-
-

This gives more in depth information about each backup such as the timestamp -from when it was made, the size, the type which can be either a full backup -(FULL), and incremental backup (INCR) or a merge (MERGE). We can also see -the address of the cluster we backed up the data from. The range column is -used by merges. If the backup is a merge the range has the start timestamp -the end timestamp and the number of backups that where merge in the format -start - end : count. Event and aliases are the number of events and full -text search indexes aliases that where restored. Finally the complete field -indicates if the backup finished successfully or not.

-
-
-

It is possible to get information for an individual backup as well by using -the following command:

-
-
-
-
$ cbbackupmgr info -a ~/backup_archive -r London --backup 2019-03-15T12_18_00.514284Z
-
-Backup                       | Size     | Type  | Source                 | Range  | Events  | Aliases  | Complete  |
-2019-03-15T12_18_00.514284Z  | 52.42MB  | FULL  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-
- -   Bucket          | Size     | Items  | Mutations | Tombstones | Views  | FTS  | Indexes  | CBAS  |
- -   beer-sample     | 6.85MB   | 7303   | 7303      | 0          | 1      | 0    | 1        | 0     |
- -   gamesim-sample  | 2.86MB   | 586    | 586       | 0          | 1      | 0    | 1        | 0     |
- -   travel-sample   | 42.72MB  | 31591  | 31591     | 0          | 0      | 0    | 10       | 0     |
-
-
-
-

The command above provides information about all buckets in the specific -backup. This includes the size of the data in each bucket, the number of -items which includes mutations and tombstones, the number of mutations -and the number of deletions. For backups made with cbbackupmgr older -than 6.5 mutations and deletions will always be 0 as this is only -supported by archives made with cbbackupmgr 6.5+. For the buckets it will -also shows the number of view, full text indexes, the number of GSIs and the -number of analytics metadata records.

-
-
-

To see information for one bucket only we would use the --collection-string -flag and give it the name of the bucket and this would return just the -information for that bucket.

-
-
-

To see from one level to the last the --all flag must be provided this will -show the information from the provided level to the bucket level. An example -can be seen below:

-
-
-
-
$ cbbackupmgr info -a ~/backup_archive --all
-
-Name         | UUID                                 | Size     | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3 | 55.56MB  | 1        |
-
-*  Name        | Size     | # Backups  |
-*  Manchester  | 55.56MB  | 1          |
-
-+    Backup                      | Size     | Type   | Source  | Cluster UUID                     | Range                                      | Events  | Aliases  | Complete  |
-+    2019-03-15T13_52_27.18301Z  | 55.56MB  | MERGE  | MERGE   | c044f5eeb1dc16d0cd49dac29074b5f9 | 2019-03-15T13_51_44-2019-03-15T13_51_58:3  | 0       | 1        | true      |
-
--      Bucket          | Size     | Items  | Mutations | Tombstones | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample     | 6.85MB   | 7303   | 7303      | 0          | 1      | 0    | 1        | 0     |
--      gamesim-sample  | 2.86MB   | 586    | 586       | 0          | 1      | 0    | 1        | 0     |
--      travel-sample   | 42.72MB  | 31591  | 31591     | 0          | 0      | 0    | 10       | 0     |
-
-
-
-

To display only a certain number of layers when displaying info in tabular -format you may provide the --depth flag with a non-zero positive integer. -This gives more fine grained control over what information is displayed that ---all. Using the --all flag will implicitly override any value provided to ---depth.

-
-
-
-
$ cbbackupmgr info -a ~/backup_archive --depth 1
-
-Name         | UUID                                 | Size     | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3 | 55.56MB  | 1        |
-
-
-
-

The info command also allows for JSON output which can be useful for -automation or users that would rather get the information in json format. -The output json will have the format seen below. Note that all sizes will be -in bytes and that for non-merge backups the range field will be empty.

-
-
-
-
{
-  "name": NAME,
-  "archive_uuid": UUID,
-  "size": SIZE,
-  "repos": [
-    {
-      "name": NAME,
-      "size": SIZE,
-      "count": BACKUP_COUNT,
-      "backups": [
-        {
-          "date": DATE,
-          "complete: BOOL
-          "size": SIZE,
-          "type": ["incr"| "full" | "merge"],
-          "source": SOURCE,
-          "range":[all backups date involved in the merge],
-          "event": COUNT,
-          "fts_alias": COUNT,
-          "source_cluster_uuid": UUID,
-          "buckets": [
-            {
-              "name": BUCKET_NAME,
-              "size": SIZE,
-              "items": COUNT,
-              "mutations": COUNT,
-              "tombstones": COUNT,
-              "views_count": COUNT,
-              "fts_count": COUNT,
-              "analytics": COUNT
-            }
-          ]
-        }
-      ]
-    }
-  ]
-
-
-
-

The info command also supports getting information about an archive which -is stored directly in AWS S3. This will download a very minimal amount of -data e.g. it won’t download the whole archive.

-
-
-
-
$ cbbackupmgr info -a s3://bucket/backup_archive --all --obj-staging-dir ~/backup-archive
-
-Name         | UUID                                 | Size     | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3 | 55.56MB  | 1        |
-
-*  Name        | Size     | # Backups  |
-*  Manchester  | 55.56MB  | 1          |
-
-+    Backup                      | Size     | Type   | Source  | Cluster UUID                     | Range                                      | Events  | Aliases  | Complete  |
-+    2019-03-15T13_52_27.18301Z  | 55.56MB  | MERGE  | MERGE   | c044f5eeb1dc16d0cd49dac29074b5f9 | 2019-03-15T13_51_44-2019-03-15T13_51_58:3  | 0       | 1        | true      |
-
--      Bucket          | Size     | Items  | Mutations | Tombstones | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample     | 6.85MB   | 7303   | 7303      | 0          | 1      | 0    | 1        | 0     |
--      gamesim-sample  | 2.86MB   | 586    | 586       | 0          | 1      | 0    | 1        | 0     |
--      travel-sample   | 42.72MB  | 31591  | 31591     | 0          | 0      | 0    | 10       | 0     |
-
-
-
-

The info command allows you to limit the range of backups that are included in -the output. This can be used to answer a few important questions before -performing a merge/restore. "Is backup X included in a merge/restore with this -range?" or "When I merge/restore with this range, will it include a full -backup?".

-
-
-

Lets tackle the question "When I merge with this range, will it include a full -backup?"

-
-
-
-
$ cbbackupmgr info -a /archive -r repo
-Name | Size    | # Backups |
-repo | 88.11MB | 5         |
-
-+  Backup                              | Size    | Type | Source                 | Cluster UUID                     | Range | Events | Aliases | Complete |
-+  2020-07-18T18_19_54.057655243+01_00 | 17.62MB | FULL | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_21.319063119+01_00 | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_26.250684761+01_00 | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_30.765048405+01_00 | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_41.39884127+01_00  | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-
-
-
-

The above repository contains five backups in total where only the first is a -full backup. We would like to merge all the backups in August and want to -ensure that we will be merging a full backup.

-
-
-
-
$ cbbackupmgr info -a /archive -r repo --start 01-08-2020 --end 31-08-2020
-Name | Size    | # Backups |
-repo | 70.49MB | 4         |
-
-+  Backup                              | Size    | Type | Source                 | Cluster UUID                     | Range | Events | Aliases | Complete |
-+  2020-08-18T18_20_21.319063119+01_00 | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_26.250684761+01_00 | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_30.765048405+01_00 | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-+  2020-08-18T18_20_41.39884127+01_00  | 17.62MB | INCR | http://172.20.1.1:8091 | 8527d470b07095084cf45e1a772a7ba9 | N/A   | 0      | 0       | true     |
-
-
-
-

Given the output above, it’s clear that merging all the backups taken in August -will not result in a merged full backup. To result in a merged full backup, we -would also need to include backups taken in July.

-
-
-
-
$ cbbackupmgr merge -a /archive -r repo --start 01-07-2020 --end 31-08-2020
-(1/5) Merging backup '2020-07-18T18_19_54.057655243+01_00'
-Copied all data in 1.342126932s (Avg. 2.98MB/Sec)                                                   50000 items / 2.98MB
-[==============================================================================================================] 100.00%
-Merge bucket 'default' succeeded
-Mutations merged: 50000, Mutations failed to merged: 0
-Deletions merged: 0, Deletions failed to merged: 0
-
-(2/5) Merging backup '2020-08-18T18_20_21.319063119+01_00'
-Copied all data in 2.601879822s (Avg. 1.49MB/Sec)                                                   50000 items / 2.98MB
-[==============================================================================================================] 100.00%
-Merge bucket 'default' succeeded
-Mutations merged: 50000, Mutations failed to merged: 0
-Deletions merged: 0, Deletions failed to merged: 0
-
-(3/5) Merging backup '2020-08-18T18_20_26.250684761+01_00'
-Copied all data in 3.85210489s (Avg. 1016.84KB/Sec)                                                 50000 items / 2.98MB
-[==============================================================================================================] 100.00%
-Merge bucket 'default' succeeded
-Mutations merged: 50000, Mutations failed to merged: 0
-Deletions merged: 0, Deletions failed to merged: 0
-
-(4/5) Merging backup '2020-08-18T18_20_30.765048405+01_00'
-Copied all data in 5.057777071s (Avg. 610.10KB/Sec)                                                 50000 items / 2.98MB
-[==============================================================================================================] 100.00%
-Merge bucket 'default' succeeded
-Mutations merged: 50000, Mutations failed to merged: 0
-Deletions merged: 0, Deletions failed to merged: 0
-
-(5/5) Merging backup '2020-08-18T18_20_41.39884127+01_00'
-Copied all data in 6.24708519s (Avg. 508.42KB/Sec)                                                  50000 items / 2.98MB
-[==============================================================================================================] 100.00%
-Merge bucket 'default' succeeded
-Mutations merged: 50000, Mutations failed to merged: 0
-Deletions merged: 0, Deletions failed to merged: 0
-Merge completed successfully
-
-$ cbbackupmgr info -a /archive -r repo
-Name | Size    | # Backups |
-repo | 24.11MB | 1         |
-
-+  Backup                             | Size    | Type         | Source | Cluster UUID                     | Range                                         | Events | Aliases | Complete |
-+  2020-08-18T18_20_41.39884127+01_00 | 24.11MB | MERGE - FULL | Merge  | 8527d470b07095084cf45e1a772a7ba9 | 2020-07-18T18_19_54 - 2020-08-18T18_20_21 - 5 | 0      | 0       | true     |
-
-
-
-

As we can clearly see, after we modified our backup range to include July, we -correctly merged all five backups including the full backup. Which is indicated -by the MERGE - FULL type for the resulting backup.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-cloud(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-merge.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-merge.html deleted file mode 100644 index 57fe5f28..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-merge.html +++ /dev/null @@ -1,810 +0,0 @@ - - - - - - - - - -cbbackupmgr-merge(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr merge [--archive <archive_dir>] [--repo <repo_name>]
-                  [--start <start>] [--end <end>]
-                  [--date-range <range>] [--threads <num>]
-                  [--no-progress-bar] [--passphrase <passphrase>]
-                  [--km-key-url <url>] [--km-endpoint <endpoint>]
-                  [--km-region <region>] [--km-access-key-id <id>]
-                  [--km-secret-access-key <key>] [--km-auth-file <path>]
-
-
-
-
-

DESCRIPTION

-
-
-

The merge command is an Enterprise Edition command which can be used to merge -two or more backups together. Since cbbackupmgr is a utility that always -attempts to take incremental backups it is necessary to reclaim disk space from -time to time. Merging data will deduplicate similar keys in backup files -being merged together in order to create a single smaller backup file. Doing -merges should replace the full backup step by taking multiple incremental -backups of a Couchbase cluster and converting them into a single full backup. -Since this process takes place in the backup archive there is no cluster -overhead to merging data together. See cbbackupmgr-strategies(7) for -suggestions on using the merge command in your backup process.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of required parameters for the merge command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The archive directory to merge data in.

-
-
-r,--repo <repo_name>
-
-

The name of the backup repository to merge data in.

-
-
-
-
-
-

Optional

-
-
-
--start <start>
-
-

The first backup to merge. See START AND END for information on what values -are accepted.

-
-
--end <end>
-
-

The final backup to merge. See START AND END for information on what values -are accepted.

-
-
--date-range <range>
-
-

This flag will accept a comma separated range in the same formats as ---start and --end. See START AND END section for more information on the -accepted formats.

-
-
-t,--threads <num>
-
-

Specifies the number of concurrent vBuckets that will be merged at a time. -Increasing the threads will make the merge faster but will also increase the -resource used by the client. This parameter defaults to 1 but it is -recommended to increase it to match the number of CPUs in the client machine.

-
-
--no-progress-bar
-
-

By default, a progress bar is printed to stdout so that the user can see how -long the merge is expected to take, the amount of data that is being transferred -per second, and the amount of data that has been merged. Specifying this flag -disables the progress bar and is useful when running automated jobs.

-
-
-
-
-
-

Encryption (Developer Preview)

-
-
-
--passphrase <passphrase>
-
-

Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing.

-
-
--km-key-url <url>
-
-

Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable CB_KM_KEY_URL. For more on how to authenticate using the different -providers see cbbackupmgr-encryption(7).

-
-

For AWS the expected key format is awskms://<KEY-ID|KEY-ALIAS>, for example -awskms://alias\keyAlias.

-
-
-

For GCP the expected key format is gcpkms://<KEY-RESOURCE-ID>, for example -gcpkms://projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key.

-
-
-

For Azure key vault the expected key format is azurekeyvault://<KEY-IDENTIFIER> -for example: -azurekeyvault://vault-name.vault.azure.net/object-type/object-name/object-version.

-
-
-

For HashiCorp Vault the expected format is hashivaults://<HOST>/<KEY-NAME> -for example: -hashivaults://127.0.0.1:8200/keyName.

-
-
-
--km-region <region>
-
-

Required when using AWS KMS, it allows you to set the key region.

-
-
--km-endpoint <endpoint>
-
-

The host or address to use as your KMS. It will override the default SDK one.

-
-
--km-access-key-id <id>
-
-

The user ID used to connect to the key management service. It can also be -provided via CB_KM_ACCESS_KEY_ID environmental variable. Please refer -to cbbackupmgr-encryption(7) for the required authentication for each -provider.

-
-
--km-secret-access-key <key>
-
-

The key used to connect to the key management service. It can also be -provided via the CB_KM_SECRET_ACCESS_KEY environmental variable. Please -refer to cbbackupmgr-encryption(7) for the required authentication for -each provider.

-
-
--km-auth-file <path>
-
-

The path to a file containing the authentication credentials for the key -management service. It can also be provided via the CB_KM_AUTH_FILE -environmental variable. Please refer to cbbackupmgr-encryption(7) for the -required authentication for each provider.

-
-
-
-
-
-
-
-

START AND END

-
-
-

This sub-command accepts a --start and --end flag. These flags accept -multiple values to allow you to flexibly operate on a range of backups.

-
-
-

Indexes

-
-

Indexes may be supplied to operate on a range of backups, for example ---start 1 --end 2 will include start at the first backup and will finish with -the second backup. Note that the first backup is 1 and not 0 and that the ---end flag is inclusive.

-
-
-
-

Short Dates

-
-

Short dates may be supplied in the format day-month-year. For example ---start 01-08-2020 --end 31-08-2020 will operate on all the backups which -were taken during August of 2020. Note that the end date is inclusive.

-
-
-

When supplying short dates, you may supply start or oldest as a placeholder -for the date on which the first backup in this repository was taken. The -keywords end or latest may be used as a placeholder for the date last -backup in the repository was taken.

-
-
-
-

Backup Names

-
-

Backup names may be supplied as they exist on disk. For example ---start 2020-08-13T20_01_08.894226137+01_00 --end 2020-08-13T20_01_12.348300092+01_00 -will cause the sub-command to operate on all the backups which inclusively fall -between these two backups.

-
-
-

When supplying backup names, you may supply start or oldest as a -placeholder for the first backup in the repository. The keywords end or -latest may be used as a placeholder for the final backup in the repository.

-
-
-
-
-
-

EXAMPLES

-
-
-

In order to merge data, you need to have a backup repository with at least two -backups. Below is an example of merging a backup repository named "example" that -has two backups in it. The first backup contains the initial dataset. The second -backup was taken after four items were updated.

-
-
-
-
$ cbbackupmgr info -a /data/backups -r example --all
-Name     | Size    | # Backups  |
-example  | 2.44MB  | 2          |
-
-+  Backup                           | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-02T07_58_32.42306+01_00  | 2.38MB  | FULL  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 2.38MB  | 8192   | 8192       | 0           | 0      | 0    | 0        | 0     |
-
-+  Backup                           | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-03T08_00_22.62763+01_00  | 65.47KB  | INCR  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 65.25KB  | 4      | 4          | 0           | 0      | 0    | 0        | 0     |
-
-$ cbbackupmgr merge -a /data/backup -r example --start 2020-06-02T07_58_32.42306+01_00 \
- --end 2020-06-03T08_00_22.62763+01_00
-
-$ cbbackupmgr info -a /data/backups -r example --all
-Name     | Size    | # Backups  |
-example  | 2.38MB  | 1          |
-
-+  Backup                           | Size    | Type          | Source  | Cluster UUID                      | Range                                      | Events  | Aliases  | Complete  |
-+  2020-06-03T08_00_22.62763+01_00  | 2.38MB  | MERGE - FULL  | Merge   | c044f5eeb1dc16d0cd49dac29074b5f9  | 2020-06-02T07_58_32-2020-06-02T08_00_22:2  | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 2.38MB  | 8192   | 8192       | 0           | 0      | 0    | 0        | 0     |
-
-
-
-

Upon completion of the merge the number of items in the backup files is the -same. This is because the keys in the second backup were also contained in the -first backup, but the keys in the second backup contained newer values and -overwrote the keys in the first backup during the merge. The timestamp of the -backup folder is also the same as the timestamp of the latest backup because the -new backup is a snapshot of the cluster at that point in time.

-
-
-
-
-

DISCUSSION

-
-
-

It is important that internally the merge command is able to merge backups -together without corrupting the backup archive or leaving it in an intermediate -state. In order to ensure this behavior cbbackupmgr always creates a new backup -and completely merges all data before removing any backup files. When a merge is -started a .merge_status file is created in the backup repository to track the -merge progress. cbbackupmgr then copies the first backup to the .merge folder -and begins merging the other backups into .merge folder. After each backup is -merged the .merge_status file is updated to track the merge progress. if all -backups are merged together successfully, cbbackupmgr will start deleting the -old backups and then copy the fully merged backup into a folder containing the -same name as the backup specified by the --end flag. If the cbbackupmgr utility -fails during this process, then the merge will either be completed or the -partial merge files will be removed from the backup repository during the next -invocation of the cbbackupmgr.

-
-
-

Since the merge command creates a new backup file before it removes the old ones -it is necessary to have at least as much free space as the backups that are to -be merge together.

-
-
-

For more information on suggestions for how to use the merge command in your -backup process see cbbackupmgr-strategies(7)

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_ENCRYPTION_PASSPHRASE
-
-

Specifies the passphrase used for encryption.

-
-
CB_KM_KEY_URL
-
-

Specifies the URL identifying the encryption key on the KMS. See ---km-key-url for the expected format and accepted KMSs.

-
-
CB_KM_ACCESS_ID
-
-

Specifies the key/user ID used to connect to the KMS.

-
-
CB_KM_SECRET_ACCESS_KEY
-
-

Specifies the secret key/token used to connect to the KMS.

-
-
CB_KM_AUTH_FILE
-
-

Specifies a path to a file containing the required credentials to connect to -the KMS.

-
-
-
-
-
-
-

FILES

-
-
-
-
.merge_status
-
-

File storing information on the progress of the merge.

-
-
.merge
-
-

Directory storing intermediate merge data.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-info(1), -cbbackupmgr-strategies(1), -cbbackupmgr-encryption(7), -cbbackupmgr-network-filesystems(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-network-filesystems.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-network-filesystems.html deleted file mode 100644 index 8adb2754..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-network-filesystems.html +++ /dev/null @@ -1,562 +0,0 @@ - - - - - - - - -cbbackupmgr-network-filesystems(7) - - - - - -
-
-

DESCRIPTION

-
-
-

A document which covers some important topics related to using cbbackupmgr with -common network filesystems such as NFS and CIFS/SMB.

-
-
-
-
-

PERFORMANCE

-
-
-

The cbbackupmgr tool uses SQLite when storing data on disk, SQLite has a -read-modify-write update pattern; this means that for each key we insert into -our SQLite index we have to perform a read to the underlying storage. For local -filesystems, this isn’t significantly expensive because the operating systems -filesystem cache is extremely likely to cache these repeated reads. This isn’t -always the case for network filesystems.

-
-
-

On the most commonly used network filesystems, these reads will not be cached -meaning the underlying client must perform one or more network operations -resulting in significantly higher latency than would be experienced when using -a local filesystem.

-
-
-

Some newer versions of network filesystems such as NFSv4 have better support -for locking, for example -delegation. However it’s worth -noting that delegation isn’t a mandatory feature and it depends on whether the -client/server supports it and actively accepts/grants delegations.

-
-
-

Newer versions of CIFS/SMB support opportunistic locking which provides similar -functionality to NFS delegation i.e. allow cached read/writes. Note that again -this is only supported in later versions of CIFS/SMB and depends on -client/server support.

-
-
-
-
-

NFS

-
-
-

As stated above, cbbackupmgr uses SQLite when storing data on disk, internally -SQLite uses file locking to guard access to the database and it is one of the -features required to allow SQLite to expose its atomic commit behavior. This -means there’s a few things to consider:

-
-
-
    -
  • -

    Some NFS server implementations only provide facade level locking, in some -cases (for instance where other processes are accessing the archive) this can -result in database corruption.

    -
    • -
  • -

    Some NFS server implementations have a limit on the number of locks that can -be held at once. This means it’s likely that backups will only work when -paired with a version greater than 6.6.0 (both the backup client/cluster).

    -
    • -
-
-
-
-
-

CIFS/SMB

-
-
-

Given that SQLite uses file locking to expose its atomic commit behavior, and -that most CIFS/SMB implementations don’t support requesting advisory bytes range -locks, you must mount your CIFS/SMB share using the nobrl option. For more -information see mount.cifs(8).

-
-
-

During normal operation cbbackupmgr will create and overwrite unix style -hidden files ("dotfiles"). Depending on the version of CIFS/SMB the config -option hide dot files = no may need to be set otherwise these operations -will fail with permission denied errors. Please note that by default, this -option will be set to yes. When hitting permission denied errors for -files beginning with a . you should retry setting hide dot files = no -before contacting support.

-
-
-

As with NFS, due care and attention should be taken to ensure that cbbackupmgr -is the only process accessing the archive whilst it’s in use; this is because -some systems don’t honor opportunistic locks and may therefore modify files -which are actively locked, resulting in corruption.

-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-backup(1), -cbbackupmgr-restore(1), -cbbackupmgr-merge(1),

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-remove.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-remove.html deleted file mode 100644 index 7ed0e0bd..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-remove.html +++ /dev/null @@ -1,935 +0,0 @@ - - - - - - - - - -cbbackupmgr-remove(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr remove [--archive <archive_dir>] [--repo <repo_name>]
-                   [--backups <backup_range>] [--obj-access-key-id <access_key_id>]
-                   [--obj-cacert <cert_path>] [--obj-endpoint <endpoint>]
-                   [--obj-no-ssl-verify] [--obj-region <region>]
-                   [--obj-staging-dir <staging_dir>]
-                   [--obj-secret-access-key <secret_access_key>]
-                   [--s3-force-path-style] [--s3-log-level <level>]
-
-
-
-
-

DESCRIPTION

-
-
-

Removes a backup repository from the backup archive. All contents of the -specified repository will be deleted from disk.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of parameters for the remove command.

-
-
-
-
-a,--archive <archive_dir>
-
-

The location of the archive directory. If the archive is stored in S3 prefix -the archive path with s3://${BUCKET_NAME}/.

-
-
-r,--repo <repo_name>
-
-

The name of the backup repository to remove.

-
-
-
-
-

Optional

-
-
-
--start <start>
-
-

The first backup to remove. See START AND END for information on what values -are accepted.

-
-
--end <end>
-
-

The final backup to remove. See START AND END for information on what values -are accepted.

-
-
--backups <backup_range>
-
-

This flag will accept either a single backup directory -(e.g. --backups 2019-08-23T09_36_56.957232625Z) a comma separated range in -the same formats as --start and --end. See START AND END section for more -information on the accepted formats.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-
-
-

START AND END

-
-
-

This sub-command accepts a --start and --end flag. These flags accept -multiple values to allow you to flexibly operate on a range of backups.

-
-
-

Indexes

-
-

Indexes may be supplied to operate on a range of backups, for example ---start 1 --end 2 will include start at the first backup and will finish with -the second backup. Note that the first backup is 1 and not 0 and that the ---end flag is inclusive.

-
-
-
-

Short Dates

-
-

Short dates may be supplied in the format day-month-year. For example ---start 01-08-2020 --end 31-08-2020 will operate on all the backups which -were taken during August of 2020. Note that the end date is inclusive.

-
-
-

When supplying short dates, you may supply start or oldest as a placeholder -for the date on which the first backup in this repository was taken. The -keywords end or latest may be used as a placeholder for the date last -backup in the repository was taken.

-
-
-
-

Backup Names

-
-

Backup names may be supplied as they exist on disk. For example ---start 2020-08-13T20_01_08.894226137+01_00 --end 2020-08-13T20_01_12.348300092+01_00 -will cause the sub-command to operate on all the backups which inclusively fall -between these two backups.

-
-
-

When supplying backup names, you may supply start or oldest as a -placeholder for the first backup in the repository. The keywords end or -latest may be used as a placeholder for the final backup in the repository.

-
-
-
-
-
-

EXAMPLES

-
-
-

The remove command is used to remove a backup repository from disk. Below is an -example of how to run the remove command. The /data/backup directory is used as -the archive directory and the backup repository is named "example".

-
-
-
-
$ cbbackupmgr info --archive /data/backups --repo example --all
-Name     | Size    | # Backups  |
-example  | 4.38MB  | 3          |
-
-+  Backup                            | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-02T07_49_11.281004+01_00  | 1.69MB  | FULL  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 1.69MB  | 4096   | 4096       | 0           | 0      | 0    | 0        | 0     |
-
-+  Backup                            | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-03T07_49_52.577901+01_00  | 1.34MB  | INCR  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 1.34MB  | 2048   | 2048       | 0           | 0      | 0    | 0        | 0     |
-
-+  Backup                            | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-04T07_50_06.908787+01_00  | 1.34MB  | INCR  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 1.34MB  | 2048   | 2048       | 0           | 0      | 0    | 0        | 0     |
-
-
-$ cbbackupmgr remove -a /data/backup -r example
-Backup repository `example` deleted successfully from archive `/tmp/backup`
-
-$ cbbackupmgr info -a ~/data/backups --all
-Name     | UUID                                  | Size  | # Repos  |
-backups  | 0db3337c-96b0-4b3a-a7fb-bbfd53790e5f  | 0B    | 0        |
-
-
-
-

The remove command can also be used to remove backups from inside a repository -given the following archive:

-
-
-
-
$ cbbackupmgr info -a /data/backup -r example
-Name     | Size     | # Backups  |
-example  | 36.01MB  | 3          |
-
-  +  Backup                            | Size     | Type  | Source                 | Range  | Events  | Aliases  | Complete  |
-  +  2019-09-18T11_13_58.136188+01_00  | 12.00MB  | FULL  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-  +  2019-09-19T11_14_29.026324+01_00  | 12.00MB  | INCR  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-  +  2019-09-19T11_14_40.410627+01_00  | 12.00MB  | INCR  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-  +  2019-09-20T11_14_43.410627+01_00  | 12.00MB  | INCR  | http://localhost:8091  | N/A    | 0       | 1        | true      |
-
-
-
-

We can delete the first two backups using indices range such as 1,2:

-
-
-
-
$ cbbackupmgr remove -a ~/backup_repo -r example --backups 1,2
-Backup `2019-09-18T11_13_58.136188+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-Backup `2019-09-19T11_14_29.026324+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-All backups in range `1,2` were deleted
-
-
-
-

To delete only backup 2019-09-19T11_14_29.026324+01_0 we can do as follows:

-
-
-
-
$ cbbackupmgr remove -a ~/backup_repo -r example --backups 2019-09-19T11_14_29.026324+01_0
-Backup `2019-09-19T11_14_29.026324+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-
-
-
-

To delete backups between the 18-09-2019 and 19-09-2019 we can do as follows:

-
-
-
-
$ cbbackupmgr remove -a ~/backup_repo -r example --backups 18-09-2019,19-09-2019
-Backup `2019-09-18T11_13_58.136188+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-Backup `2019-09-19T11_14_29.026324+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-Backup `2019-09-19T11_14_40.410627+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-Backup `2019-09-19T11_14_29.026324+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-All backups in range `18-09-2019,18-09-2019` were deleted
-
-
-
-

Finally, we can also provide exact backup archive name ranges such as -2019-09-18T11_13_58.136188+01_00,2019-09-19T11_14_29.026324+01_00.

-
-
-
-
$ cbbackupmgr remove -a ~/backup_repo -r example --backups 2019-09-18T11_13_58.136188+01_00,2019-09-19T11_14_29.026324+01_00
-Backup `2019-09-18T11_13_58.136188+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-Backup `2019-09-19T11_14_29.026324+01_00` deleted successfully from archive `/Users/carlosbetancourt/backup_repo` repository `example`
-All backups in range `2019-09-18T11_13_58.136188+01_00,2019-09-19T11_14_29.026324+01_00` were deleted
-
-
-
-

Removing data that is stored in AWS S3 is very similar to removing a local repository/backup, below is an example -command which remove a given backup:

-
-
-
-
$ cbbackupmgr remove -a s3://storage/backup_archive -r backup_repo --obj-staging-dir ~/backup-archive --backups '2020-04-22T14_10_18.372643628+01_00'
-Backup `2020-04-22T14_10_18.372643628+01_00` deleted successfully from archive `backup_archive` repository `backup_repo`
-
-
-
-
-
-

DISCUSSION

-
-
-

The remove command is used to safely remove a backup repository from an archive. -This command is provided because only the cbbackupmgr utility should ever be -used to access or modify a backup archive. Removing a backup repository will -remove data permanently from disk. Data is not recoverable once it has been -removed.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-config(1), -cbbackupmgr-info(1), -cbbackupmgr-strategies(1)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-restore.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-restore.html deleted file mode 100644 index 33aef03e..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-restore.html +++ /dev/null @@ -1,1988 +0,0 @@ - - - - - - - - - -cbbackupmgr-restore(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr restore [--archive <archive_dir>] [--repo <repo_name>]
-                    [--cluster <host>] [--username <username>]
-                    [--password <password>] [--client-cert <path>]
-                    [--client-cert-password <password>] [--client-key <path>]
-                    [--client-key-password <password>] [--start <start>]
-                    [--end <end>] [--include-data <collection_string_list>]
-                    [--exclude-data <collection_string_list>]
-                    [--map-data <collection_string_mappings>]
-                    [--disable-cluster-analytics] [--disable-analytics]
-                    [--disable-views] [--disable-gsi-indexes]
-                    [--disable-ft-indexes] [--disable-ft-alias]
-                    [--disable-data] [--disable-eventing]
-                    [--disable-bucket-query] [--disable-cluster-query]
-                    [--replace-ttl <type>]
-                    [--replace-ttl-with <timestamp>] [--force-updates]
-                    [--threads <integer>] [--vbucket-filter <integer_list>]
-                    [--no-progress-bar] [--auto-create-buckets]
-                    [--autoremove-collections] [--continue-on-cs-failure]
-                    [--restore-partial-backups] [--obj-access-key-id <access_key_id>]
-                    [--obj-cacert <cert_path>] [--obj-endpoint <endpoint>]
-                    [--obj-read-only-mode] [--obj-no-ssl-verify]
-                    [--obj-region <region>] [--obj-staging-dir <staging_dir>]
-                    [--obj-secret-access-key <secret_access_key>]
-                    [--s3-force-path-style] [--s3-log-level <level>]
-                    [--point-in-time <time>]
-                    [--filter-keys <regexp>] [--filter-values <regexp>]
-                    [--passphrase <passphrase>] [--km-key-url <url>]
-                    [--km-endpoint <endpoint>] [--km-region <region>]
-                    [--km-access-key-id <id>] [--km-secret-access-key <key>]
-                    [--km-auth-file <path>] [--purge] [--resume]
-
-
-
-
-

DESCRIPTION

-
-
-

Restores data from the backup archive to a target Couchbase cluster. By default -all data, index definitions, view definitions and full-text index definitions -are restored to the cluster unless specified otherwise in the repos backup -config or through command line parameters when running the restore command.

-
-
-

The restore command is capable of restoring a single backup or a range of -backups. When restoring a single backup, all data from that backup is restored. -If a range of backups is restored, then cbbackupmgr will take into account -any failovers that may have occurred in between the time that the backups were -originally taken. If a failover did occur in between the backups, and the backup -archive contains data that no longer exists in the cluster, then the data that -no longer exists will be skipped during the restore. If no failovers occurred in -between backups then restoring a range of backups will restore all data from -each backup. If all data must be restored regardless of whether a failover -occurred in between the original backups, then data should be restored one -backup at a time.

-
-
-

The restore command is guaranteed to work during rebalances and failovers. If a -rebalance is taking place, cbbackupmgr will track the movement of vbuckets -around a Couchbase cluster and ensure that data is restored to the appropriate -node. If a failover occurs during the restore then the client will wait 180 -seconds for the failed node to be removed from the cluster. If the failed node -is not removed in 180 seconds then the restore will fail, but if the failed node -is removed before the timeout then data will continue to be restored.

-
-
-

Note that if you are restoring indexes then it is highly likely that you will -need to take some manual steps in order to properly restore them. This is -because by default indexes will only be built if they are restored to the exact -same index node that they were backed up from. If the index node they were -backed up from does not exist then the indexes will be restored in round-robin -fashion among the current indexer nodes. These indexes will be created, but not -built and will required the administrator to manually build them. We do this -because we cannot know the optimal index topology ahead of time. By not building -the indexes the administrator can move each index between nodes and build them -when they deem that the index topology is optimal.

-
-
-
-
-

OPTIONS

-
-
-

Below is a list of required and optional parameters for the restore command.

-
-
-

Required

-
-
-
-a,--archive <archive_dir>
-
-

The directory containing the backup repository to restore data from. When -restoring from an archive stored in S3 prefix the archive path with -s3://${BUCKET_NAME}/.

-
-
-r,--repo <repo_name>
-
-

The name of the backup repository to restore data from.

-
-
-c,--cluster <hostname>
-
-

The hostname of one of the nodes in the cluster to restore data to. See the -Host Formats section below for hostname specification details.

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
-
-
-
-

Optional

-
-
-
--start <start>
-
-

The first backup to restore. See START AND END for information on what values -are accepted.

-
-
--end <end>
-
-

The final backup to restore. See START AND END for information on what values -are accepted.

-
-
--include-data <collection_string_list>
-
-

Overrides the repository configuration to restore only the data specified in -the <collection_string_list>. This flag takes a comma separated list of -collection strings and can’t be specified at the same time as ---exclude-data. Note that including data at the scope/collection level is -an Enterprise Edition feature.

-
-
--exclude-data <collection_string_list>
-
-

Overrides the repository configuration to skip restoring the data specified -in the <collection_string_list>. This flag takes a comma separated list of -collection strings and can’t be specified at the same time as ---include-data. Note that excluding data at the scope/collection level is -an Enterprise Edition feature.

-
-
--filter-keys
-
-

Only restore data where the key matches a particular regular expression. -The regular expressions provided must follow -RE2 syntax.

-
-
--filter-values
-
-

Only restore data where the value matches a particular regular expression. -The regular expressions provided must follow -RE2 syntax.

-
-
--enable-bucket-config
-
-

Enables restoring the bucket configuration.

-
-
--disable-views
-
-

Skips restoring view definitions for all buckets.

-
-
--disable-gsi-indexes
-
-

Skips restoring gsi index definitions for all buckets.

-
-
--disable-ft-indexes
-
-

Skips restoring full-text index definitions for all buckets.

-
-
--disable-ft-alias
-
-

Skips restoring full-text alias definitions.

-
-
--disable-data
-
-

Skips restoring all key-value data for all buckets.

-
-
--disable-cluster-analytics
-
-

Skips restoring analytics cluster level analytics metadata e.g. Synonyms.

-
-
--disable-analytics
-
-

Skips restoring bucket level analytics metadata.

-
-
--disable-eventing
-
-

Skips restoring the eventing service metadata.

-
-
--disable-bucket-query
-
-

Skips restoring bucket level Query Service metadata.

-
-
--disable-cluster-query
-
-

Skips restoring cluster level Query Service metadata.

-
-
--force-updates
-
-

Forces data in the Couchbase cluster to be overwritten even if the data in -the cluster is newer. By default updates are not forced and all updates use -Couchbase’s conflict resolution mechanism to ensure that if newer data -exists on the cluster that is not overwritten by older restore data.

-
-
--map-data <collection_string_mappings>
-
-

Specified when you want to restore source data into a different location. For -example this argument may be used to remap buckets/scopes/collections with -the restriction that they must be remapped at the same level. For example a -bucket may only be remapped to a bucket, a scope to a scope and a collection -to a collection. The argument expects a comma separated list of collection -string mappings e.g. -bucket1=bucket2,bucket3.scope1=bucket3.scope2,bucket4.scope.collection1=bucket4.scope.collection2 -If used to remap a bucket into a collection then it will only restore data -for the data service and will skip data for all the other services.

-
-
--replace-ttl <type>
-
-

Sets a new expiration (time-to-live) value for the specified keys. This -parameter can either be set to "none", "all" or "expired" and should be used -along with the --replace-ttl-with flag. If "none" is supplied then the TTL -values are not changed. If "all" is specified then the TTL values for all -keys are replaced with the value of the --replace-ttl-with flag. If -"expired" is set then only keys which have already expired will have the -TTL’s replaced. For more information about the behavior of --replace-ttl -see the REPLACE TTL.

-
-
--replace-ttl-with <timestamp>
-
-

Updates the expiration for the keys specified by the --replace-ttl parameter. -The parameter has to be set when --replace-ttl is set to "all". There are two -options, RFC3339 time stamp format (2006-01-02T15:04:05-07:00) or "0". When -"0" is specified the expiration will be removed. Please note that the RFC3339 -value is converted to a Unix time stamp on the cbbackupmgr client. It is -important that the time on both the client and the Couchbase Server are the -same to ensure expiry happens correctly. For more information about the -behavior of --replace-ttl-with see the REPLACE TTL.

-
-
--vbucket-filter <list>
-
-

Specifies a list of VBuckets that should be restored. VBuckets are specified -as a comma separated list of integers. If this parameter is not set then all -vBuckets which were backed up are restored.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man-in-the-middle attack if -you use this flag. Either this flag or the --cacert flag must be specified -when using an SSL encrypted connection.

-
-
--cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag -must be specified when using an SSL encrypted connection.

-
-
-t,--threads <num>
-
-

Specifies the number of concurrent clients to use when restoring data. Fewer -clients means restores will take longer, but there will be less cluster -resources used to complete the restore. More clients means faster restores, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the restore is -taking place.

-
-
--no-progress-bar
-
-

By default, a progress bar is printed to stdout so that the user can see how -long the restore is expected to take, the amount of data that is being -transferred per second, and the amount of data that has been restored. -Specifying this flag disables the progress bar and is useful when running -automated jobs.

-
-
--auto-create-buckets
-
-

It will create the destination buckets if not present in the server.

-
-
--autoremove-collections
-
-

Automatically delete scopes/collections which are known to be deleted in the -backup. See SCOPE_COLLECTION_DELETION for more details.

-
-
--continue-on-cs-failure
-
-

It’s possible that during a restore, a checksum validation will fail; in this -case the restore will fail fast. Supplying this flag will mean that the restore -will attempt to continue upon receiving a checksum failure. See CHECKSUM FAILURE -for more information.

-
-
--restore-partial-backups
-
-

Allow a restore to continue when the final backup in the restore range is -incomplete. This flag is incompatible with the --obj-read-only flag.

-
-
--point-in-time <time>
-
-

(Beta) Specifies the point in time to restore to. The value accepted is ISO8601 date -time format (YYYY-MM-DDTHH:MM:SS). This feature is currently in Beta and is not -supported, this should only be used in test environments.

-
-
--purge
-
-

If the last restore failed before it finished, then remove it’s progress -(which is persisted to disk) then restart from zero. Note that only the -restore progress is purge, no backup data will be removed.

-
-
--resume
-
-

If the last restore failed before it finished, then try to continue from -where it left off.

-
-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-read-only
-
-

Enable read only mode. When interacting with a cloud archive modifications -will be made e.g. a lockfile will be created, log rotation will take place -and the modified logs will be uploaded upon completion of the subcommand. -This flag disables these features should you wish to interact with an archive -in a container where you lack write permissions. This flag should be used -with caution and you should be aware that your logs will not be uploaded to -the cloud. This means that it’s important that if you encounter an error you -don’t remove you staging directory (since logs will still be created in there -and collected by the collect-logs subcommand).

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-

Encryption (Developer Preview)

-
-
-
--passphrase <passphrase>
-
-

Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing.

-
-
--km-key-url <url>
-
-

Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable CB_KM_KEY_URL. For more on how to authenticate using the different -providers see cbbackupmgr-encryption(7).

-
-

For AWS the expected key format is awskms://<KEY-ID|KEY-ALIAS>, for example -awskms://alias\keyAlias.

-
-
-

For GCP the expected key format is gcpkms://<KEY-RESOURCE-ID>, for example -gcpkms://projects/project-id/locations/location/keyRings/keyring/cryptoKeys/key.

-
-
-

For Azure key vault the expected key format is azurekeyvault://<KEY-IDENTIFIER> -for example: -azurekeyvault://vault-name.vault.azure.net/object-type/object-name/object-version.

-
-
-

For HashiCorp Vault the expected format is hashivaults://<HOST>/<KEY-NAME> -for example: -hashivaults://127.0.0.1:8200/keyName.

-
-
-
--km-region <region>
-
-

Required when using AWS KMS, it allows you to set the key region.

-
-
--km-endpoint <endpoint>
-
-

The host or address to use as your KMS. It will override the default SDK one.

-
-
--km-access-key-id <id>
-
-

The user ID used to connect to the key management service. It can also be -provided via CB_KM_ACCESS_KEY_ID environmental variable. Please refer -to cbbackupmgr-encryption(7) for the required authentication for each -provider.

-
-
--km-secret-access-key <key>
-
-

The key used to connect to the key management service. It can also be -provided via the CB_KM_SECRET_ACCESS_KEY environmental variable. Please -refer to cbbackupmgr-encryption(7) for the required authentication for -each provider.

-
-
--km-auth-file <path>
-
-

The path to a file containing the authentication credentials for the key -management service. It can also be provided via the CB_KM_AUTH_FILE -environmental variable. Please refer to cbbackupmgr-encryption(7) for the -required authentication for each provider.

-
-
-
-
-
-
-
-

START AND END

-
-
-

This sub-command accepts a --start and --end flag. These flags accept -multiple values to allow you to flexibly operate on a range of backups.

-
-
-

Indexes

-
-

Indexes may be supplied to operate on a range of backups, for example ---start 1 --end 2 will include start at the first backup and will finish with -the second backup. Note that the first backup is 1 and not 0 and that the ---end flag is inclusive.

-
-
-
-

Short Dates

-
-

Short dates may be supplied in the format day-month-year. For example ---start 01-08-2020 --end 31-08-2020 will operate on all the backups which -were taken during August of 2020. Note that the end date is inclusive.

-
-
-

When supplying short dates, you may supply start or oldest as a placeholder -for the date on which the first backup in this repository was taken. The -keywords end or latest may be used as a placeholder for the date last -backup in the repository was taken.

-
-
-
-

Backup Names

-
-

Backup names may be supplied as they exist on disk. For example ---start 2020-08-13T20_01_08.894226137+01_00 --end 2020-08-13T20_01_12.348300092+01_00 -will cause the sub-command to operate on all the backups which inclusively fall -between these two backups.

-
-
-

When supplying backup names, you may supply start or oldest as a -placeholder for the first backup in the repository. The keywords end or -latest may be used as a placeholder for the final backup in the repository.

-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

RBAC

-
-
-

When performing a backup/restore with a user which is using RBAC, there are a -couple of things that should be taken into consideration each of which is -highlighted in this section.

-
-
-

Bucket Level

-
-

Bucket level data may be backed up/restored using the data_backup (Data -Backup & Restore) role.

-
-
-

The data_backup role does not have access to cluster level data such as:

-
-
-
    -
  • -

    Analytics Synonyms

    -
    • -
  • -

    Eventing Metadata

    -
    • -
  • -

    FTS Aliases

    -
    • -
-
-
-

Backing up/restoring cluster level data with the data_backup role will cause -permission errors like the one below.

-
-
-
-
Error backing up cluster: {"message":"Forbidden. User needs one of the following permissions","permissions":["cluster.fts!read"]}
-
-
-
-

When presented with an error message such as the one above, there’s two clear -options.

-
-
-

The first option is to provide the user with the required credentials using -either the cli, REST API or Couchbase Server WebUI. This can be done by editing -the user and adding the required role. See Cluster Level for more information -about the required roles.

-
-
-

Secondly, backing up/restoring the specific service can be disabled. For -backups this must be done when configuring the repository with the config -command using the --disable style flags. For restore, these flags may be used -directly to disable one or more services. See the backup/restore documentation -for more information.

-
-
-
-

Cluster Level

-
-

Backing up/restoring cluster level data requires additional RBAC roles, each of -which is highlighted below:

-
-
-
-
Analytics Synonyms
-
-

analytics_admin (Analytics Admin)

-
-
Eventing Metadata
-
-

eventing_admin (Eventing Full Admin)

-
-
FTS Aliases
-
-

fts_admin (Search Admin)

-
-
-
-
-

These additional roles are required since this is cluster level data which may -encompass multiple buckets.

-
-
-
-
-
-

Supported Backup Versions

-
-
-

The restore sub-command currently supports restoring backups created by all -previous versions of cbbackupmgr, however, this won’t always be the case.

-
-
-

The ForestDB storage format (used by cbbackupmgr versions prior to, but not -including 6.5.0) is now deprecated.

-
-
-

This means that future releases of cbbackupmgr will not be able to restore -backups created by versions of cbbackupmgr prior to but not including 6.5.0.

-
-
-

These backups are still safe and usable, however, they must be restored/merged -with a version of cbbackupmgr which supports interacting with 6.0.x archives -e.g. 6.5.x, 6.6.x, 7.0.x and 7.1.x.

-
-
-

Example

-
-

Imagine you have a backup created by a 6.0.x version of cbbackupmgr, this -will use the ForestDB storage format. You’d like to restore this backup, -however, the latest version no longer supports interacting with this format.

-
-
-

In this case, you could either:

-
-
-
    -
  1. -

    Restore the backup using cbbackupmgr from 6.5.x, 6.6.x, 7.0.x or 7.1.x.

    -
    2. -
  2. -

    Merge two or more backups using cbbackupmgr from 6.5.x, 6.6.x, 7.0.x or 7.1.x then restore it using the latest version.

    -
    3. -
-
-
-

Although in this case you were unable to directly restore the backup with the -latest version of cbbackupmgr, the data is still indirectly accessible for -restore.

-
-
-
-
-
-

REPLACE TTL

-
-
-

The behavior of the --replace-ttl/--replace-ttl-with flags is well defined, -however, there are some conditions where the behavior may seem surprising or -unexpected due to conflict resolution.

-
-
-

Imagine the case where a backup contains one or more documents which have an -expiry which has now elapsed. There are several possible scenarios which could -take place when restoring these documents when using the --replace-ttl and ---replace-ttl-with flags. These scenarios are enumerated below.

-
-
-

RESTORING TO A NEW CLUSTER/BUCKET

-
-

When restoring to a new cluster it’s expected that all the documents which match -the all/expired condition will be restored with their new/updated ttl values.

-
-
-
-

RESTORING TO THE SAME BUCKET

-
-

The most interesting/unexpected cases occur when restoring the backup to the -same bucket at some point in the future.

-
-
-

EXPIRED DOCUMENTS HAVE NOT BEEN PURGED

-
-

In the event that the restore takes place and the expired documents have not -been purged yet, conflict resolution will take precedence and the documents will -not be restored. This behavior will manifest itself as skipped mutations which -will be displayed in restore sub-command output.

-
-
-
-
Restoring backup '2021-05-17T11_00_15.843794944+01_00'
-Copied all data in 1.773s (Avg. 21.03MiB/Sec)                                                                           31591 items / 21.03MiB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 21.03MiB          | Mon, 17 May 2021 11:00:25 +0100/s | Mon, 17 May 2021 11:00:26 +0100 | 1.785s   |
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 21.03MiB    | 21.03MiB/s        | Mon, 17 May 2021 11:00:25 +0100 | Mon, 17 May 2021 11:00:26 +0100 | 1.713s   |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 0        | 0       | 31591   | 0        | 0       | 0       | 0        | 0       | 0       |
-
-Restore completed successfully
-
-
-
-
-

EXPIRED DOCUMENTS HAVE BEEN PURGED

-
-

If the restore is performed after the user-defined purge interval where a -compaction has taken place, the documents would be restored because the expired -documents would no longer exist in the cluster.

-
-
-
-

FORCING UPDATES

-
-

The above behavior may be overridden by using the --force-updates flag which -will bypass conflict resolution and result in the documents from the backup -being restored.

-
-
-

The --force-updates flag will affect all the documents being restored and not -just those which contain an expiry which is being replaced. This may result in -documents being overwritten with older versions from the backup; if the expired -documents keys are known beforehand, a mixed use of --force-updates and ---filter-keys may be more precise.

-
-
-
-
-
-
-

EXAMPLES

-
-
-

The restore command can be used to restore a single backup or range of backups -in a backup repository. In the examples below, we will look a few different ways -to restore data from a backup repository. All examples will assume that the -backup archive is located at /data/backups and that all backups are located in -the "example" backup repository.

-
-
-

The first thing to do when getting ready to restore data is to decide which -backups to restore. The easiest way to do this is to use the info command to see -which backups are available to restore.

-
-
-
-
$ cbbackupmgr info --archive /data/backups --repo example --all
-Name     | Size    | # Backups  |
-example  | 4.38MB  | 3          |
-
-+  Backup                            | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-02T07_49_11.281004+01_00  | 1.69MB  | FULL  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 1.69MB  | 4096   | 4096       | 0           | 0      | 0    | 0        | 0     |
-
-+  Backup                            | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-03T07_49_52.577901+01_00  | 1.34MB  | INCR  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 1.34MB  | 2048   | 2048       | 0           | 0      | 0    | 0        | 0     |
-
-+  Backup                            | Size    | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+  2020-06-04T07_50_06.908787+01_00  | 1.34MB  | INCR  | http://localhost:8091  | c044f5eeb1dc16d0cd49dac29074b5f9  | N/A    | 0       | 1        | true      |
-
--    Bucket   | Size    | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--    example  | 1.34MB  | 2048   | 2048       | 0           | 0      | 0    | 0        | 0     |
-
-
-
-

From the information of the backup repository we can see we have three backups -that we can restore in the "examples" backup repository. If we just want to -restore one of them we set the --start and --end flags in the restore command -to the same backup name and specify the cluster that we want to restore the data -to. In the example below we will restore only the oldest backup.

-
-
-
-
$ cbbackupmgr restore -a /data/backups -r example \
- -c couchbase://127.0.0.1 -u Administrator -p password \
- --start 2020-06-02T07_49_11.281004+01_00 \
- --end 2020-06-02T07_49_11.281004+01_00
-
-
-
-

If we want to restore only the two most recent backups then we specify the ---start and --end flags with different backup names in order to specify the -range we want to restore.

-
-
-
-
$ cbbackupmgr restore -a /data/backups -r example \
- -c couchbase://127.0.0.1 -u Administrator -p password \
- --start 2020-06-02T07_49_11.281004+01_00 \
- --end 2020-06-03T07_49_52.577901+01_00
-
-
-
-

If we want to restore all of the backups in the "examples" directory then we can -omit the --start and --end flags since their default values are the oldest and -most recent backup in the backup repository.

-
-
-
-
$ cbbackupmgr restore -a /data/backups -r example \
- -c couchbase://127.0.0.1 -u Administrator -p password
-
-
-
-

Restore also allows filtering the data restored by document key and/or value by passing -regular expressions to the flags --filter-keys and --filter-values respectively. -Say we backup the sample bucket 'beer-sample' if we only wanted to restore only the -documents that have a key that starts with '21st_amendment_brewery_cafe'. This can be -done using the flag --filter-keys as shown bellow.

-
-
-
-
$ cbbackupmg restore -c http://127.0.0.1:8091 -u Administrator -p password \
- -a /data/backups -r beer --filter-keys '^21st_amendment_brewery_cafe.*'
-
-
-
-

Restore also allows filtering by value. Let’s say we only want to restore documents that -contain the JSON field address. This could be done by passing the regular expression -{.*"address":.*} to the --filter-values flag as illustrated below.

-
-
-
-
$ cbbackupmgr -c http://127.0.0.1:8091 -u Administrator -p password \
- -a /data/backups -r beer --filter-values '{.*"address":.*}'
-
-
-
-

Finally, we can combine both flags to filter by both key and value. Imagine you want to -restore the values for beers that start with the key '21st_amendment_brewery_cafe' and -have the JSON field "category":"North American Ale". This can be done by using -the command bellow.

-
-
-
-
$ cbbackupmgr -c http://127.0.0.1:8091 -u Administrator -p password \
- -a /data/backups -r beer --filter-values '{.*"category":"North American Ale".*}' \
- --filter-keys '^21st_amendment_brewery_cafe.*'
-
-
-
-

The regular expressions provided must follow RE2 syntax.

-
-
-
-
-

CHECKSUM FAILURE

-
-
-

A checksum failure may occur during a restore and indicates that a document -has changed since the creation of the backup. Depending on the type of corruption -we may be able to restore by skipping only the corrupted documents. However, if the -size of the data file has changed (e.g. not a bit flip or byte for byte modification) -all documents after the corruption (for that vBucket) will be unusable.

-
-
-
-
-

AUTOMATIC COLLECTION CREATION

-
-
-

By design, users may not recreate the _default collection once it has been -deleted. Therefore, this means that the _default collection can’t (and won’t) -be recreated if it’s missing. Before performing a transfer, a check will take -place to see if the _default collection will be required when it’s missing. -If this is the case, the command will exit early and you will be required to -remap the _default collection using the --map-data flag.

-
-
-
-
-

AUTOMATIC COLLECTION DELETION

-
-
-

During a backup cbbackupmgr will take note of which scopes/collections were -create/deleted/modified up to the point that the backup began. This behavior -can be leveraged to automatically delete any scopes/collections which are -marked as deleted in the backup. We will only delete scopes/collections which -are identical to the ones which are stored in the backup; ones which match by -both id and name.

-
-
-
-
-

REMAPPING

-
-
-

During a transfer, scopes/collections can be remapped from one location to -another. There are several rules that are enforced when remapping -scopes/collections, they are as follows:

-
-
-
    -
  • -

    You must be running an Enterprise Edition version of Couchbase Server.

    -
    • -
  • -

    You may not remap the _default scope (discussed in THE DEFAULT SCOPE).

    -
    • -
  • -

    You may only remap scopes/collections at the same level meaning scopes may -be remapped to other scopes, and collections to other collections, however, a -scope can’t be remapped to a collection or vice versa.

    -
    • -
  • -

    Scopes/collections may only be remapped within the same bucket. For example -the mapping bucket1.scope.collection=bucket2.scope.collection is invalid.

    -
    • -
  • -

    Scopes/collections may only be remapped once. For example the mapping -bucket1.scope1=bucket1.scope2,bucket1.scope1=bucket1.scope3 is invalid.

    -
    • -
  • -

    Remapping may only take place at one level at once meaning that if a parent -bucket/scope is already remapped, the child scopes/collections may not also -be remapped. For example the mapping -bucket1.scope1=bucket1.scope2,bucket1.scope1.collection1=bucket1.scope3.collection9 -is invalid.

    -
    • -
-
-
-

REMAPPING A SCOPE/COLLECTION WITHOUT RENAMING

-
-

During a transfer, it’s possible for a scope/collection to encounter a conflict -(for example, because it has been recreated). It may not be preferable to -rename the scope/collection during the transfer.

-
-
-

For this reason, the --map-data flag, allows you to remap a scope/collection -to itself; this indicates that the scope/collection that exists in the target -(with a different id) should be treated as the same.

-
-
-

As an example, the following error message indicates that a collection has been -recreated prior to a restore.

-
-
-
-
Error restoring cluster: collection 8 with name 'collection1' in the scope '_default' exists with a different name/id on the cluster, a manual remap is required
-
-
-
-

Using the --map-data flag with the argument -bucket._default.collection1=bucket._default.collection1 would cause -cbbackupmgr to treat collection1 (with id 8) as collection1 (with the id -it exists with in the target).

-
-
-
-

THE DEFAULT SCOPE

-
-

As mentioned in AUTOMATIC COLLECTION CREATION, it’s not possible to recreate -the _default scope/collection. This means you can’t remap the _default -scope because the tool may be unable to create a destination scope/collection. -This may be worked around by remapping each collection inside the _default -scope.

-
-
-
-

BUCKET TO COLLECTION REMAPPING

-
-

As discussed in REMAPPING, it’s not possible to remap data at different levels; -buckets must be remapped to buckets, scopes to scopes and collections to -collections. However, there is one supported edge case, which is remapping a -bucket into a collection to allow migration from a collection unaware to -collection aware datasets.

-
-
-

To remap a bucket into a collection using --map-data you may supply ---map-data bucket._default._default=bucket.scope.collection. This -functionality is compatible with cross bucket mapping, for example you may also -supply --map-data bucket1._default._default=bucket2.scope.collection.

-
-
-

Note that once you’ve provided a mapping to remap a bucket into a collection -you may not remap that bucket elsewhere. For example --map-data -bucket1._default._default=bucket2.scope.collection,bucket1=bucket3 is invalid.

-
-
-
-

REMAPPING MULTIPLE DATA SOURCES INTO A SINGLE TARGET SOURCE

-
-

As outlined in the rules discussed in REMAPPING, it’s not possible to remap a -bucket/scope/collection multiple times, however, it is possible to remap to a -single destination multiple times. For example the mapping -bucket1=dest,bucket2=dest,bucket3=dest is valid.

-
-
-

Although valid, this manor of remapping is dangerous and can result in data not -being transferred due to conflicting key spaces. If this style of remapping is -detected a warning will be printed before proceeding. -:!supports_automatic_collection_deletion:

-
-
-
-
-
-

RESTORING A COLLECTION AWARE BACKUP TO A COLLECTION UNAWARE CLUSTER

-
-
-

The restore sub-command supports restoring collection aware backups to -collection unaware cluster. When restoring a collection aware backup to a -cluster which doesn’t support collections, cbbackupmgr will restore the -_default._default collection into the target bucket; no data will be -transferred for any other collections.

-
-
-

This allows you to utilize a collection aware cluster, without using the -collections feature and still be able to restore your data to a cluster which -is running a previous version of Couchbase which is collection unaware.

-
-
-
-
-

DISCUSSION

-
-
-

The restore command works by replaying the data recorded in backup files. During -a restore each key-value pair backed up by cbbackupmgr will be sent to the -cluster as either a "set" or "delete" operation. The restore command replays -data from each file in order of backup time to guarantee that older backup data -does not overwrite newer backup data. The restore command uses Couchbase’s -conflict resolution mechanism by default to ensure this behavior. The conflict -resolution mechanism can be disable by specifying the --force-updates flag when -executing a restore.

-
-
-

Starting in Couchbase 4.6 each bucket can have different conflict resolution -mechanisms. cbbackupmgr will backup all meta data used for conflict resolution, -but since each conflict resolution mechanism is different cbbackupmgr will -prevent restores to a bucket when the source and destination conflict resolution -methods differ. This is done because by default cbbackupmgr will use the -conflict resolution mechanism of the destination bucket to ensure an older value -does not overwrite a newer value. If you want to restore a backup to a bucket -with a different conflict resolution type you can do by using the ---force-updates flag. This is allowed because forcing updates means that -cbbackupmgr will skip doing conflict resolution on the destination bucket.

-
-
-

Also keep in mind that unlike backups, restores cannot be resumed if they fail.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
CB_ENCRYPTION_PASSPHRASE
-
-

Specifies the passphrase used for encryption.

-
-
CB_KM_KEY_URL
-
-

Specifies the URL identifying the encryption key on the KMS. See ---km-key-url for the expected format and accepted KMSs.

-
-
CB_KM_ACCESS_ID
-
-

Specifies the key/user ID used to connect to the KMS.

-
-
CB_KM_SECRET_ACCESS_KEY
-
-

Specifies the secret key/token used to connect to the KMS.

-
-
CB_KM_AUTH_FILE
-
-

Specifies a path to a file containing the required credentials to connect to -the KMS.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-backup(1), -cbbackupmgr-info(1), -cbbackupmgr-cloud(7) -cbbackupmgr-network-filesystems(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-strategies.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-strategies.html deleted file mode 100644 index 0195a42b..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-strategies.html +++ /dev/null @@ -1,606 +0,0 @@ - - - - - - - - - -cbbackupmgr-strategies(7) - - - - - -
-
-

DESCRIPTION

-
-
-

Backing up data is an important part of any production database deployment. In -this section we will go through the most common cbbackupmgr backup strategies -and also provide recommendations for how and where to run cbbackupmgr. To -describe each strategy we will assume that users will want to back up their -cluster daily, but all strategies can be easily modified to accommodate -different backup intervals.

-
-
-

In the strategies below we will refer to taking a full backup even though -cbbackupmgr is an incremental-only tool. When this term is used we are referring -to the first backup taken in a backup repository.

-
-
-
-
-

PERIODIC MERGE

-
-
-

The recommended backup strategy for cbbackupmgr is the Periodic Merge strategy. -This backup strategy is intended to have the lowest cluster overhead and will -require the least amount of time. The periodic merge strategy takes full -advantage the fact that cbbackupmgr only takes incremental backups as well as -its ability to merge incremental backup. In a Periodic Merge strategy we would -take an incremental backup each day as follows:

-
-
-
-
Sunday:    Config Backup Repo    (0 backups in backup repo)
-Sunday:    Backup 1              (1 backup in backup repo)
-Monday:    Backup 2              (2 backups in backup repo)
-Tuesday:   Backup 3              (3 backups in backup repo)
-Wednesday: Backup 4              (4 backups in backup repo)
-Thursday:  Backup 5              (5 backups in backup repo)
-Friday:    Backup 6              (6 backups in backup repo)
-Saturday:  Backup 7              (7 backups in backup repo)
-Sunday:    Backup 8              (8 backups in backup repo)
-Sunday:    Merge 1-7             (2 backups in backup repo)
-Monday:    Backup 9              (3 backups in backup repo)
-Tuesday:   Backup 10             (4 backups in backup repo)
-Wednesday: Backup 11             (5 backups in backup repo)
-Thursday:  Backup 12             (6 backups in backup repo)
-Friday:    Backup 13             (7 backups in backup repo)
-Saturday:  Backup 14             (8 backups in backup repo)
-Sunday:    Backup 15             (9 backups in backup repo)
-Sunday:    Merge 7-14            (2 backups in backup repo)
-
-
-
-

Each day we increase the amount of incremental backups in the backup repository, -but in order to reclaim space we merge the incremental backups from the previous -week on Sundays in order to reclaim space. This strategy is effective because -incremental backups are the quickest way to back up a cluster and when we need -to consolidate the data we can do so without using any cluster resource. This -strategy is useful because as data grows it may become infeasible to back up the -entire cluster in a reasonable amount of time. By always taking incremental -backups and merging at a later time, we can completely avoid full cluster -backups.

-
-
-
-
-

Full/Incremental Approach

-
-
-

The Full/Incremental Approach is similar to the Periodic Merge strategy, but -has one key difference. Instead of relying on the merge feature to merge backups -together we create a new backup repository periodically and back up the entire -cluster again. We then take incremental backups daily. Below is an example of -what this strategy would look like:

-
-
-
-
Sunday:    Config Backup Repo 1  (0 backups in backup repo)
-Sunday:    Backup 1              (1 backup in backup repo)
-Monday:    Backup 2              (2 backups in backup repo)
-Tuesday:   Backup 3              (3 backups in backup repo)
-Wednesday: Backup 4              (4 backups in backup repo)
-Thursday:  Backup 5              (5 backups in backup repo)
-Friday:    Backup 6              (6 backups in backup repo)
-Saturday:  Backup 7              (7 backups in backup repo)
-Sunday:    Config Backup Repo 2  (0 backup in backup repo)
-Sunday:    Backup 8              (1 backups in backup repo)
-Monday:    Backup 9              (2 backups in backup repo)
-Tuesday:   Backup 10             (3 backups in backup repo)
-Wednesday: Backup 11             (4 backups in backup repo)
-Thursday:  Backup 12             (5 backups in backup repo)
-Friday:    Backup 13             (6 backups in backup repo)
-Saturday:  Backup 14             (7 backups in backup repo)
-Sunday:    Create Backup Repo 3  (0 backups in backup repo)
-Sunday:    Backup 15             (1 backup in backup repo)
-
-
-
-

In this strategy we don’t use the merge command, but instead rely on creating a -new backup repository each week on Sunday. In order to free up space in this -strategy we would delete old backup repositories once they are deemed to be too -old to be useful. This strategy is useful for small to medium sized clusters due -to the requirement to take full backups.

-
-
-
-
-

Full Backup Only

-
-
-

In this strategy we only take full backups. This strategy is only useful for -small clusters and puts the most strain on the cluster compared to other -strategies. A backup strategy using only full backups would look like this:

-
-
-
-
Sunday:    Config Backup Repo 1  (0 backups in backup repo)
-Sunday:    Backup 1              (1 backup in backup repo)
-Monday:    Config Backup Repo 2  (0 backups in backup repo)
-Monday:    Backup 2              (1 backup in backup repo)
-Tuesday:   Config Backup Repo 3  (0 backups in backup repo)
-Tuesday:   Backup 3              (1 backup in backup repo)
-Wednesday: Config Backup Repo 4  (0 backups in backup repo)
-Wednesday: Backup 4              (1 backup in backup repo)
-Thursday:  Config Backup Repo 5  (0 backups in backup repo)
-Thursday:  Backup 5              (1 backup in backup repo)
-Friday:    Config Backup Repo 6  (0 backups in backup repo)
-Friday:    Backup 6              (1 backup in backup repo)
-Saturday:  Config Backup Repo 7  (0 backups in backup repo)
-Saturday:  Backup 7              (1 backup in backup repo)
-Sunday:    Config Backup Repo 8  (0 backups in backup repo)
-Sunday:    Backup 8              (1 backup in backup repo)
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-config(1), -cbbackupmgr-backup(1), -cbbackupmgr-merge(1)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-tutorial.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-tutorial.html deleted file mode 100644 index 7dbb2f43..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr-tutorial.html +++ /dev/null @@ -1,890 +0,0 @@ - - - - - - - - - -cbbackupmgr-tutorial(7) - - - - - -
-
-

DESCRIPTION

-
-
-

A tutorial that goes gives examples of how to use all of the commands in -cbbackupmgr effectively.

-
-
-
-
-

TUTORIAL

-
-
-

In this tutorial we will show how to take backups and restore data using -cbbackupmgr. This tutorial uses a cluster that contains both the travel-sample -and beer-sample buckets installed and requires modifying some of the documents -in the travel-sample bucket. To make it easier to set up a cluster and edit/get -documents the following scripts are provided at -http://github.com/couchbaselabs/backup-tutorial. You can then find scripts -corresponding to your version of Couchbase. We will reference other scripts in -this github repository later in the tutorial so it is recommended that you -download these scripts. The only requirement for running the scripts is that you -have curl installed. To automatically setup the cluster in the appropriate state -required for this tutorial download and install Couchbase and then run the -01-initialize.sh script. If you do not want to use this script then you can -navigate through the Couchbase setup process and initialize the cluster with all -available services and install the travel-sample and beer-sample sample data -bucket.

-
-
-

Using this cluster we will show how the incremental/merge approach taken by -cbbackupmgr reduces time and overhead on your cluster.

-
-
-

Configuring a Backup

-
-

Before getting started with cbbackupmgr we must first decide the directory where -to store all of our backups. This directory is referred to as the backup -archive. The backup archive contains one or more backup repositories. These -backup repositories are where your backups will be contained. The easiest way to -think of a backup repository is that it corresponds directly to a single cluster -that you want to back up. The backup repository also contains a configuration -for how to back that cluster up. A backup repository is created by using the -config subcommand. In this tutorial we will use a backup archive located at -/data/backup. The backup archive is automatically created if the directory -specified is empty. Below is an example of how to create a backup repository -called "cluster" which will backup all data and index definitions from all -buckets in the target cluster.

-
-
-
-
$ cbbackupmgr config -a /data/backup -r cluster
-
-Backup repository `cluster` created successfully in archive `/data/backup`
-
-
-
-

One of the most important aspects of backup repository creation is that we can -configure that backup repository in many different ways to change the way -backups in each backup repository are taken. Let’s say we want a separate backup -of only the index definitions in the travel-sample bucket. To do this we can -create a separate backup repository called "single".

-
-
-
-
$ cbbackupmgr config -a /data/backup -r single \
-  --include-data travel-sample --disable-data
-
-Backup repository `single` created successfully in archive `/data/backup`
-
-
-
-

The config subcommand provides many options in order to customize how you backup -your data. See the cbbackupmgr-config(1) page for more information -about what options are available and how they are used.

-
-
-
-

Backing up a Cluster

-
-

Now that we have created some backup repositories we should take a look at our -backup archive to see what it looks like. The easiest way to do this is to use -the info subcommand. This subcommand is used to retrieve information form a -backup archive. It will return the size as well as information about the -repositories, backups and item counts.

-
-
-
-
$ cbbackupmgr info -a /data/backup --all
-Name         | UUID                                  | Size  | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3  | 0B    | 2        |
-*  Name     | Size  | # Backups  |
-*  cluster  | 0B    | 0          |
-
-
-
-

The info subcommand gives us a directory print out of all of the backup -repositories and backups in the backup archive. Since there are no backups yet -we can just see our archives information in the output of this command. There is -also information about how much disk space each repository contains. More -information about the info subcommand can be found in the -cbbackupmgr-info(1) page.

-
-
-

Now that we have our backup repositories configured it’s time to start taking -backups. Since the backup repository contains all of the configuration -information for how the backup should be taken we just need to specify the -backup repository name and the information for the target cluster we intend to -back up. Below is an example of how to take a backup on the "cluster" backup -repository. We will assume that we have our cluster running on localhost.

-
-
-
-
$ cbbackupmgr backup -c 127.0.0.1 -u Administrator -p password -a /data/backup -r cluster
-Backing up to 2020-03-25T08_08_11.770436Z
-Copied all data in 33.02s (Avg. 759.44KB/Sec)  38894 items / 24.47MB
-beer-sample             [===================================] 100.00%
-travel-sample           [===================================] 100.00%
-
-Backup successfully completed
-Backed up bucket "beer-sample" succeeded
-Mutations backed up: 7303, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-Backed up bucket "travel-sample" succeeded
-Mutations backed up: 31591, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-
-
-
-

When the backup command is executed it will by default print out a progress bar -which is helpful for understanding how long your backup will take to complete -and the rate of data movement. While the backup is running the progress bar will -give an estimated time to completion, but this will change to average backup -rate when the backup finishes. Information is also provided on the total data -and items already backed up and the current rate of data movement. If the backup -completes successfully you will see the "Backup completed successfully" message -and a break down of mutations and deletions backed up per bucket.

-
-
-

Let’s also run the backup on the "single" backup repository to see how the two -backup runs differ.

-
-
-
-
$ cbbackupmgr backup -a /data/backup -r single \
-  -c couchbase://127.0.0.1 -u Administrator -p password
-
-Backing up to 2020-03-25T08_08_58.903046Z
-Copied all data in 1s (Avg. 480B/Sec)                 0 items / 480B
-travel-sample           [==================================] 100.00%
-Backup successfully completed
-Backed up bucket "travel-sample" succeeded
-Mutations backed up: 0, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-
-
-
-

Since the "single" backup repository is only configured to back up index -definitions for the travel-sample bucket we do not see a progress bar for the -beer-sample bucket. We can also see that the backup executed quicker since -there was much less data to actually back up.

-
-
-

Since we now have backups in our backup archive let’s take a look at the state -of our backup archive has changed by using the info subcommand.

-
-
-
-
$ cbbackupmgr info -a /data/backup --all
-Name         | UUID                                  | Size     | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3  | 54.35MB  | 2        |
-
-*  Name     | Size     | # Backups  |
-*  cluster  | 54.33MB  | 1          |
-
-+    Backup                       | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+    2020-03-25T08_08_11.770436Z  | 54.33MB  | FULL  | http://localhost:9000  | ffa8024f899ca6acc7c59cf0f541dbdd  | N/A    | 0       | 0        | true      |
-
--      Bucket       | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample  | 18.43MB  | 7303   | 7303       | 0           | 1      | 0    | 1        | 0     |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 35.91MB  | 31591  | 31591      | 0           | 0      | 0    | 10       | 0     |
-
-*  Name    | Size     | # Backups  |
-*  single  | 16.20KB  | 1          |
-
-+    Backup                       | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+    2020-03-25T08_08_58.903046Z  | 16.20KB  | FULL  | http://localhost:9000  | ffa8024f899ca6acc7c59cf0f541dbdd  | N/A    | 0       | 0        | true      |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 16.20KB  | 0      | 0          | 0           | 0      | 0    | 10       | 0     |
-
-
-
-

Now that we have some backups defined the output of the info subcommand is much -more useful. We can see that our "cluster" backup repository contains one backup -with a name corresponding to the time the backup was taken. That backup also -contains two buckets and we can see the number of views, indexes, mutations, -tombstones and more in the bucket. The "single" backup repository also contains -one backup, but this backup only contains the travel-sample bucket and contains -0 data items but we can see it has 10 indexes. Note that the info command also -supports JSON output if that is preferable. See cbbackupmgr-info(1) page.

-
-
-

One of the most important features of cbbackupmgr is that it is an -incremental-only backup utility. This means that once we have backed up some -data we will never need to back it up again. In order to simulate some changes -on the cluster we can run the 02-modify.sh script from the backup-tutorial -github repository mentioned at the beginning of the tutorial. If you do not have -this script then you will need to modify two documents and add two new documents -to the travel-sample bucket. After we have modified some data we will run the -backup subcommand on the "cluster" backup repository again.

-
-
-
-
$ cbbackupmgr backup -a /data/backup -r cluster -c couchbase://127.0.0.1 -u Administrator -p password
-
-Backing up to 2020-03-25T08_41_21.461311Z
-Copied all data in 3s (Avg. 18.98KB/Sec)           4 items / 56.95KB
-travel-sample           [==================================] 100.00%
-beer-sample             [==================================] 100.00%
-
-Backup successfully completed
-Backed up bucket "beer-sample" succeeded
-Mutations backed up: 0, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-Backed up bucket "travel-sample" succeeded
-Mutations backed up: 4, Mutations failed to backup: 0
-Deletions backed up: 0, Deletions failed to backup: 0
-
-
-
-

In this backup notice that since we updated 2 items and created two items that -this is all that we need back up during this run. If we list the backup archive -using the info subcommand then we will see that the backup archive looks like -something like what is below. We can see that our incremental contains 4 items -all of which are mutations. Those correspond to the 2 modifications and the 2 -additions.

-
-
-
-
$ cbbackupmgr info -a /data/backup --all
-Name         | UUID                                  | Size     | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3  | 86.37MB  | 2        |
-
-*  Name     | Size     | # Backups  |
-*  cluster  | 86.35MB  | 2          |
-
-+    Backup                       | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+    2020-03-25T08_08_11.770436Z  | 54.33MB  | FULL  | http://localhost:9000  | ffa8024f899ca6acc7c59cf0f541dbdd  | N/A    | 0       | 0        | true      |
-
--      Bucket       | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample  | 18.43MB  | 7303   | 7303       | 0           | 1      | 0    | 1        | 0     |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 35.91MB  | 31591  | 31591      | 0           | 0      | 0    | 10       | 0     |
-
-+    Backup                       | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+    2020-03-25T08_41_21.461311Z  | 32.02MB  | INCR  | http://localhost:9000  | ffa8024f899ca6acc7c59cf0f541dbdd  | N/A    | 0       | 0        | true      |
-
--      Bucket       | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample  | 16.00MB  | 0      | 0          | 0           | 1      | 0    | 1        | 0     |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 16.02MB  | 4      | 4          | 0           | 0      | 0    | 10       | 0     |
-
-*  Name    | Size     | # Backups  |
-*  single  | 16.20KB  | 1          |
-
-+    Backup                       | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+    2020-03-25T08_08_58.903046Z  | 16.20KB  | FULL  | http://localhost:9000  | ffa8024f899ca6acc7c59cf0f541dbdd  | N/A    | 0       | 0        | true      |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 16.20KB  | 0      | 0          | 0           | 0      | 0    | 10       | 0     |
-
-
-
-

The backup subcommand provides many options in order to customize how you backup -your data. See the cbbackupmgr-backup(1) page for more information -about what options are available and how they are used.

-
-
-
-

Restoring a Backup

-
-

Now that we have some backup data let’s restore that data backup to the cluster. -In order to restore data we just need to know the name of the backup that we -want to restore. To find the name we can again use the info subcommand in order -to see what is in our backup archive. The backup name will always be a -timestamp. For example, let’s say we want to restore the -2016-03-22T10_26_08.933579821-07_00 from the "cluster" backup repository. In -order to do this we run the command below.

-
-
-
-
$ cbbackupmgr restore -a /data/backup -r cluster \
- -c http://127.0.0.1:8091 -u Administrator -p password \
- --start 2016-03-22T14_00_16.892277632-07_00 \
- --end 2016-03-22T14_00_16.892277632-07_00 --force-updates
-
-(1/1) Restoring backup 2016-03-22T14_00_16.892277632-07_00
-Copied all data in 2s (Avg. 19.96MB/Sec)       38894 items / 39.91MB
-travel-sample           [==================================] 100.00%
-beer-sample             [==================================] 100.00%
-
-Restore completed successfully
-
-
-
-

In the command above we use the --start and --end flags to specify the range of -backups we want to restore. Since we are only restoring one backup we specify -the same value for both --start and --end. We also added the --force-updates -flag in order to skip Couchbase conflict resolution. This tells cbbackupmgr to -force overwrite key-value pairs being restored even if the key-value pair on the -cluster is newer and the one being restored. If we look at the two values that -we updated on the cluster we will now see that they have been reverted back to -what they were at the time we took the initial backup. If you used the script in -the backup-tutorial github repository to update documents then you can use the -03-inspect.sh script to see the state of the updated documents after the -restore.

-
-
-

The restore subcommand also allows for you to exclude data that was backed up -from the restore and provides various other options. See the -cbbackupmgr-restore(1) page for more information on restoring data.

-
-
-
-

Merging backups

-
-

Using an incremental backup solution means that each backup we take increases -disk space. Since disk space in not infinite we need to be able to reclaim this -disk space. In order to do this we use the cbbackupmgr-merge(1) -subcommand to merge two or more backups together. Since we have two backups in -the "cluster" backup repository we will merge these backups together using the -command below.

-
-
-
-
$ cbbackupmgr merge -a /data/backup -r cluster \
-  --start oldest --end latest
-
-Merge completed successfully
-
-
-
-

After merging the backups together we can use the list subcommand to see the -effect the merge subcommand we just ran had on the backup archive.

-
-
-
-
$ cbackupmgr info -a /data/backup --all                                                                                                                                                                                                                                                                                                                                                                                     20-03-25 - 8:52:36
-Name         | UUID                                  | Size     | # Repos  |
-backup_repo  | 32c97d5f-821a-4284-840b-9ee7cf8733a3  | 54.35MB  | 2        |
-
-*  Name     | Size     | # Backups  |
-*  cluster  | 54.33MB  | 1          |
-
-+    Backup                       | Size     | Type          | Source  | Cluster UUID                      | Range                                      | Events  | Aliases  | Complete  |
-+    2020-03-25T08_41_21.461311Z  | 54.33MB  | MERGE - FULL  | Merge   | ffa8024f899ca6acc7c59cf0f541dbdd  | 2020-03-25T08_08_11-2020-03-25T08_41_21:2  | 0       | 0        | true      |
-
--      Bucket       | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      beer-sample  | 18.43MB  | 7303   | 7303       | 0           | 1      | 0    | 1        | 0     |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 35.91MB  | 31593  | 31593      | 0           | 0      | 0    | 10       | 0     |
-
-*  Name    | Size     | # Backups  |
-*  single  | 16.20KB  | 1          |
-
-+    Backup                       | Size     | Type  | Source                 | Cluster UUID                      | Range  | Events  | Aliases  | Complete  |
-+    2020-03-25T08_08_58.903046Z  | 16.20KB  | FULL  | http://localhost:9000  | ffa8024f899ca6acc7c59cf0f541dbdd  | N/A    | 0       | 0        | true      |
-
--      Bucket         | Size     | Items  | Mutations  | Tombstones  | Views  | FTS  | Indexes  | CBAS  |
--      travel-sample  | 16.20KB  | 0      | 0          | 0           | 0      | 0    | 10       | 0     |
-
-
-
-

We can see from the list command that there is now a single backup in the -"cluster" backup repository. This backup has a name that reflects the name of -the most recent backup in the merge. It also has 31593 data items in the -travel-sample bucket. This is two more items than the original backup we took -because the second backup had two new items. The two items that were updated -were deduplicated during the merge so they do not add extra items to the count -displayed by the info subcommand.

-
-
-

For more information on how the merge command works as well as information on -other ways the merge command can be used see the cbbackupmgr-merge(1) -page.

-
-
-
-

Removing a Backup Repository

-
-

If no longer need a backup repository then we can use the remove subcommand to -remove the backup repository. Below is an example showing how to remove the -"cluster" backup repository.

-
-
-
-
$ cbbackupmgr remove -a /data/backup -r cluster
-
-Backup repository `cluster` deleted successfully from archive `/data/backup`
-
-
-
-

If we now run the list subcommand you will see that the "cluster" backup -repository no longer exists. For more information on the remove subcommand see -the cbbackupmgr-remove(1) page.

-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-backup(1), -cbbackupmgr-config(1), -cbbackupmgr-info(1), -cbbackupmgr-merge(1), -cbbackupmgr-remove(1), -cbbackupmgr-restore(1), -cbbackupmgr-strategies(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr.html b/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr.html deleted file mode 100644 index ad5c7511..00000000 --- a/scripts/cbtools/share/doc/cbbackupmgr/cbbackupmgr.html +++ /dev/null @@ -1,790 +0,0 @@ - - - - - - - - - -cbbackupmgr(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbbackupmgr [--version] [--help] <command> [<args>]
-
-
-
-
-

DESCRIPTION

-
-
-

cbbackupmgr is a high performance backup and restore client for Couchbase -Server.

-
-
-

See cbbackupmgr-tutorial(7) to get started. For more information on -how specific commands work you can run "cbbackupmgr <command> --help". For -backup strategies see cbbackupmgr-strategies(7).

-
-
-
-
-

OPTIONS

-
-
-
-
--version
-
-

Prints the cbbackupmgr suite version that the cbbackupmgr program came from.

-
-
--help
-
-

Prints the synopsis and a list of commands. If a cbbackupmgr command is -named, this option will bring up the manual page for that command.

-
-
-
-
-
-
-

CBBACKUPMGR COMMANDS

-
-
-
-
cbbackupmgr-backup(1)
-
-

Backs up data from a Couchbase cluster.

-
-
cbbackupmgr-collect-logs(1)
-
-

Collects diagnostic information.

-
-
cbbackupmgr-compact(1)
-
-

Compacts a backup.

-
-
cbbackupmgr-config(1)
-
-

Creates a new backup repository.

-
-
cbbackupmgr-info(1)
-
-

Command that displays information about the backups in the archive.

-
-
cbbackupmgr-merge(1)
-
-

Merges backups together.

-
-
cbbackupmgr-remove(1)
-
-

Deletes a backup repository.

-
-
cbbackupmgr-restore(1)
-
-

Restores a backup from the archive.

-
-
-
-
-
-
-

IDENTIFIER TERMINOLOGY

-
-
-
-
<archive>
-
-

The root directory containing multiple backup repositories. This is the -top-level backup directory and contains all backup data as well as backup -logs.

-
-
<repository>
-
-

Contains a backup configuration used for taking actual backups. A repository -should be created for a specific Couchbase cluster and it will contain -multiple incremental backups.

-
-
<backup>
-
-

A backup of a Couchbase cluster at a given point in time. All backups are -incremental backups.

-
-
<bucket>
-
-

A backup may consist of one or more buckets. Each bucket is stored separately.

-
-
<collection_string>
-
-

A dot separated string which indicates the location of documents on a -Couchbase cluster. Collection strings are expected in the format -'bucket.scope.collection'. The collection string 'food.produce.fruit' refers -to the 'fruit' collection inside the 'produce' scope which is contained -in the 'food' bucket. In the case where the bucket name contains a '.' -character you should escape the '.' character. e.g. 'fo\.od.produce.fruit' -is a valid collection string. You are also free to omit the scope/collection -portion of the collection string; in this case the absence of a scope/collection -is treated as a wildcard indicating all collections inside a scope or all scopes -inside a bucket. For example the collection string 'food.produce' indicates that -we are that we are talking about all of the collections that fall inside of the -'produce' scope inside of the 'food' bucket.

-
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_ARCHIVE_PATH
-
-

Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden.

-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
-
-
-
-
-

DISCUSSION

-
-
-

The cbbackupmgr command is used for backing up Couchbase clusters, managing -those backups, and restoring them.

-
-
-

The cbbackupmgr command was built around the concept of taking only incremental -backups. This concept is important because as the data in a cluster grows it -becomes increasingly difficult to take full backups in a reasonable amount of -time. By taking incremental backups we are able to reduce the time it takes to -backup a cluster by ensuring that we transfer the smallest amount of data -possible each time we back the cluster up.

-
-
-

A consequence of taking incremental backups is that we must know about the -previous backups that we have taken in order to know where we left off. This -means that the cbbackupmgr command must manage the backups it has taken. The -cbbackupmgr command does this by using the concept of a backup archive and a -backup repository. A backup repository is a directory that contains a backup -configuration for backing up a specific cluster. Normally there will be one -backup repository per Couchbase cluster. Each time you want to back up this -cluster you will specify this backup repository with the cbbackupmgr-backup -command and the backup tool will automatically find where the last backup -finished and incrementally backup new data in that cluster.

-
-
-

The backup archive is the top-level directory and contains one or more backup -repositories and a logs folder. Logging for all backup repositories is -contained in the logs folder in the backup.log file.

-
-
-

In an incremental approach the amount of data being stored in the backup archive -is always increasing. To handle this issue the backup command allows backups to -be merged together. This allows data to be deduplicated resulting in a single -backup that takes up less disk space than the multiple previous backups. More -information about how to take advantage of incremental backups and merges is -contained in cbbackupmgr-strategies(7).

-
-
-

The minimum hardware requirement for running cbbackupmgr is four CPU cores, -8GiB RAM. The recommend hardware is sixteen CPU cores, 16GiB RAM and SSD disks.

-
-
-
-
-

OPERATIONS DURING MAJOR CLUSTER CONFIGURATION CHANGES

-
-
-

Operations (commands or sub-commands) which connect to a cluster are not -supported during major cluster configuration changes.

-
-
-

For example, performing an import/export, making a backup or performing a -restore whilst changing the TLS configuration/security settings is unsupported.

-
-
-

These types of changes (e.g. changing the TLS mode to strict) are not expected -to be time consuming so it’s generally expected that operations should be -started after completing the configuration change.

-
-
-

Please note that this does not include rebalances; operations may be performed -during a rebalance. The reason for this distinction, is that major cluster -configuration changes are generally quick, whilst rebalances for large data -sets may be time consuming.

-
-
-
-
-

FURTHER DOCUMENTATION

-
-
-

A tutorial for getting started with the backup command is also available in -cbbackupmgr-tutorial(7).

-
-
-

A guide for production backup strategies is available in -cbbackupmgr-strategies(7).

-
-
-
-
-

AUTHORS

-
-
-

cbbackupmgr is a Couchbase Enterprise Edition tool and was written by Couchbase.

-
-
-
-
-

REPORTING BUGS

-
-
-

Report urgent issues to the Couchbase Support Team at support@couchbase.com. -Bugs can be reported to the Couchbase Jira Bug Tracker at -http://www.couchbase.com/issues.

-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-tutorial(7), -cbbackupmgr-config(7), -cbbackupmgr-backup(7), -cbbackupmgr-restore(7)

-
-
-
-
-

CBBACKUPMGR

-
-
-

Part of the cbbackupmgr(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbdatarecovery/cbdatarecovery.html b/scripts/cbtools/share/doc/cbdatarecovery/cbdatarecovery.html deleted file mode 100644 index e7dfb911..00000000 --- a/scripts/cbtools/share/doc/cbdatarecovery/cbdatarecovery.html +++ /dev/null @@ -1,1375 +0,0 @@ - - - - - - - - -cbdatarecovery(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbdatarecovery [--cluster <connection_string>] [--cacert <path>]
-               [--username <username>] [--password <password>]
-               [--client-cert <path>] [--client-cert-password <password>]
-               [--client-key <path>] [--client-key-password <password>]
-               [--data-path <path>] [--map-data <mappings>]
-               [--vbucket-filter <filter>] [--filter-keys <regex>]
-               [--filter-values <regex>] [--replace-ttl <type>]
-               [--replace-ttl-with <timestamp>]
-               [--include-data <collection_strings>]
-               [--exclude-data <collection_strings>]
-               [--vbucket-state <state>] [--log-file <path>]
-               [--threads <threads>] [--force-updates]
-               [--create-missing-collections] [--no-ssl-verify]
-               [--verbose] [--no-progress-bar]
-
-
-
-
-

DESCRIPTION

-
-
-

cbdatarecovery transfers key value data stored in Couchstore files from a -Couchbase Server data directory into an active cluster. This can be used to -recover data from offline/failed over nodes.

-
-
-
-
-

OPTIONS

-
-
-

Required

-
-
-
-c, --cluster <connection_string>
-
-

A connection string representing a Couchbase node/cluster which will be the -destination for the recovered data. See the HOST FORMATS section for more -information.

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
-d, --data-path <path>
-
-

The path to a Couchbase Server data directory. For example -/opt/couchbase/lib/couchbase/data.

-
-
-
-
-
-

Optional

-
-
-
--cacert <path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag -must be specified when using an SSL encrypted connection.

-
-
--map-data <mappings>
-
-

Specified when you want to transfer source data into a different location. -For example this argument may be used to remap buckets/scopes/collections -with the restriction that they must be remapped at the same level. For -example a bucket may only be remapped to a bucket, a scope to a scope and a -collection to a collection. The argument expects a comma separated list of -collection string mappings e.g. -bucket1=bucket2,bucket3.scope1=bucket3.scope2,bucket4.scope.collection1=bucket4.scope.collection2

-
-
--vbucket-filter <filter>
-
-

Specifies a list of vBuckets that should be transferred. vBuckets are -specified as a comma separated list of integers/ranges. For example 0,1,2 -and 0-2 represent the same subset of vBuckets. If no filter is provided, -then all vBuckets will be transferred.

-
-
--filter-keys <regex>
-
-

Only transfer data where the key matches a particular regular expression. -The regular expressions provided must follow -RE2 syntax.

-
-
--filter-values <regex>
-
-

Only transfer data where the value matches a particular regular expression. -The regular expressions provided must follow -RE2 syntax.

-
-
--replace-ttl <type>
-
-

Sets a new expiration (time-to-live) value for the specified keys. This -parameter can either be set to "none", "all" or "expired" and should be used -along with the --replace-ttl-with flag. If "none" is supplied then the TTL -values are not changed. If "all" is specified then the TTL values for all -keys are replaced with the value of the --replace-ttl-with flag. If -"expired" is set then only keys which have already expired will have the -TTL’s replaced.

-
-
--replace-ttl-with <timestamp>
-
-

Updates the expiration for the keys specified by the --replace-ttl parameter. -The parameter has to be set when --replace-ttl is set to "all". There are two -options, RFC3339 time stamp format (2006-01-02T15:04:05-07:00) or "0". When -"0" is specified the expiration will be removed. Please note that the RFC3339 -value is converted to a Unix time stamp on the cbbackupmgr client. It is -important that the time on both the client and the Couchbase Server are the -same to ensure expiry happens correctly.

-
-
--include-data <collection_strings>
-
-

Only transfer data included in this comma list of collection strings. Note -that this flag can’t be specified at the same time --exclude-data.

-
-
--exclude-data <collection_strings
-
-

Don’t transfer the data for the buckets/scopes/collections in this comma -separated list of collection strings. Note that this flag can’t be specified -at the same time as --include-data.

-
-
--vbucket-state <state>
-
-

Only transfer vBuckets which are in the provided stated. Accepts the values -active, replica or dead.

-
-
--threads <threads>
-
-

Specifies the number of concurrent clients to use when transferring data. -Fewer clients means restores will take longer, but there will be less cluster -resources used to complete the restore. More clients means faster restores, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the restore is -taking place.

-
-
--force-updates
-
-

Forces data in the Couchbase cluster to be overwritten even if the data in -the cluster is newer. By default updates are not forced and all updates use -Couchbase’s conflict resolution mechanism to ensure that if newer data exists -on the cluster that is not overwritten by older restore data.

-
-
--create-missing-collections
-
-

Automatically create any scopes/collections which exist in the data directory -on disk but not in the remote cluster. This behavior is disabled by default.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man-in-the-middle attack if -you use this flag. Either this flag or the --cacert flag must be specified -when using an SSL encrypted connection.

-
-
--verbose
-
-

Increase logging verbosity; useful when debugging. Disabled by default.

-
-
--no-progress-bar
-
-

By default, a progress bar is printed to stdout so that the user can see how -long the transfer is expected to take, the amount of data that is being -transferred per second, and the amount of data that has been transferred. -Specifying this flag disables the progress bar and is useful when running -automated jobs.

-
-
-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

AUTOMATIC COLLECTION CREATION

-
-
-

By design, users may not recreate the _default collection once it has been -deleted. Therefore, this means that the _default collection can’t (and won’t) -be recreated if it’s missing. Before performing a transfer, a check will take -place to see if the _default collection will be required when it’s missing. -If this is the case, the command will exit early and you will be required to -remap the _default collection using the --map-data flag.

-
-
-
-
-

REMAPPING

-
-
-

During a transfer, scopes/collections can be remapped from one location to -another. There are several rules that are enforced when remapping -scopes/collections, they are as follows:

-
-
-
    -
  • -

    You must be running an Enterprise Edition version of Couchbase Server.

    -
    • -
  • -

    You may not remap the _default scope (discussed in THE DEFAULT SCOPE).

    -
    • -
  • -

    You may only remap scopes/collections at the same level meaning scopes may -be remapped to other scopes, and collections to other collections, however, a -scope can’t be remapped to a collection or vice versa.

    -
    • -
  • -

    Scopes/collections may only be remapped within the same bucket. For example -the mapping bucket1.scope.collection=bucket2.scope.collection is invalid.

    -
    • -
  • -

    Scopes/collections may only be remapped once. For example the mapping -bucket1.scope1=bucket1.scope2,bucket1.scope1=bucket1.scope3 is invalid.

    -
    • -
  • -

    Remapping may only take place at one level at once meaning that if a parent -bucket/scope is already remapped, the child scopes/collections may not also -be remapped. For example the mapping -bucket1.scope1=bucket1.scope2,bucket1.scope1.collection1=bucket1.scope3.collection9 -is invalid.

    -
    • -
-
-
-

REMAPPING A SCOPE/COLLECTION WITHOUT RENAMING

-
-

During a transfer, it’s possible for a scope/collection to encounter a conflict -(for example, because it has been recreated). It may not be preferable to -rename the scope/collection during the transfer.

-
-
-

For this reason, the --map-data flag, allows you to remap a scope/collection -to itself; this indicates that the scope/collection that exists in the target -(with a different id) should be treated as the same.

-
-
-

As an example, the following error message indicates that a collection has been -recreated prior to a restore.

-
-
-
-
Error restoring cluster: collection 8 with name 'collection1' in the scope '_default' exists with a different name/id on the cluster, a manual remap is required
-
-
-
-

Using the --map-data flag with the argument -bucket._default.collection1=bucket._default.collection1 would cause -cbbackupmgr to treat collection1 (with id 8) as collection1 (with the id -it exists with in the target).

-
-
-
-

THE DEFAULT SCOPE

-
-

As mentioned in AUTOMATIC COLLECTION CREATION, it’s not possible to recreate -the _default scope/collection. This means you can’t remap the _default -scope because the tool may be unable to create a destination scope/collection. -This may be worked around by remapping each collection inside the _default -scope.

-
-
-
-

BUCKET TO COLLECTION REMAPPING

-
-

As discussed in REMAPPING, it’s not possible to remap data at different levels; -buckets must be remapped to buckets, scopes to scopes and collections to -collections. However, there is one supported edge case, which is remapping a -bucket into a collection to allow migration from a collection unaware to -collection aware datasets.

-
-
-

To remap a bucket into a collection using --map-data you may supply ---map-data bucket._default._default=bucket.scope.collection. This -functionality is compatible with cross bucket mapping, for example you may also -supply --map-data bucket1._default._default=bucket2.scope.collection.

-
-
-

Note that once you’ve provided a mapping to remap a bucket into a collection -you may not remap that bucket elsewhere. For example --map-data -bucket1._default._default=bucket2.scope.collection,bucket1=bucket3 is invalid.

-
-
-
-

REMAPPING MULTIPLE DATA SOURCES INTO A SINGLE TARGET SOURCE

-
-

As outlined in the rules discussed in REMAPPING, it’s not possible to remap a -bucket/scope/collection multiple times, however, it is possible to remap to a -single destination multiple times. For example the mapping -bucket1=dest,bucket2=dest,bucket3=dest is valid.

-
-
-

Although valid, this manor of remapping is dangerous and can result in data not -being transferred due to conflicting key spaces. If this style of remapping is -detected a warning will be printed before proceeding.

-
-
-
-
-
-

OPERATIONS DURING MAJOR CLUSTER CONFIGURATION CHANGES

-
-
-

Operations (commands or sub-commands) which connect to a cluster are not -supported during major cluster configuration changes.

-
-
-

For example, performing an import/export, making a backup or performing a -restore whilst changing the TLS configuration/security settings is unsupported.

-
-
-

These types of changes (e.g. changing the TLS mode to strict) are not expected -to be time consuming so it’s generally expected that operations should be -started after completing the configuration change.

-
-
-

Please note that this does not include rebalances; operations may be performed -during a rebalance. The reason for this distinction, is that major cluster -configuration changes are generally quick, whilst rebalances for large data -sets may be time consuming.

-
-
-
-
-

EXAMPLES

-
-
-

The cbdatarecovery tool may be used to recover key value data from a -Couchbase Server data directory. By default, all the active vBuckets for all -the detected buckets will be transferred.

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data
-Recovering to '172.20.1.1'
-Copied all data in 505ms (Avg. 11.06MB/Sec)                                                                               17722 items / 11.06MB
-[=====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 11.06MB           | Tue, 23 Feb 2021 11:13:51 +0000/s | Tue, 23 Feb 2021 11:13:52 +0000 | 535ms    |
-
-| Bucket
-| ------
-| Name        | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| beer-sample | Succeeded | 740.46KB    | 740.46KB/s        | Tue, 23 Feb 2021 11:13:51 +0000 | Tue, 23 Feb 2021 11:13:51 +0000 | 21ms     |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 1836     | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-| Bucket
-| ------
-| Name           | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| gamesim-sample | Succeeded | 29.82KB     | 29.82KB/s         | Tue, 23 Feb 2021 11:13:51 +0000 | Tue, 23 Feb 2021 11:13:51 +0000 | 3ms      |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 137      | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.31MB     | 10.31MB/s         | Tue, 23 Feb 2021 11:13:51 +0000 | Tue, 23 Feb 2021 11:13:52 +0000 | 222ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15749    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-

To recover only a subset of the available buckets, the --include-data and ---exclude-flags may be used. Note that these flags are mutually exclusive -meaning they can’t be used at the same time.

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data --include-data travel-sample
-Recovering to '172.20.1.1'
-Copied all data in 273ms (Avg. 10.31MB/Sec)                                                                              15749 items / 10.31MB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 10.31MB           | Tue, 23 Feb 2021 11:14:14 +0000/s | Tue, 23 Feb 2021 11:14:15 +0000 | 300ms    |
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.31MB     | 10.31MB/s         | Tue, 23 Feb 2021 11:14:14 +0000 | Tue, 23 Feb 2021 11:14:15 +0000 | 189ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15749    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data --exclude-data travel-sample
-Recovering to '172.20.1.1'
-Copied all data in 202ms (Avg. 770.28KB/Sec)                                                                              1973 items / 770.28KB
-[=====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 770.28KB          | Tue, 23 Feb 2021 11:26:43 +0000/s | Tue, 23 Feb 2021 11:26:43 +0000 | 227ms
-
-| Bucket
-| ------
-| Name        | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| beer-sample | Succeeded | 740.46KB    | 740.46KB/s        | Tue, 23 Feb 2021 11:26:43 +0000 | Tue, 23 Feb 2021 11:26:43 +0000 | 26ms     |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 1836     | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-| Bucket
-| ------
-| Name           | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| gamesim-sample | Succeeded | 29.82KB     | 29.82KB/s         | Tue, 23 Feb 2021 11:26:43 +0000 | Tue, 23 Feb 2021 11:26:43 +0000 | 4ms      |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 137      | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-

To recover only a subset of the available vBuckets, the --vbucket-filter flag -may be used. Note that by default all the available vBuckets will be recovered.

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data --vbucket-filter 0-128
-Recovering to '172.20.1.1'
-Copied all data in 199ms (Avg. 5.31MB/Sec)                                                                                 7971 items / 5.31MB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 5.31MB            | Tue, 23 Feb 2021 11:28:05 +0000/s | Tue, 23 Feb 2021 11:28:05 +0000 | 227ms
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 5.31MB      | 5.31MB/s          | Tue, 23 Feb 2021 11:28:05 +0000 | Tue, 23 Feb 2021 11:28:05 +0000 | 114ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 7971     | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-

When recovering data to a different cluster or a cluster which has changed over -time, it’s possible for some scopes/collections to have been dropped. This may -cause the recovery to fail because the required scopes/collections no longer -exist. The --auto-create-collections flag can be used to automatically create -any missing scopes/collections.

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data --auto-create-collections
-Recovering to '172.20.1.1'
-Copied all data in 534ms (Avg. 11.06MB/Sec)                                                                               17722 items / 11.06MB
-[=====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 11.06MB           | Tue, 23 Feb 2021 11:15:19 +0000/s | Tue, 23 Feb 2021 11:15:20 +0000 | 563ms    |
-
-| Bucket
-| ------
-| Name        | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| beer-sample | Succeeded | 740.46KB    | 740.46KB/s        | Tue, 23 Feb 2021 11:15:20 +0000 | Tue, 23 Feb 2021 11:15:20 +0000 | 25ms     |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 1836     | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-| Bucket
-| ------
-| Name           | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| gamesim-sample | Succeeded | 29.82KB     | 29.82KB/s         | Tue, 23 Feb 2021 11:15:20 +0000 | Tue, 23 Feb 2021 11:15:20 +0000 | 3ms      |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 137      | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.31MB     | 10.31MB/s         | Tue, 23 Feb 2021 11:15:20 +0000 | Tue, 23 Feb 2021 11:15:20 +0000 | 200ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15749    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-

To recover only replica vBuckets, the --vbucket-state flag may be used. -This flag accepts either active or replica resulting in the recovery of -only the vBuckets in the provided state. Note that by default only active -vBuckets will be recovered.

-
-
-

In the examples below, note the difference in the number of items recovered -when recovering replica vBuckets; this is due to the way active/replica -vBuckets are distributed.

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data
-Recovering to '172.20.1.1'
-Copied all data in 279ms (Avg. 10.31MB/Sec)                                                                              15749 items / 10.31MB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 10.31MB           | Tue, 23 Feb 2021 11:17:52 +0000/s | Tue, 23 Feb 2021 11:17:53 +0000 | 306ms
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.31MB     | 10.31MB/s         | Tue, 23 Feb 2021 11:17:52 +0000 | Tue, 23 Feb 2021 11:17:53 +0000 | 194ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15749    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data --vbucket-state replica
-Recovering to '172.20.1.1'
-Copied all data in 188ms (Avg. 10.33MB/Sec)                                                                              15893 items / 10.33MB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 10.33MB           | Tue, 23 Feb 2021 11:17:55 +0000/s | Tue, 23 Feb 2021 11:17:55 +0000 | 213ms
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.33MB     | 10.33MB/s         | Tue, 23 Feb 2021 11:17:55 +0000 | Tue, 23 Feb 2021 11:17:55 +0000 | 101ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15893    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-

When recovering data to a cluster which has changed over time, it’s possible -for one or more target scopes/collections to have been recreated. The ---map-data flag can be used to recover data to a scope/collection whose id -has changed (but the name remains the same).

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data \
-    --map-data travel-sample.inventory=travel-sample.inventory
-Recovering to '172.20.1.1'
-Copied all data in 290ms (Avg. 10.31MB/Sec)                                                                              15749 items / 10.31MB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 10.31MB           | Tue, 23 Feb 2021 11:46:38 +0000/s | Tue, 23 Feb 2021 11:46:38 +0000 | 318ms
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.31MB     | 10.31MB/s         | Tue, 23 Feb 2021 11:46:38 +0000 | Tue, 23 Feb 2021 11:46:38 +0000 | 201ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15749    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-

The --map-data flag may also be used to remap data from one location to -another, for example to rename a scope/collection. Note the addition of the ---auto-create-collections flag because in this case the scope storage did -not exist.

-
-
-
-
$ cbdatarecovery -c 172.20.1.1 -u Administrator -p asdasd -d /opt/couchbase/lib/couchbase/data \
-    --map-data travel-sample.inventory=travel-sample.storage --auto-create-collections
-Recovering to '172.20.1.1'
-Copied all data in 401ms (Avg. 10.31MB/Sec)                                                                              15749 items / 10.31MB
-[====================================================================================================================================] 100.00%
-
-| Transfer
-| --------
-| Status    | Avg Transfer Rate | Started At                        | Finished At                     | Duration |
-| Succeeded | 10.31MB           | Tue, 23 Feb 2021 11:20:23 +0000/s | Tue, 23 Feb 2021 11:20:23 +0000 | 429ms
-
-| Bucket
-| ------
-| Name          | Status    | Transferred | Avg Transfer Rate | Started At                      | Finished At                     | Duration |
-| travel-sample | Succeeded | 10.31MB     | 10.31MB/s         | Tue, 23 Feb 2021 11:20:23 +0000 | Tue, 23 Feb 2021 11:20:23 +0000 | 227ms    |
-|
-| Mutations                    | Deletions                    | Expirations                  |
-| ---------                    | ---------                    | -----------                  |
-| Received | Errored | Skipped | Received | Errored | Skipped | Received | Errored | Skipped |
-| 15749    | 0       | 0       | 0        | 0       | 0       | 0        | 0       | 0       |
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbdocloader/cbdocloader.html b/scripts/cbtools/share/doc/cbdocloader/cbdocloader.html deleted file mode 100644 index daa196b9..00000000 --- a/scripts/cbtools/share/doc/cbdocloader/cbdocloader.html +++ /dev/null @@ -1,953 +0,0 @@ - - - - - - - - -cbdocloader(1) - - - - - -
-
-

Description

-
-
-

cbdocloader has been deprecated, please use cbimport with the ---format sample flag instead.

-
-
-

The cbdocloader tool loads Couchbase sample datasets into Couchbase Server. -Sample datasets are zip files, provided by Couchbase, which contain documents and index definitions. -These datasets are intended to allow users to explore Couchbase features prior to loading their own datasets.

-
-
-

For Linux, the tool’s location is /opt/couchbase/bin/cbdocloader; for Windows, C:\Program Files\Couchbase\Server\bin\cbdocloader; and for Mac OS X, /Applications/Couchbase Server.app/Contents/Resources/couchbase-core/bin/cbdocloader.

-
-
-
-
-

Syntax

-
-
-

The syntax for cbdocloader is as follows:

-
-
-
-
cbdocloader --cluster [hostname] --username [username]
---p [password] --bucket [bucket-name] --bucket-quota [quota]
---dataset [path] --thread [number] --verbose
-
-
-
-
-
-

Options

-
-
-

Command options are as follows:

-
- ---- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
OptionDescription

-h --help

Show this help message and exit.

-c [hostname], --cluster [hostname]

The hostname of one of the nodes in the cluster to load data into. -See the Host Formats section below for hostname specification details.

-u [username], --user [username]

The username for cluster authentication. -The user must have the appropriate privileges to create a bucket, write data, and create view, secondary index and full-text index definitions. -This can be specified by means of the environment variable CB_REST_USERNAME.

-p [password], --password [password]

The password for cluster authentication. -The user must have the appropriate privileges to create a bucket, write data, and create view, secondary index and full-text index definitions. -This can be specified by means of the environment variable CB_REST_PASSWORD.

--client-cert [path]

The path to a client certificate. -The path to a client certificate used to authenticate when connecting to a cluster. May be supplied with --client-key -as an alternative to the --username and --password flags.

--client-key [path]

The path to the client private key. -The path to the client private key whose public key is contained in the certificate provided to the --client-cert -flag. May be supplied with --client-key as an alternative to the --username and --password flags.

-m [quota], --bucket-quota [quota]

The amount of memory to assign to the bucket. -If the bucket already exists, this parameter is ignored.

-d [path], --dataset [path]

The path of the dataset to be loaded. -The path can refer to either a zip file or a directory to load data from.

-t [number],--threads [number]

Specifies the number of concurrent clients to use when loading data. -Fewer clients means data loading will take longer, but with fewer cluster resources used. -More clients means faster data loading, but with higher cluster-resource usage. -This parameter defaults to 1 if it is not specified. -This parameter is recommended not to be set to be higher than the number of CPUs on the machine where the command is being run.

-v,--verbose

Prints log messages to stdout. -This flag is useful for debugging errors in the data loading process.

--no-ssl-verify

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the server -you connect to. You are vulnerable to a man-in-the-middle attack if you use -this flag. Either this flag or the --cacert flag must be specified when using -an SSL encrypted connection.

--cacert

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag must -be specified when using an SSL encrypted connection.

-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

OPERATIONS DURING MAJOR CLUSTER CONFIGURATION CHANGES

-
-
-

Operations (commands or sub-commands) which connect to a cluster are not -supported during major cluster configuration changes.

-
-
-

For example, performing an import/export, making a backup or performing a -restore whilst changing the TLS configuration/security settings is unsupported.

-
-
-

These types of changes (e.g. changing the TLS mode to strict) are not expected -to be time consuming so it’s generally expected that operations should be -started after completing the configuration change.

-
-
-

Please note that this does not include rebalances; operations may be performed -during a rebalance. The reason for this distinction, is that major cluster -configuration changes are generally quick, whilst rebalances for large data -sets may be time consuming.

-
-
-
-
-

Examples

-
-
-

To load the dataset travel-sample.zip, which is located in /opt/couchbase/samples/, into a bucket with a memory quota of 1024MB, run the following command.

-
-
-
-
cbdocloader -c couchbase://127.0.0.1 -u Administrator \
--p password -m 1024 -b travel-sample \
--d /opt/couchbase/samples/travel-sample.zip
-
-
-
-

To increase the parallelism of data-loading, use the threads option. -In the example below, 4 threads are specified.

-
-
-
-
cbdocloader -c couchbase://127.0.0.1 -u Administrator \
--p password -m 1024 -b travel-sample \
--d /opt/couchbase/samples/travel-sample.zip -t 4
-
-
-
-

The cbdocloader command can also be used to load data from a folder. -The folder must contain files that correspond to the samples format. -See the Sample Data Format section below, for more information. -The following example show to load data from folder /home/alice/census-sample:

-
-
-
-
cbdocloader -c couchbase://127.0.0.1 -u Administrator \
--p password -m 1024 -b census-sample -d /home/alice/census-sample
-
-
-
-
-
-

Sample Data Format

-
-
-

The cbdocloader command is used to load data from zip files or folders that correspond to the Couchbase sample data format. -This is exemplified as follows:

-
-
-
-
+ sample_folder
-  + design_docs
-    indexes.json
-    design_doc.json
-  + docs
-    document1.json
-    document2.json
-    document3.json
-    document4.json
-
-
-
-

The top level directory can be given any name and will always contain two folders. -The design_docs folder is where index definitions are kept. -This folder will contain zero or more JSON files that contain the various indexes that should be created when the sample dataset is loaded. -Global Secondary Indexes (GSI) should always be in a file named indexes.json. -Below is an example of the format for defining GSI indexes.

-
-
-
-
{
- "statements": [
-   {
-     "statement": "CREATE PRIMARY INDEX on `bucket1`",
-     "args": null
-   },
-   {
-     "statement": "CREATE INDEX by_type on `bucket1`(name) WHERE _type='User'"
-     "args": null
-   }
- ]
-}
-
-
-
-

GSI indexes are defined as a JSON document where each index definition is contained in a list called statements. -Each element in the list is an object that contains two keys. -The statement key contains the actual index definition, and the args key is used if the statement contains any positional arguments.

-
-
-

All other files in the design_docs folder are used to define view design documents, and each design document should be put -into a separate file. -These files can be named anything, but should always have the .json file extension. -Below is an example of a view design document definition.

-
-
-
-
{
-   "_id": "_design/players"
-   "views": {
-     "total_experience": {
-       "map": "function(doc,meta){if(doc.jsonType ==
-       "reduce": "_sum"
-     },
-     "player_list": {
-       "map": "function (doc, meta){if(doc.jsonType ==
-     }
-   }
- }
-
-
-
-

In the document above, the _id field is used to name the design document. -This name should always be prefixed with _design/. -The other field in the top level of the document is the views field. -This field contains a map of view definitions. -The key for each element in the map is the name of the view. -Each view must contain a map element that defines the map function, and may also contain an optional reduce element that defines the reduce function.

-
-
-

View design documents support map-reduce views as well as spatial views. -Below is an example of a spatial view definition. -Spatial views follow similar rules as the map-reduce views above.

-
-
-
-
 {
-   "_id": "_design/spatial"
-   "spatial": {
-	 	"position": "<spatial view function definition>",
-		"location": "<spatial view function definition>"
-   }
- }
-
-
-
-

Note that spatial views only use a single function to define the index. -As a result this function is defined as the value of the spatial views name.

-
-
-

The other folder at the top level directory of a sample data folder is the docs folder. -This folder will contain all of the documents to load into Couchbase. -Each document in this folder is contained in a separate file and each file should contain a single JSON document. -The key name for the document will be the name of the file. -Each file should also have a .json file extension which will be removed from the key name when the data is loaded. -Since each document to be loaded is in a separate file, there can potentially be many files. -The docs folder allows subfolders to help categorize documents.

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbexport/cbexport-json.html b/scripts/cbtools/share/doc/cbexport/cbexport-json.html deleted file mode 100644 index 5766bcfd..00000000 --- a/scripts/cbtools/share/doc/cbexport/cbexport-json.html +++ /dev/null @@ -1,1003 +0,0 @@ - - - - - - - - - -cbexport-json(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbexport json [--cluster <url>] [--bucket <bucket_name>] [--format <data_format>]
-              [--username <username>] [--password <password>] [--client-cert <path>]
-              [--client-cert-password <password>] [--client-key <path>]
-              [--client-key-password <password>] [--cacert <path>] [--no-ssl-verify]
-              [--threads <num>] [--log-file <path>] [--include-key <key>]
-              [--include-data <collection_string_list>]
-              [--exclude-data <collection_string_list>] [--verbose]
-              [--scope-field <scope_field>] [--collection-field <collection_field>]
-
-
-
-
-

DESCRIPTION

-
-
-

Exports JSON data from Couchbase. The cbexport-json command supports exporting -JSON documents to a file with a document on each line or a file that contain a -JSON list where each element is a document. The file format to export to can be -specified with the --format flag. See the DATASET FORMATS section below for more -details on the supported file formats.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of required and optional parameters for the cbexport-json -command.

-
-
-

Required

-
-
-
-c,--cluster <url>
-
-

The hostname of a node in the cluster to export data from. See the HOST -FORMATS section below for details about hostname specification formats.

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
-b,--bucket <bucket_name>
-
-

The name of the bucket to export data from.

-
-
-f,--format <format>
-
-

The format of the dataset specified (lines or list). See the DATASET FORMATS -section below for more details on the formats supported by cbexport.

-
-
-o,--output <path>
-
-

The path to the location of the file that JSON documents from Couchbase -should be exported to. This may be an absolute or relative path, but must -point to a file. The file does not have to exist when the command is -invoked.

-
-
-
-
-
-

Optional

-
-
-
--include-key <key>
-
-

Couchbase stores data as key value pairs where the value is a JSON document -and the key is an identifier for retrieving that document. By default -cbexport will only export the value portion of the document. If you wish to -include the key in the exported document then this option should be -specified. The value passed to this option should be the field name that the -key is stored under. If the value passed already exists as a field in the -document, it will be overridden with the key. If the JSON document is not an -object it will be turned into one and the value added to a field named 'value'. -If the key value passed is 'value' then the key will not be written. It will -display a warning for any document it has converted into an object.

-
-
--include-data <collection_string_list>
-
-

A comma separated list of collection strings to include when exporting from -the bucket. Only scopes/collections included in this will be included in the -output file. The expected format is scope1.collection1,scope2.collection2. -This argument is mutually exclusive with --exclude-data.

-
-
--exclude-data <collection_string_list>
-
-

A comma separated list of collection strings to exclude when exporting from -the bucket. Any scopes/collections included in this list will not be included -in the output file. The expected format is -scope1.collection1,scope2.collection2. This argument is mutually exclusive -with --include-data.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man-in-the-middle attack if -you use this flag. Either this flag or the --cacert flag must be specified -when using an SSL encrypted connection.

-
-
--cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag -must be specified when using an SSL encrypted connection.

-
-
--scope-field <scope_field>
-
-

When exporting from a collection aware cluster this field will be created -in each JSON document; it will be used to store the name of the scope the -document came from. This flag is required when exporting from a bucket that -has non-default scopes and collections.

-
-
--collection-field <collection_field>
-
-

When exporting from a collection aware cluster this field will be created -in each JSON document; it will be used to store the name of the collection -the document came from. This flag is required when exporting from a bucket -that has non-default scopes and collections.

-
-
-t,--threads <num>
-
-

Specifies the number of concurrent clients to use when exporting data. Fewer -clients means exports will take longer, but there will be less cluster -resources used to complete the export. More clients means faster exports, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the export is -taking place.

-
-
-l,--log-file <path>
-
-

Specifies a log file for writing debugging information about cbexport -execution.

-
-
-v,--verbose
-
-

Specifies that logging should be sent to stdout. If this flag is specified -along with the -l/--log-file option then the verbose option is ignored.

-
-
-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

DATASET FORMATS

-
-
-

The cbexport command supports the formats listed below.

-
-
-

LINES

-
-

The lines format specifies a file that contains one JSON document on every line -in the file. This format is specified by setting the --format option to "lines". -Below is an example of a file in lines format.

-
-
-
-
{"key": "mykey1", "value": "myvalue1"}
-{"key": "mykey2", "value": "myvalue2"}
-{"key": "mykey3", "value": "myvalue3"}
-{"key": "mykey4", "value": "myvalue4"}
-
-
-
-
-

LIST

-
-

The list format specifies a file which contains a JSON list where each element -in the list is a JSON document. The file may only contain a single list, but the -list may be specified over multiple lines. This format is specified by setting -the --format option to "list". Below is an example of a file in list format.

-
-
-
-
[
-  {
-    "key": "mykey1",
-    "value": "myvalue1"
-  },
-  {"key": "mykey2", "value": "myvalue2"},
-  {"key": "mykey3", "value": "myvalue3"},
-  {"key": "mykey4", "value": "myvalue4"}
-]
-
-
-
-
-
-
-

EXAMPLES

-
-
-

To export data to /data/lines.json using the lines format and running with 4 -threads the following command can be run.

-
-
-
-
$ cbexport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -o /data/lines.json -f lines -t 4
-
-
-
-

To export data to /data/list.json using the list format the following command -can be run.

-
-
-
-
$ cbexport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -o /data/list.json -f list
-
-
-
-

To export data from a collections aware cluster with the scope and collection being -added to the scope/collection field the following command can be run.

-
-
-
-
$ cbexport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -o /data/list.json -f list --scope-field scope --collection-field collection
-
-[
-  {
-    "scope": "myscope1",
-    "collection": "mycollection1",
-    "key": "mykey1",
-    "value": "myvalue1",
-  }
-]
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbimport-json(1)

-
-
-
-
-

CBEXPORT

-
-
-

Part of the cbexport(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbexport/cbexport.html b/scripts/cbtools/share/doc/cbexport/cbexport.html deleted file mode 100644 index a07a98c5..00000000 --- a/scripts/cbtools/share/doc/cbexport/cbexport.html +++ /dev/null @@ -1,645 +0,0 @@ - - - - - - - - - -cbexport(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbexport [--version] [--help] <command> [<args>]
-
-
-
-
-

DESCRIPTION

-
-
-

cbexport is used to export data from Couchbase in various different formats.

-
-
-

For more information on how specific commands work you can run "cbexport -<command> --help".

-
-
-
-
-

OPTIONS

-
-
-
-
--version
-
-

Prints the cbexport suite version that the cbexport program came from.

-
-
--help
-
-

Prints the synopsis and a list of commands.

-
-
-
-
-

If a cbexport command is named, this option will bring up the manual page for -that command.

-
-
-
-
-

CBEXPORT COMMANDS

-
-
-
-
cbexport-json(1)
-
-

Exports data from Couchbase into a JSON file.

-
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
-
-
-
-
-

COLLECTION AWARE EXPORTS

-
-
-

The default behavior of export for the cluster versions which support -collections is to export all available scopes/collections in a bucket.

-
-
-
-
-

DISCUSSION

-
-
-

The cbexport command is used to export data from Couchbase to various different -formats. Each supported export format is a subcommand of the cbexport utility.

-
-
-
-
-

OPERATIONS DURING MAJOR CLUSTER CONFIGURATION CHANGES

-
-
-

Operations (commands or sub-commands) which connect to a cluster are not -supported during major cluster configuration changes.

-
-
-

For example, performing an import/export, making a backup or performing a -restore whilst changing the TLS configuration/security settings is unsupported.

-
-
-

These types of changes (e.g. changing the TLS mode to strict) are not expected -to be time consuming so it’s generally expected that operations should be -started after completing the configuration change.

-
-
-

Please note that this does not include rebalances; operations may be performed -during a rebalance. The reason for this distinction, is that major cluster -configuration changes are generally quick, whilst rebalances for large data -sets may be time consuming.

-
-
-
-
-

FURTHER DOCUMENTATION

-
-
-

A manual page is available for each of the supported sources listed above.

-
-
-
-
-

AUTHORS

-
-
-

cbexport[1] is a Couchbase Source tool and was written by Couchbase.

-
-
-
-
-

REPORTING BUGS

-
-
-

Report urgent issues to the Couchbase Support Team at support@couchbase.com. -Bugs can be reported to the Couchbase Jira Bug Tracker at -http://www.couchbase.com/issues.

-
-
-
-
-

SEE ALSO

-
-
-

cbexport-json(1)

-
-
-
-
-

CBEXPORT

-
-
-

Part of the cbexport(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbimport/cbimport-csv.html b/scripts/cbtools/share/doc/cbimport/cbimport-csv.html deleted file mode 100644 index bce14db9..00000000 --- a/scripts/cbtools/share/doc/cbimport/cbimport-csv.html +++ /dev/null @@ -1,1186 +0,0 @@ - - - - - - - - - -cbimport-csv(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbimport csv [--cluster <url>] [--bucket <bucket_name>] [--dataset <path>]
-             [--username <username>] [--password <password>] [--client-cert <path>]
-             [--client-cert-password <password>] [--client-key <path>]
-             [--client-key-password <password>] [--generate-key <key_expr>]
-             [--limit-rows <num>] [--skip-rows <num>] [--field-separator <char>]
-             [--cacert <path>] [--no-ssl-verify] [--threads <num>]
-             [--error-log <path>] [--log-file <path>] [--verbose]
-             [--field-delimiter <char>] [--generator-delimiter <char>]
-             [--ignore-fields <fields>]
-             [--scope-collection-exp <scope_collection_expression>]
-
-
-
-
-

DESCRIPTION

-
-
-

Imports CSV and other forms of separated value type data into Couchbase. By -default data files should contain comma separated values, but if for example you -are importing data that is tab separated you can use the --field-separator flag -to specify that tabs are used instead of commas.

-
-
-

The cbimport command also supports custom key-generation for each document in -the imported file. Key generation is done with a combination of pre-existing -fields in a document and custom generator functions supplied by cbimport. See -the KEY GENERATION section below for details about key generators.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of required and optional parameters for the cbimport-csv -command.

-
-
-

Required

-
-
-
-c,--cluster <url>
-
-

The hostname of a node in the cluster to import data into. See the HOST -FORMATS section below for details about hostname specification formats.

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
-b,--bucket <bucket_name>
-
-

The name of the bucket to import data into.

-
-
-d,--dataset <uri>
-
-

The URI of the dataset to be loaded. cbimport supports loading data from a -local file or from a URL. When importing data from a local file the path -must be prefixed with file://.

-
-
-g,--generate-key <key_expr>
-
-

Specifies a key expression used for generating a key for each document -imported. See the Key Generation section below for more information on -specifying key generators. If the resulting key is not unique the values -will be overridden, resulting in fewer documents than expected being imported. -To ensure that the key is unique add #MONO_INCR# or #UUID# to the key -generator expression.

-
-
-
-
-
-

Optional

-
-
-
--field-delimiter <char>
-
-

Specifies the character to be used to denote field references in the key -generator expression. It defaults to %. See the KEY GENERATION section.

-
-
--generator-delimiter <char>
-
-

Specifies the character to be used to denote generator references in the key -generator expression. It defaults to #. See the KEY GENERATION section.

-
-
--field-separator <num>
-
-

Specifies the field separator to use when reading the dataset. By default -the separator is a comma. To read tab separated files you can specify a tab -in this field. Tabs are specified as \t.

-
-
--limit-rows <num>
-
-

Specifies that the utility should stop loading data after reading a certain -amount of rows from the dataset. This option is useful when you have a large -dataset and only want to partially load it.

-
-
--skip-rows <num>
-
-

Specifies that the utility should skip some rows before we start importing -data. If this flag is used together with the --limit-rows flag then we will -import the number of rows specified by --limit-rows after we have skipped the -rows specified by --skip-rows.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man-in-the-middle attack if -you use this flag. Either this flag or the --cacert flag must be specified -when using an SSL encrypted connection.

-
-
--infer-types
-
-

By default all values in a CSV files are interpreted as strings. If infer -types is set then cbimport will look at each value and decide whether it is -a string, integer, or boolean value and put the inferred type into the -document.

-
-
--omit-empty
-
-

Some values in a CSV row will not contain any data. By default these values -are put into the generated JSON document as an empty string. To omit fields -that contain empty values specify this flag.

-
-
--cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag -must be specified when using an SSL encrypted connection.

-
-
--scope-collection-exp <scope_collection_expression>
-
-

When importing to a collection aware cluster you may optionally choose to -provide a scope/collection expression which will be used to determine which -scope/collection to insert the imported document in. This flag closely -resembles the behavior/syntax of the --generate-key flag. For example, to -use a static scope/collection use --scope-collection-exp scope.collection. -To use information from the CSV row, specify the column name between % -characters. For example, ---scope-collection-exp %scope_column%.%collection_column%. Columns that -contain a % character may be escaped using %%. For more information about -the accepted format, see the SCOPE/COLLECTION PARSER section.

-
-
-t,--threads <num>
-
-

Specifies the number of concurrent clients to use when importing data. Fewer -clients means imports will take longer, but there will be less cluster -resources used to complete the import. More clients means faster imports, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the import is -taking place.

-
-
-e,--errors-log <path>
-
-

Specifies a log file where JSON documents that could not be loaded are -written to. A document might not be loaded if a key could not be generated -for the document or if the document is not valid JSON. The errors file is -written in the "json lines" format (one document per line).

-
-
-l,--log-file <path>
-
-

Specifies a log file for writing debugging information about cbimport -execution.

-
-
-v,--verbose
-
-

Specifies that logging should be sent to stdout. If this flag is specified -along with the -l/--log-file option then the verbose option is ignored.

-
-
--ignore-fields <fields>
-
-

Specify a comma separated list of field names that will be excluded from -the imported document. The field reference syntax is the same as the one -used in KEY GENERATORS to refer to fields.

-
-
-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

KEY GENERATORS

-
-
-

Key generators are used in order to generate a key for each document loaded. -Keys can be generated by using a combination of characters, the values of a -given row in a document, and custom generators. Row substitutions are done by -wrapping the column name in "%" and custom generators are wrapped in "#". Below -is an example of a key generation expression.

-
-
-

Given the CSV dataset:

-
-
-
-
fname,age
-alice,40
-barry,36
-
-
-
-

Key Generator Expression:

-
-
-
-
--generate-key key::%fname%::#MONO_INCR#
-
-
-
-

The following keys would be generated:

-
-
-
-
key::alice::1
-key::barry::2
-
-
-
-

In the example above we generate a key using the value in each row of the -fname column and a custom generator. To specify that we want to substitute the -value in the fname column we put the name of the column fname between two -percent signs. This is an example of field substitution and it allows the -ability to build keys out of data that is already in the dataset.

-
-
-

This example also contains a generator function MONO_INCR which will increment -by 1 each time the key generator is called. Since this is the first time this -key generator was executed it returns 1. If we executed the key generator again -it would return 2 and so on. The starting value of the MONO_INCR generator is 1 -by default, but it can be changed by specifying a number in brackets after the -MONO_INCR generator name. To start generating monotonically incrementing values -starting at 100 for example, the generator MONO_INCR[100] would be specified. -The cbimport command current contains a monotonic increment generator -(MONO_INCR) and a UUID generator (UUID).

-
-
-

Any text that isn’t wrapped in "%" or "#" is static text and will be in the -result of all generated keys. If a key needs to contain a "%" or "#" in static -text then they need to be escaped by providing a double "%" or "#" -(ex. "%%" or "##"). The delimiter characters can be changed to avoid having to -escape them by using the --field-delimiter and --generator-delimiter flags.

-
-
-

If a key cannot be generated because the field specified in the key generator is -not present in the document then the key will be skipped. To see a list of -document that were not imported due to failed key generation users can specify -the --errors-log <path> parameter to dump a list of all documents that could not -be imported to a file.

-
-
-
-
-

SCOPE/COLLECTION PARSER

-
-
-

Scope/collection parsers are used in order to determine which scope/collection -to insert documents into. There are currently two supported parsers text/field.

-
-
-

Given the CSV dataset:

-
-
-
-
 product, stock, type, subtype
-apple, 100, produce, fruit
-
-
-
-

Scope/collection expression:

-
-
-
-
--scope-collection-exp %type%.%subtype%
-
-
-
-

The row would be inserted into the 'fruit' collection inside the 'produce' -scope.

-
-
-

Given the CSV dataset:

-
-
-
-
fname,age
-alice,40
-barry,36
-
-
-
-

Scope/collection expression:

-
-
-
-
--scope-collection-exp uk.manchester
-
-
-
-

In this case, no fields in the row will be used to determine the scope or -collection; all the rows would be inserted into the 'manchester' collection -inside the 'uk' scope.

-
-
-

There is nothing stopping the mixture of text/field expressions the following -are all valid expressions.

-
-
-
-
uk.%city%
-uk.%city%-5
-%country%.%city%::%town%
-
-
-
-
-
-

EXAMPLES

-
-
-

In the examples below we will show examples for importing data from the files -below.

-
-
-
-
./data/people.csv
-
-  fname,age
-  alice,40
-  barry,36
-
-./data/people.tsv
-
-  fname  age
-  alice  40
-  barry  36
-
-
-
-

To import data from /data/people.csv using a key containing the fname column -and utilizing 4 threads the following command can be run.

-
-
-
-
$ cbimport csv -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d file:///data/people.csv -g key::%fname% -t 4
-
-
-
-

To import data from /data/people.tsv using a key containing the fname column -and the UUID generator the following command would be run.

-
-
-
-
$ cbimport csv -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d file:///data/people.tsv --field-separator $_\t_ \
- -g key::%fname%::#UUID# -t 4
-
-
-
-

To import data from /data/list.csv using a key containing the "name" column and -then a unique id separated by a # we could use the --generator-delimiter flag -to avoid escaping the # sign. An example would be:

-
-
-
-
$ cbimport csv -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d file:///data/list.csv --generator-delimiter '£' \
--g key::%name%#£UUID£ -t 4
-
-
-
-

If the dataset in not available on the local machine where the command is run, -but is available via an HTTP URL we can still import the data using cbimport. If -we assume that the data is located at http://data.org/people.csv then we can -import the data with the command below.

-
-
-
-
$ cbimport csv -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d http://data.org/people.csv -g key::%fname%::#UUID# -t 4
-
-
-
-

If the CSV dataset contains information which would allow importing into -scopes/collections then an command like the one below could be used.

-
-
-
-
product, stock, type, subtype
-apple, 100, produce, fruit
-
-
-
-
-
$ cbimport csv -c couchbase://127.0.0.1 -u Administrator -p password
- -b default -d file://data/list.csv -g %product%
- --scope-collection-exp %type%.%subtype%
-
-
-
-

This command would place the row into the fruit collection inside the -produce scope.

-
-
-
-
-

DISCUSSION

-
-
-

The cbimport-csv command is used to quickly import data from various files -containing CSV, TSV or other separated format data. While importing CSV the -cbimport command only utilizes a single reader. As a result importing large -dataset may benefit from being partitioned into multiple files and running a -separate cbimport process on each file.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbimport-json(1)

-
-
-
-
-

CBIMPORT

-
-
-

Part of the cbimport(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbimport/cbimport-json.html b/scripts/cbtools/share/doc/cbimport/cbimport-json.html deleted file mode 100644 index a4019b36..00000000 --- a/scripts/cbtools/share/doc/cbimport/cbimport-json.html +++ /dev/null @@ -1,1273 +0,0 @@ - - - - - - - - - -cbimport-json(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbimport json [--cluster <url>] [--bucket <bucket_name>] [--dataset <path>]
-              [--format <data_format>] [--username <username>] [--password <password>]
-              [--client-cert <path>] [--client-cert-password <password>]
-              [--client-key <path>] [--client-key-password <password>]
-              [--generate-key <key_expr>] [--cacert <path>] [--no-ssl-verify]
-              [--threads <num>] [--error-log <path>] [--log-file <path>] [--verbose]
-              [--ignore-fields <fields>] [--field-delimiter <char>]
-              [--generator-delimiter <char>] [--bucket-quota <quota>]
-              [--scope-collection-exp <scope_collection_expression>]
-              [--bucket-replicas <replicas>]
-
-
-
-
-

DESCRIPTION

-
-
-

Imports JSON data into Couchbase. The cbimport command supports files that have -a JSON document on each line, files that contain a JSON list where each element -is a document, and the Couchbase Samples format. The file format can be -specified with the --format flag. See the DATASET FORMATS section below for more -details on the supported file formats.

-
-
-

The cbimport command also supports custom key-generation for each document in -the imported file. Key generation is done with a combination of pre-existing -fields in a document and custom generator functions supplied by cbimport. See -the KEY GENERATION section below for details about key generators.

-
-
-
-
-

OPTIONS

-
-
-

Below are a list of required and optional parameters for the cbimport command.

-
-
-

Required

-
-
-
-c,--cluster <url>
-
-

The hostname of a node in the cluster to import data into. See the HOST -FORMATS section below for details about hostname specification formats.

-
-
-u,--username <username>
-
-

The username for cluster authentication. The user must have the appropriate -privileges to take a backup.

-
-
-p,--password <password>
-
-

The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password.

-
-
--client-cert <path>
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with --client-key as an alternative to the ---username and --password flags. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
--client-cert-password <password>
-
-

The password for the certificate provided to the --client-cert flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key <path>
-
-

The path to the client private key whose public key is contained in the -certificate provided to the --client-cert flag. May be supplied with ---client-cert as an alternative to the --username and --password -flags. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
--client-key-password <password>
-
-

The password for the key provided to the --client-key flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information.

-
-
-b,--bucket <bucket_name>
-
-

The name of the bucket to import data into.

-
-
-d,--dataset <uri>
-
-

The URI of the dataset to be loaded. cbimport supports loading data from a -local files only. When importing data from a local file the path must be -prefixed with file://.

-
-
-f,--format <format>
-
-

The format of the dataset specified (lines, list, sample). See the DATASET -FORMATS section below for more details on the formats supported by cbimport.

-
-
-
-
-
-

Optional

-
-
-
-g,--generate-key <key_expr>
-
-

Specifies a key expression used for generating a key for each document -imported. This parameter is required for list and lines formats, but not for -the sample format. See the KEY GENERATION section below for more information -on specifying key generators. If the resulting key is not unique the values -will be overridden, resulting in fewer documents than expected being imported. -To ensure that the key is unique add #MONO_INCR# or #UUID# to the key -generator expression.

-
-
--field-delimiter <char>
-
-

Specifies the character to be used to denote field references in the key -generator expression. It defaults to %. See the KEY GENERATION section.

-
-
--generator-delimiter <char>
-
-

Specifies the character to be used to denote generator references in the key -generator expression. It defaults to #. See the KEY GENERATION section.

-
-
--no-ssl-verify
-
-

Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man-in-the-middle attack if -you use this flag. Either this flag or the --cacert flag must be specified -when using an SSL encrypted connection.

-
-
--cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the --no-ssl-verify flag -must be specified when using an SSL encrypted connection.

-
-
--limit-docs <num>
-
-

Specifies that the utility should stop loading data after reading a certain -amount of docs from the dataset. This option is useful when you have a large -dataset and only want to partially load it.

-
-
--skip-docs <num>
-
-

Specifies that the utility should skip some docs before we start importing -data. If this flag is used together with the --limit-rows flag then we will -import the number of rows specified by --limit-rows after we have skipped the -rows specified by --skip-rows.

-
-
--scope-collection-exp <scope_collection_expression>
-
-

When importing to a collection aware cluster you may optionally choose to -provide a scope/collection expression which will be used to determine which -scope/collection to insert the imported document in. This flag closely -resembles the behavior/syntax of the --generate-key flag. For example, to -use a static scope/collection use --scope-collection-exp scope.collection. -To use information from the JSON document, specify the column name between -% characters. For example, ---scope-collection-exp %scope_field%.%collection_field%. Fields that -contain a % character may be escaped using %%. For more information about -the accepted format, see the SCOPE/COLLECTION PARSER section.

-
-
-t,--threads <num>
-
-

Specifies the number of concurrent clients to use when importing data. Fewer -clients means imports will take longer, but there will be less cluster -resources used to complete the import. More clients means faster imports, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the import is -taking place.

-
-
-e,--errors-log <path>
-
-

Specifies a log file where JSON documents that could not be loaded are -written to. A document might not be loaded if a key could not be generated -for the document or if the document is not valid JSON. The errors file is -written in the "lines" format (one document per line).

-
-
-l,--log-file <path>
-
-

Specifies a log file for writing debugging information about cbimport -execution.

-
-
-v,--verbose
-
-

Specifies that logging should be sent to stdout. If this flag is specified -along with the -l/--log-file option then the verbose option is ignored.

-
-
--ignore-fields <fields>
-
-

Specify a comma separated list of field names that will be excluded from -the imported document. The field reference syntax is the same as the one -used in KEY GENERATORS to refer to fields.

-
-
--bucket-quota <quota>
-
-

When importing a sample dataset using --format sample this argument will -set the memory quota for the sample bucket. Note that this will edit the -bucket settings if the bucket already exists, and will create it if it -doesn’t.

-
-
--bucket-replicas <replicas>
-
-

When importing a sample dataset using --format sample this argument will -set the number of replicas for the sample bucket. Note that this will edit -the bucket settings if the bucket already exists, and will create it if it -doesn’t.

-
-
-
-
-
-
-
-

HOST FORMATS

-
-
-

When specifying a host/cluster for a command using the -c/--cluster flag, the following formats -are accepted:

-
-
-
    -
  • -

    <addr>:<port>

    -
    • -
  • -

    http://<addr>:<port>

    -
    • -
  • -

    https://<addr>:<port>

    -
    • -
  • -

    couchbase://<addr>:<port>

    -
    • -
  • -

    couchbases://<addr>:<port>

    -
    • -
  • -

    couchbase://<srv>

    -
    • -
  • -

    couchbases://<srv>

    -
    • -
  • -

    <addr>:<port>,<addr>:<port>

    -
    • -
  • -

    <scheme>://<addr>:<port>,<addr>:<port>

    -
    • -
-
-
-

The <port> portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, http:// and couchbase:// will both default to 8091 where -https:// and couchbases:// will default to 18091. When connecting to a host/cluster using a -non-default port, the <port> portion of the host format must be specified.

-
-
-

Connection Strings (Multiple nodes)

-
-

The -c/--cluster flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of <addr>:<port> strings where <scheme> only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used.

-
-
-

For example, all of the following are valid connection strings:

-
-
-
    -
  • -

    localhost,[::1]

    -
    • -
  • -

    10.0.0.1,10.0.0.2

    -
    • -
  • -

    http://10.0.0.1,10.0.0.2

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2

    -
    • -
  • -

    couchbase://10.0.0.1,10.0.0.2

    -
    • -
  • -

    couchbases://10.0.0.1:12345,10.0.0.2:12345

    -
    • -
-
-
-
-

SRV Records

-
-

The -c/--cluster flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows:

-
-
-
    -
  • -

    The <scheme> portion must be either couchbase:// or couchbases://

    -
    • -
  • -

    The <srv> portion should be a hostname with no port

    -
    • -
  • -

    The <srv> portion must not be a valid IP address

    -
    • -
-
-
-

For example, all of the following are valid connection string using an SRV record:

-
-
-
    -
  • -

    couchbase://hostname

    -
    • -
  • -

    couchbases://hostname

    -
    • -
-
-
-
-

Alternate Addressing (CAO/K8S)

-
-

Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -network=external query parameter to force connection via the defined -alternate addressing.

-
-
-

For example, the following are valid connection strings:

-
-
-
    -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=default

    -
    • -
  • -

    https://10.0.0.1:12345,10.0.0.2?network=external

    -
    • -
-
-
-
-
-
-

CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)

-
-
-

This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done.

-
-
-

PEM ENCODED CERTIFICATE/KEY

-
-

An unencrypted PEM encoded certificate/key may be supplied by using: -- --client-cert <path> -- --client-key <path>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

The file passed to --client-key must contain at most one private key, the key can be in one of the following formats: -- PKCS#1 -- PKCS#8 -- EC

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY

-
-

An encrypted PKCS#8 formatted key may be provided using: -- --client-cert <path> -- --client-key <path> -- --client-key-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate, and an optional chain required to authenticate -the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-

ENCRYPTED PKCS#12 CERTIFICATE/KEY

-
-

An encrypted PKCS#12 certificate/key may be provided using: -- --client-cert <path> -- --client-cert-password <password>

-
-
-

The file passed to --client-cert must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate.

-
-
-

Currently, only the following key types are supported: -- RSA -- ECDSA -- ED25519

-
-
-
-
-
-

DATASET FORMATS

-
-
-

The cbimport command supports the formats listed below.

-
-
-

LINES

-
-

The lines format specifies a file that contains one JSON document on every line -in the file. This format is specified by setting the --format option to "lines". -Below is an example of a file in lines format.

-
-
-
-
{"key": "mykey1", "value": "myvalue1"}
-{"key": "mykey2", "value": "myvalue2"}
-{"key": "mykey3", "value": "myvalue3"}
-{"key": "mykey4", "value": "myvalue4"}
-
-
-
-
-

LIST

-
-

The list format specifies a file which contains a JSON list where each element -in the list is a JSON document. The file may only contain a single list, but the -list may be specified over multiple lines. This format is specified by setting -the --format option to "list". Below is an example of a file in list format.

-
-
-
-
[
-  {
-    "key": "mykey1",
-    "value": "myvalue1"
-  },
-  {"key": "mykey2", "value": "myvalue2"},
-  {"key": "mykey3", "value": "myvalue3"},
-  {"key": "mykey4", "value": "myvalue4"}
-]
-
-
-
-
-

SAMPLE

-
-

The sample format specifies a ZIP file or folder containing multiple documents. -This format is intended to load Couchbase sample data sets. Unlike the lines and -list formats the sample format may also contain indexes, views, and full-text -index definitions. The folder structure is specified below.

-
-
-
-
+ (root folder)
-  + docs
-    key1.json
-    key2.json
-    ...
-  + design_docs
-    indexes.json
-    views.json
-
-
-
-

All documents in the samples format are contained in the docs folder and there -is one file per document. Each filename in the docs folder is the key name for -the JSON document contained in the file. If the filename contains a .json -extension then the extension is excluded from the key name during the import. -This name can be overridden if the --generate-key option is specified. The docs -folder may also contain sub-folders of documents to be imported. Sub-folders can -be used to organize large amounts of documents into a more readable categorized -form.

-
-
-

The design_docs folder contains index definitions. The filename indexes.json is -reserved for secondary indexes. All other file names are used for view indexes.

-
-
-
-
-
-

KEY GENERATORS

-
-
-

Key generators are used in order to generate a key for each document loaded. -Keys can be generated by using a combination of characters, the values of a -given field in a document, and custom generators. Field substitutions are done -by wrapping the field name in "%" and custom generators are wrapped in "#". -Below is an example of a key generation expression.

-
-
-

Given the document:

-
-
-
-
{
-  "name": "alice",
-  "age": 40,
-  "address": {
-    "street": "Broadway",
-    "country": "USA"
-  }
-}
-
-
-
-

Key Generator Expression:

-
-
-
-
  --generate-key key::%name%::#MONO_INCR#
-
-
-
-

The following key would be generated:

-
-
-
-
  key::alice::1
-
-
-
-

In the example above we generate a key using both the value of a field in the -document and a custom generator. We use the "name" field to use the value of the -name field as part of the key. This is specified by "%name%" which tells the key -generator to substitute the value of the field "name" into the key. To reference -a nested field we would use "parent.child" syntax. For example to reference the -country we would use '%address.country%'. To reference a field that contains a -dot in the name we escape the string using `` . For example -'%`address.country`%' refers to a field named "address.country".

-
-
-

This example also contains a generator function MONO_INCR which will increment -by 1 each time the key generator is called. Since this is the first time this -key generator was executed it returns 1. If we executed the key generator again -it would return 2 and so on. The starting value of the MONO_INCR generator is 1 -by default, but it can be changed by specifying a number in brackets after the -MONO_INCR generator name. To start generating monotonically incrementing values -starting at 100 for example, the generator MONO_INCR[100] would be specified. -The cbimport command current contains a monotonic increment generator -(MONO_INCR) and a UUID generator (UUID).

-
-
-

Any text that isn’t wrapped in "%" or "#" is static text and will be in the -result of all generated keys. If a key needs to contain a "%" or "#" in static -text then they need to be escaped by providing a double "%" or "#" -(ex. "%%" or "##"). The delimiter characters can be changed to avoid having to -escape them by using the --field-delimiter and --generator-delimiter flags.

-
-
-

If a key cannot be generated because the field specified in the key generator is -not present in the document then the key will be skipped. To see a list of -document that were not imported due to failed key generation users can specify -the --errors-log <path> parameter to dump a list of all documents that could not -be imported to a file.

-
-
-
-
-

SCOPE/COLLECTION PARSER

-
-
-

Scope/collection parsers are used in order to determine which scope/collection -to insert documents into. There are currently two supported parsers text/field.

-
-
-

Given the JSON dataset:

-
-
-
-
[
-  {"product": "apple", "stock": 100, "type": "produce", "subtype": "fruit"}
-]
-
-
-
-

Scope/collection expression:

-
-
-
-
--scope-collection-exp %type%.%subtype%
-
-
-
-

The document would be inserted into the 'fruit' collection inside the 'produce' -scope.

-
-
-

Given the JSON dataset:

-
-
-
-
[
-  {"fname": "alice", "age": 40},
-  {"fname": "barry", "age": 36}
-]
-
-
-
-

Scope/collection expression:

-
-
-
-
--scope-collection-exp uk.manchester
-
-
-
-

In this case, no fields in the document will be used to determine the scope or -collection; all the documents would be inserted into the 'manchester' collection -inside the 'uk' scope.

-
-
-

There is nothing stopping the mixture of text/field expressions the following -are all valid expressions.

-
-
-
-
uk.%city%
-uk.%city%-5
-%country%.%city%::%town%
-
-
-
-
-
-

EXAMPLES

-
-
-

In the examples below we will show examples for importing data from the files -below.

-
-
-
-
./data/lines.json
-
-  {"name": "alice", "age": 37}
-  {"name": "bob", "age": 39}
-
-./data/list.json
-
-  [
-    {"name": "candice", "age": 42},
-    {"name": "daniel", "age": 38}
-  ]
-
-
-
-

To import data from /data/lines.json using a key containing the "name" field and -utilizing 4 threads the following command can be run.

-
-
-
-
$ cbimport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d file:///data/lines.json -f lines -g key::%name% -t 4
-
-
-
-

To import data from /data/list.json using a key containing the "name" field and -the UUID generator the following command would be run.

-
-
-
-
$ cbimport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d file:///data/list.json -f list -g key::%name%::#UUID# -t 4
-
-
-
-

To import data from /data/list.json using a key containing the "name" field and -then a unique id separated by a # we could use the --generator-delimiter flag -to avoid escaping the # sign. An example would be:

-
-
-
-
$ cbimport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d file:///data/list.json -f list --generator-delimiter '£' \
- -g key::%name%#£UUID£ -t 4
-
-
-
-

If the dataset in not available on the local machine where the command is run, -but is available via an HTTP URL we can still import the data using cbimport. If -we assume that the data is located at http://data.org/list.json and that the -dataset is in the JSON list format then we can import the data with the command -below.

-
-
-
-
$ cbimport json -c couchbase://127.0.0.1 -u Administrator -p password \
- -b default -d http://data.org/list.json -f list -g key::%name%::#UUID# -t 4
-
-
-
-

If the JSON dataset contains information which would allow importing into -scopes/collections then a command like the one below could be used.

-
-
-
-
[{"product": "apple", "stock": 100, "type": "produce", "subtype": "fruit"}]
-
-$ cbimport json -c couchbase://127.0.0.1 -u Administrator -p password
- -b default -d file://data/list.csv -f list -g %product%
- --scope-collection-exp %type%.%subtype%
-
-
-
-

This command would place the document into the fruit collection inside the -produce scope.

-
-
-
-
-

DISCUSSION

-
-
-

The cbimport-json command is used to quickly import data from various files -containing JSON data. While importing JSON the cbimport command only utilizes a -single reader. As a result importing large dataset may benefit from being -partitioned into multiple files and running a separate cbimport process on each -file.

-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbexport-json(1)

-
-
-
-
-

CBIMPORT

-
-
-

Part of the cbimport(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbimport/cbimport.html b/scripts/cbtools/share/doc/cbimport/cbimport.html deleted file mode 100644 index 97070799..00000000 --- a/scripts/cbtools/share/doc/cbimport/cbimport.html +++ /dev/null @@ -1,660 +0,0 @@ - - - - - - - - - -cbimport(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbimport [--version] [--help] <command> [<args>]
-
-
-
-
-

DESCRIPTION

-
-
-

cbimport is used to import data from various sources into Couchbase.

-
-
-

For more information on how specific commands work you can run -"cbimport <command> --help".

-
-
-
-
-

OPTIONS

-
-
-
-
--version
-
-

Prints the cbimport suite version that the cbimport program came from.

-
-
-h,--help
-
-

Prints the synopsis and a list of commands. If a cbimport command is named, -this option will bring up the manual page for that command.

-
-
-
-
-
-
-

CBIMPORT COMMANDS

-
-
-
-
cbimport-csv(1)
-
-

Imports data into Couchbase from a CSV file.

-
-
cbimport-json(1)
-
-

Imports data into Couchbase from a JSON file.

-
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_CLUSTER
-
-

Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden.

-
-
CB_USERNAME
-
-

Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden.

-
-
CB_PASSWORD
-
-

Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden.

-
-
CB_CLIENT_CERT
-
-

The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with CB_CLIENT_KEY as an alternative to the -CB_USERNAME and CB_PASSWORD variables. See the CERTIFICATE AUTHENTICATION -section for more information.

-
-
CB_CLIENT_CERT_PASSWORD
-
-

The password for the certificate provided to the CB_CLIENT_CERT variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information.

-
-
CB_CLIENT_KEY
-
-

The path to the client private key whose public key is contained in the -certificate provided to the CB_CLIENT_CERT variable. May be supplied with -CB_CLIENT_CERT as an alternative to the CB_USERNAME and CB_PASSWORD -variables. See the CERTIFICATE AUTHENTICATION section for more information.

-
-
CB_CLIENT_KEY_PASSWORD
-
-

The password for the key provided to the CB_CLIENT_KEY variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information.

-
-
-
-
-
-
-

DISCUSSION

-
-
-

The cbimport command is used to import data from various sources into a -Couchbase cluster. Each supported format is a subcommand of the cbimport -utility.

-
-
-
-
-

OPERATIONS DURING MAJOR CLUSTER CONFIGURATION CHANGES

-
-
-

Operations (commands or sub-commands) which connect to a cluster are not -supported during major cluster configuration changes.

-
-
-

For example, performing an import/export, making a backup or performing a -restore whilst changing the TLS configuration/security settings is unsupported.

-
-
-

These types of changes (e.g. changing the TLS mode to strict) are not expected -to be time consuming so it’s generally expected that operations should be -started after completing the configuration change.

-
-
-

Please note that this does not include rebalances; operations may be performed -during a rebalance. The reason for this distinction, is that major cluster -configuration changes are generally quick, whilst rebalances for large data -sets may be time consuming.

-
-
-
-
-

JSON SPECIFICATION

-
-
-

The cbimport utility translates data read from disk into JSON documents prior -to importing them into the provided cluster. The converted documents must -conform to the JSON specification defined in RFC 8259.

-
-
-

This means cbimport will skip importing documents that either: -1) Are not valid JSON -2) Are not valid UTF-8 data, as defined in RFC 8259 JSON transferred between - systems MUST be encoded using UTF-8.

-
-
-

Documents that are skipped during an import will be logged, along with the line -number/index from which they originated. This information can be used to single -out/fix invalid documents prior to re-importing.

-
-
-
-
-

FURTHER DOCUMENTATION

-
-
-

A manual page is available for each of the supported sources listed above.

-
-
-
-
-

AUTHORS

-
-
-

cbimport is a Couchbase Source tool and was written by Couchbase.

-
-
-
-
-

REPORTING BUGS

-
-
-

Report urgent issues to the Couchbase Support Team at support@couchbase.com. -Bugs can be reported to the Couchbase Jira Bug Tracker at -http://www.couchbase.com/issues.

-
-
-
-
-

SEE ALSO

-
-
-

cbimport-csv(1), -cbimport-json(1)

-
-
-
-
-

CBIMPORT

-
-
-

Part of the cbimport(1) suite

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbriftdump/cbriftdump.html b/scripts/cbtools/share/doc/cbriftdump/cbriftdump.html deleted file mode 100644 index a6515824..00000000 --- a/scripts/cbtools/share/doc/cbriftdump/cbriftdump.html +++ /dev/null @@ -1,822 +0,0 @@ - - - - - - - - - -cbriftdump(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbriftdump [--version] [--file <path>] [--directory <path>]
-           [--json] [--info-only] [--by-key] [--by-seqno]
-           [--no-meta] [--no-xattrs] [--no-value] [--hex-meta]
-           [--hex-xattrs] [--hex-value] [--help]
-           [--obj-access-key-id <access_key_id>] [--obj-cacert <cert_path>]
-           [--obj-endpoint <endpoint>] [--obj-no-ssl-verify]
-           [--obj-region <region>] [--obj-staging-dir <staging_dir>]
-           [--obj-secret-access-key <secret_access_key>]
-           [--s3-force-path-style] [--s3-log-level <level>]
-
-
-
-
-

DESCRIPTION

-
-
-

cbriftdump is a tool for interrogating Rift stores generated by cbbackupmgr.

-
-
-
-
-

OPTIONS

-
-
-
-
--version
-
-

Print version information then exits.

-
-
-f,--file <path>
-
-

Specify the path to a Rift meta store to be dumped; the accompanying data -store must be in the same folder for the dump to complete successfully.

-
-
-d,--directory <path>
-
-

Specify the path to a file containing multiple Rift stores e.g. the 'data' -folder in a cbbackupmgr backup repo.

-
-
--json
-
-

The output will be in the JSON format.

-
-
--info-only
-
-

Only display information about the Rift store itself, don’t dump any documents. This -option is only valid when supplied with the '--file' argument.

-
-
--by-key
-
-

Sort the documents by their keys before dumping them.

-
-
--by-seqno
-
-

Sort the documents by their sequence numbers before dumping them.

-
-
--no-meta
-
-

Do not display the documents meta data.

-
-
--no-xattrs
-
-

Do not display the documents extended attributes.

-
-
--no-value
-
-

Do not display the documents value.

-
-
--hex-meta
-
-

Display the documents meta data encoded in hex.

-
-
--hex-xattrs
-
-

Display the documents extended attributes encoded in hex.

-
-
--hex-value
-
-

Display the documents value encoded in hex.

-
-
-h,--help
-
-

Display information describing how to use this tool.

-
-
-
-
-

Cloud integration

-
-

Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0.

-
-
-

Multiple cloud providers are supported, see the list below for more information.

-
-
-
    -
  1. -

    Supported

    -
    -
      -
    • -

      AWS S3 (s3://)

      -
      • -
    • -

      GCP Google Storage (gs://)

      -
      • -
    • -

      Azure Blob Storage in 7.1.2+ (az://)

      -
      • -
    -
    -
    2. -
-
-
-

Required

-
-
-
--obj-staging-dir <staging_dir>
-
-

When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it’s not treated as a persistent store) and is -only used during the backup. NOTE: Do not use /tmp as the obj-staging-dir. -See Disk requirements in cbbackupmgr-cloud(7) for more information.

-
-
-
-
-
-

Optional

-
-
-
--obj-access-key-id <access_key_id>
-
-

The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -CB_OBJSTORE_ACCESS_KEY_ID environment variable.

-
-

When using AWS, this option expects an access key id. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account name. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-overview#storage-account-endpoints -for more information.

-
-
-

When using GCP, this option expects a client id. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-cacert <cert_path>
-
-

Specifies a CA certificate that will be used to verify the identity of the -object store being connected to.

-
-
--obj-endpoint <endpoint>
-
-

The host/address of your object store.

-
-
--obj-no-ssl-verify
-
-

Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man-in-the-middle attack.

-
-
--obj-region <region>
-
-

The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information.

-
-
--obj-secret-access-key <secret_access_key>
-
-

The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -CB_OBJSTORE_SECRET_ACCESS_KEY environment variable.

-
-

When using AWS, this option expects a secret access key. See -https://docs.aws.amazon.com/general/latest/gr/aws-sec-cred-types.html#access-keys-and-secret-access-keys -for more information.

-
-
-

When using Azure, this option expects an account key. See -https://docs.microsoft.com/en-us/azure/storage/common/storage-account-keys-manage?tabs=azure-portal -for more information.

-
-
-

When using GCP, this option expects a client secret. See -https://cloud.google.com/storage/docs/authentication for more information.

-
-
-
--obj-log-level <level>
-
-

Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below.

-
-

The valid options for the AWS SDK are debug, debug-with-signing, -debug-with-body, debug-with-request-retries, debug-with-request-errors, -and debug-with-event-stream-body.

-
-
-

The valid options for the Azure SDK are info, debug, debug-with-request-retries and -debug-with-request-retries-and-lro.

-
-
-

The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future.

-
-
-
--obj-auth-by-instance-metadata
-
-

Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints.

-
-

By default, this option is disabled for AWS.

-
-
-

By default, this option is enabled for Azure.

-
-
-

By default, this option is enabled for GCP.

-
-
-
--obj-auth-file
-
-

GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The --obj-auth-file flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers.

-
-
--obj-refresh-token
-
-

GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage.

-
-
-
-
-
-

AWS S3 Options

-
-
Optional
-
-
-
--s3-force-path-style
-
-

By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores.

-
-
-
-
-
-
-
-
-
-

FORMATS

-
-
-

The default format resembles the output of the cbsqlitedump tool; a dumped file will look like:

-
-
-
-
Key: 1
-  SeqNo: 1
-  Deleted: false
-  Size: 1B (key), 29B (meta), 50B (value)
-  Meta: {"cas":1572520644513911800,"revseqno":1}
-  Value: EvSPTKoWdIbGaroLZaMOZmMVyqUescxwxZsNkmcYwPGiBkPIPO
-
-
-
-

If the json format argument is supplied the output will look like:

-
-
-
-
{"key":"1","value":"EvSPTKoWdIbGaroLZaMOZmMVyqUescxwxZsNkmcYwPGiBkPIPO","meta":{"cas":1572520644513911800,"revseqno":1},"deleted":false}
-
-
-
-

If the info only argument is supplied the output will give information about the rift store itself:

-
-
-
-
Meta Info
-  Filename              : 0000_meta.sqlite.0
-  Size (bytes)          : 12288
-  Cache size            : -2000
-  Page size             : 4096
-  Table name            : vbucket_0
-  User version          : 0
-  First sequence number : 1
-  Last sequence number  : 1
-  Mutations             : 1
-  Deletions             : 0
-
-Data Info
-  Filename              : 0000_data.rift.0
-  Size (bytes)          : 83
-  Dead space (bytes)    : 0
-  Fragmentation         : 0.00%
-
-
-
-
-
-

ENVIRONMENT AND CONFIGURATION VARIABLES

-
-
-
-
CB_OBJSTORE_STAGING_DIRECTORY
-
-

Specifies the path to the staging directory. If the --obj-staging-dir -argument is provided in the command line then this value is overridden.

-
-
CB_OBJSTORE_REGION
-
-

Specifies the object store region. If the --obj-region argument is provided -in the command line then this value is overridden.

-
-
CB_OBJSTORE_ACCESS_KEY_ID
-
-

Specifies the object store access key id. If the --obj-access-key-id -argument is provided in the command line this value is overridden.

-
-
CB_OBJSTORE_SECRET_ACCESS_KEY
-
-

Specifies the object store secret access key. If the ---obj-secret-access-key argument is provided in the command line this value -is overridden.

-
-
CB_OBJSTORE_REFRESH_TOKEN
-
-

Specifies the refresh token to use. If the --obj-refresh-token argument is -provided in the command line, this value is overridden.

-
-
CB_AWS_ENABLE_EC2_METADATA
-
-

By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint.

-
-
-
-
-
-
-

SEE ALSO

-
-
-

cbbackupmgr-backup(1), -cbbackupmgr-cloud(7)

-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/doc/cbsqlitedump/cbsqlitedump.html b/scripts/cbtools/share/doc/cbsqlitedump/cbsqlitedump.html deleted file mode 100644 index 9d58f6c8..00000000 --- a/scripts/cbtools/share/doc/cbsqlitedump/cbsqlitedump.html +++ /dev/null @@ -1,565 +0,0 @@ - - - - - - - - - -cbsqlitedump(1) - - - - - -
-
-

SYNOPSIS

-
-
-
cbsqlitedump [--version] [--help] [--file <path>] [--directory <path>]
-             [--info-only] [--bykey] [--byseq] [--no-body] [--no-meta] [--json]
-
-
-
-
-

DESCRIPTION

-
-
-

cbsqlitedump is used to dump SQLite files generated by cbbackupmgr.

-
-
-
-
-

OPTIONS

-
-
-
-
--version
-
-

Prints the cbsqlitedump suite version that the cbsqlitedump program came -from.

-
-
-h,--help
-
-

Prints the synopsis and usage instructions for the command.

-
-
-f,--file <path>
-
-

Specify the file to be dumped.

-
-
-d,--directory <path>
-
-

Specify the path to the directory to be dumped.

-
-
--info-only
-
-

Displays only information of the specified file, no documents will be dumped.

-
-
--bykey
-
-

Sorts per table output by key. If a whole directory is dumped the output of -each table will be sorted by key.

-
-
--byseq
-
-

Sorts per table output by sequence number. If a whole directory is dumped the output of -each table will be sorted by sequence number.

-
-
--no-meta
-
-

Do not return any of the documents metadata.

-
-
--no-xattrs
-
-

Do not return any of the documents extended attributes.

-
-
--no-value
-
-

Do not return the documents value.

-
-
--hex-meta
-
-

Return the metadata encoded in hex.

-
-
--hex-xattrs
-
-

Return the extended attributes encoded in hex.

-
-
--hex-value
-
-

Return the value encoded in hex.

-
-
-j,--json
-
-

The output will be in JSON format.

-
-
-
-
-
-
-

FORMATS

-
-
-

The default format resembles the output of the forestdb_dump tool. A dumped -file will look like:

-
-
-
-
Key: pymc0
-  Deleted: false
-  Size: 5B (key), 29B (meta), 10B (value)
-  Meta: {"flags":0,"expiry":0,"locktime":0,"cas":1564654245015781376,"revseqno":1,"datatype":0}
-  Value:
-    00000000  30 30 30 30 30 30 30 30  30 30                    |0000000000|
-
-
-
-

If the json format is chosen it will look like:

-
-
-
-
{"key":"pymc0","value":"0000000000","meta":{"flags":0,"expiry":0,"locktime":0,"cas":1564654245015781376,"revseqno":1,"datatype":0},"deleted":false}
-
-
-
-
-
- - \ No newline at end of file diff --git a/scripts/cbtools/share/man/man1/cbbackupmgr-backup.1 b/scripts/cbtools/share/man/man1/cbbackupmgr-backup.1 deleted file mode 100644 index 5931407a..00000000 --- a/scripts/cbtools/share/man/man1/cbbackupmgr-backup.1 +++ /dev/null @@ -1,1332 +0,0 @@ -'\" t -.\" Title: cbbackupmgr-backup -.\" Author: Couchbase -.\" Generator: Asciidoctor 2.0.16 -.\" Date: 2023-09-20 -.\" Manual: Couchbase Server Manual -.\" Source: Couchbase Server 7.2.2-6401 -.\" Language: English -.\" -.TH "CBBACKUPMGR\-BACKUP" "1" "2023-09-20" "Couchbase Server 7.2.2\-6401" "Couchbase Server Manual" -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.ss \n[.ss] 0 -.nh -.ad l -.de URL -\fI\\$2\fP <\\$1>\\$3 -.. -.als MTO URL -.if \n[.g] \{\ -. mso www.tmac -. am URL -. ad l -. . -. am MTO -. ad l -. . -. LINKSTYLE blue R < > -.\} -.SH "NAME" -cbbackupmgr-backup \- Backs up data from a Couchbase cluster -.SH "SYNOPSIS" -.sp -.nf -cbbackupmgr backup [\-\-archive ] [\-\-repo ] - [\-\-cluster ] [\-\-username ] - [\-\-password ] [\-\-client\-cert ] - [\-\-client\-cert\-password ] [\-\-client\-key ] - [\-\-client\-key\-password ] [\-\-resume] [\-\-purge] - [\-\-threads ] [\-\-cacert ] [\-\-no\-ssl\-verify] - [\-\-value\-compression ] [\-\-no\-progress\-bar] - [\-\-skip\-last\-compaction] [\-\-consistency\-check ] - [\-\-full\-backup] [\-\-obj\-access\-key\-id ] - [\-\-obj\-cacert ] [\-\-obj\-endpoint ] - [\-\-obj\-no\-ssl\-verify] [\-\-obj\-region ] - [\-\-obj\-staging\-dir ] - [\-\-obj\-secret\-access\-key ] - [\-\-s3\-force\-path\-style] [\-\-s3\-log\-level ] - [\-\-passphrase ] [\-\-km\-key\-url ] - [\-\-km\-endpoint ] [\-\-km\-region ] - [\-\-km\-access\-key\-id ] [\-\-km\-secret\-access\-key ] - [\-\-km\-auth\-file ] -.fi -.br -.SH "DESCRIPTION" -.sp -Backs up a Couchbase cluster into the backup repository specified. Before -running the backup command, a backup repository must be created. See -\fBcbbackupmgr\-config\fP(1) for more details on creating a backup -repository. The backup command uses information from the previous backup taken -in order to backup all new data on a Couchbase cluster. If no previous backup -exists then all data on the cluster is backed up. The backup is taken based on -the backup repository\(cqs backup configuration. Each backup will create a new -folder in the backup repository. This folder will contain all data from the -backup and is named to reflect the time that the backup was started. -.sp -As the backup runs, it tracks its progress which allows failed backups to be -resumed from the point where they left off. If a backup fails before it is -complete it is considered a partial backup. To attempt to complete the backup -process, the backup may be resumed with the \-\-resume flag. It may also be -deleted and resumed from the previous successful backup with the \-\-purge flag. -.sp -The backup command is capable of backing up data when there is a cluster -rebalance operation in progress. During a rebalance, the backup command will -track data as it moves around the cluster and complete the backup. However, users -should use caution when running backups during a rebalance since both the -rebalance and backup operations can be resource intensive and may cause -temporary performance degradations in other parts of the cluster. See the -\-\-threads flag for information on how to lower the impact of the backup command -on your Couchbase cluster. -.sp -The backup command is also capable of backing up data when there are server -failures in the target backup cluster. When a server failure occurs the backup -command will wait for 180 seconds for the failed server to come back online or -for the failed server to be failed over and removed from the cluster. If 180 -seconds passes without the failed server coming back online or being failed over -then the backup command will mark the data on that node as failed and attempt to -back up the rest of the data from the cluster. The backup will be marked as a -partial backup in the backup archive and will need to be either resumed or -purged when the backup command is invoked again. -.SH "OPTIONS" -.sp -Below are a list of required and optional parameters for the backup command. -.SS "Required" -.sp -\-a,\-\-archive -.RS 4 -The location of the backup archive directory. When backing up directly to S3 -prefix the archive path with \f(CRs3://${BUCKET_NAME}/\fP. -.RE -.sp -\-r,\-\-repo -.RS 4 -The name of the backup repository to backup data into. -.RE -.sp -\-c,\-\-cluster -.RS 4 -The hostname of one of the nodes in the cluster to back up. See the -Host Formats section below for hostname specification details. -.RE -.sp -\-u,\-\-username -.RS 4 -The username for cluster authentication. The user must have the appropriate -privileges to take a backup. -.RE -.sp -\-p,\-\-password -.RS 4 -The password for cluster authentication. The user must have the appropriate -privileges to take a backup. If not password is supplied to this option then -you will be prompted to enter your password. -.RE -.sp -\-\-client\-cert -.RS 4 -The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with \f(CR\-\-client\-key\fP as an alternative to the -\f(CR\-\-username\fP and \f(CR\-\-password\fP flags. See the CERTIFICATE AUTHENTICATION -section for more information. -.RE -.sp -\-\-client\-cert\-password -.RS 4 -The password for the certificate provided to the \f(CR\-\-client\-cert\fP flag, when -using this flag, the certificate/key pair is expected to be in the PKCS#12 -format. See the CERTIFICATE AUTHENTICATION section for more information. -.RE -.sp -\-\-client\-key -.RS 4 -The path to the client private key whose public key is contained in the -certificate provided to the \f(CR\-\-client\-cert\fP flag. May be supplied with -\f(CR\-\-client\-cert\fP as an alternative to the \f(CR\-\-username\fP and \f(CR\-\-password\fP -flags. See the CERTIFICATE AUTHENTICATION section for more information. -.RE -.sp -\-\-client\-key\-password -.RS 4 -The password for the key provided to the \f(CR\-\-client\-key\fP flag, when using this -flag, the key is expected to be in the PKCS#8 format. See the CERTIFICATE -AUTHENTICATION section for more information. -.RE -.SS "Optional" -.sp -\-\-resume -.RS 4 -If the previous backup did not complete successfully it can be resumed from -where it left off by specifying this flag. Note that the resume and purge -flags may not be specified at the same time. -.RE -.sp -\-\-purge -.RS 4 -If the previous backup did not complete successfully the partial backup will -be removed and restarted from the point of the previous successful backup by -specifying this flag. Note that the purge and resume flags may not be -specified at the same time. -.RE -.sp -\-\-no\-ssl\-verify -.RS 4 -Skips the SSL verification phase. Specifying this flag will allow a -connection using SSL encryption, but will not verify the identity of the -server you connect to. You are vulnerable to a man\-in\-the\-middle attack if -you use this flag. Either this flag or the \-\-cacert flag must be specified -when using an SSL encrypted connection. -.RE -.sp -\-\-cacert -.RS 4 -Specifies a CA certificate that will be used to verify the identity of the -server being connecting to. Either this flag or the \-\-no\-ssl\-verify flag -must be specified when using an SSL encrypted connection. -.RE -.sp -\-\-value\-compression -.RS 4 -Specifies a compression policy for backed up values. When Couchbase sends -data to the backup client the data stream may contain all compressed values, -all uncompressed values, or a mix of compressed and uncompressed values. To -backup all data in the same form that the backup client receives it you can -specify "unchanged". If you wish for all values to be uncompressed then you -can specify "uncompressed". This policy will however uncompress any -compressed values received from Couchbase and may increase the backup file -size. To compress all values you can specify "compressed". This will compress -any uncompressed values before writing them to disk. The default value for -this option is "compressed". -.RE -.sp -\-t,\-\-threads -.RS 4 -Specifies the number of concurrent clients to use when taking a backup. -Fewer clients means backups will take longer, but there will be less cluster -resources used to complete the backup. More clients means faster backups, -but at the cost of more cluster resource usage. This parameter defaults to 1 -if it is not specified and it is recommended that this parameter is not set -to be higher than the number of CPUs on the machine where the backup is -taking place. -.RE -.sp -\-\-no\-progress\-bar -.RS 4 -By default, a progress bar is printed to stdout so that the user can see how -long the backup is expected to take, the amount of data that is being -transferred per second, and the amount of data that has been backed up. -Specifying this flag disables the progress bar and is useful when running -automated jobs. -.RE -.sp -\-\-consistency\-check -.RS 4 -When a window larger than 1 is provided it will enable the consistency -checker. This will show a warning if the backup consistency window is larger -than the one provided in seconds. This is an Enterprise Edition feature and -is currently in developer preview. See DISCUSSION for more information. -.RE -.sp -\-\-full\-backup -.RS 4 -Force a full backup in the same backup repository. This can be used to more -easily manage backups. -.RE -.SS "Cloud integration" -.sp -Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0. -.sp -Backing up directly to object store is only supported for Couchbase Server -6.6.0 and above. It\(cqs likely that backing up older clusters will result in -significantly higher memory consumption. -.sp -Multiple cloud providers are supported, see the list below for more information. -.sp -.RS 4 -.ie n \{\ -\h'-04' 1.\h'+01'\c -.\} -.el \{\ -. sp -1 -. IP " 1." 4.2 -.\} -Supported -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -AWS S3 (\f(CRs3://\fP) -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -GCP Google Storage (\f(CRgs://\fP) -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -Azure Blob Storage in 7.1.2+ (\f(CRaz://\fP) -.RE -.RE -.SS "Required" -.sp -\-\-obj\-staging\-dir -.RS 4 -When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it\(cqs not treated as a persistent store) and is -only used during the backup. NOTE: Do not use \f(CR/tmp\fP as the \f(CRobj\-staging\-dir\fP. -See \f(CRDisk requirements\fP in \fBcbbackupmgr\-cloud\fP(7) for more information. -.RE -.SS "Optional" -.sp -\-\-obj\-access\-key\-id -.RS 4 -The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -\f(CRCB_OBJSTORE_ACCESS_KEY_ID\fP environment variable. -.sp -When using AWS, this option expects an access key id. See -\c -.URL "https://docs.aws.amazon.com/general/latest/gr/aws\-sec\-cred\-types.html#access\-keys\-and\-secret\-access\-keys" "" -for more information. -.sp -When using Azure, this option expects an account name. See -\c -.URL "https://docs.microsoft.com/en\-us/azure/storage/common/storage\-account\-overview#storage\-account\-endpoints" "" -for more information. -.sp -When using GCP, this option expects a client id. See -.URL "https://cloud.google.com/storage/docs/authentication" "" "" -for more information. -.RE -.sp -\-\-obj\-cacert -.RS 4 -Specifies a CA certificate that will be used to verify the identity of the -object store being connected to. -.RE -.sp -\-\-obj\-endpoint -.RS 4 -The host/address of your object store. -.RE -.sp -\-\-obj\-no\-ssl\-verify -.RS 4 -Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man\-in\-the\-middle attack. -.RE -.sp -\-\-obj\-region -.RS 4 -The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information. -.RE -.sp -\-\-obj\-secret\-access\-key -.RS 4 -The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -\f(CRCB_OBJSTORE_SECRET_ACCESS_KEY\fP environment variable. -.sp -When using AWS, this option expects a secret access key. See -\c -.URL "https://docs.aws.amazon.com/general/latest/gr/aws\-sec\-cred\-types.html#access\-keys\-and\-secret\-access\-keys" "" -for more information. -.sp -When using Azure, this option expects an account key. See -\c -.URL "https://docs.microsoft.com/en\-us/azure/storage/common/storage\-account\-keys\-manage?tabs=azure\-portal" "" -for more information. -.sp -When using GCP, this option expects a client secret. See -.URL "https://cloud.google.com/storage/docs/authentication" "" "" -for more information. -.RE -.sp -\-\-obj\-log\-level -.RS 4 -Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below. -.sp -The valid options for the AWS SDK are \f(CRdebug\fP, \f(CRdebug\-with\-signing\fP, -\f(CRdebug\-with\-body\fP, \f(CRdebug\-with\-request\-retries\fP, \f(CRdebug\-with\-request\-errors\fP, -and \f(CRdebug\-with\-event\-stream\-body\fP. -.sp -The valid options for the Azure SDK are \f(CRinfo\fP, \f(CRdebug\fP, \f(CRdebug\-with\-request\-retries\fP and -\f(CRdebug\-with\-request\-retries\-and\-lro\fP. -.sp -The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future. -.RE -.sp -\-\-obj\-auth\-by\-instance\-metadata -.RS 4 -Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints. -.sp -By default, this option is disabled for AWS. -.sp -By default, this option is enabled for Azure. -.sp -By default, this option is enabled for GCP. -.RE -.sp -\-\-obj\-auth\-file -.RS 4 -GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The \f(CR\-\-obj\-auth\-file\fP flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers. -.RE -.sp -\-\-obj\-refresh\-token -.RS 4 -GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage. -.RE -.SS "AWS S3 Options" -.SS "Optional" -.sp -\-\-s3\-force\-path\-style -.RS 4 -By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores. -.RE -.SS "Encryption (Developer Preview)" -.sp -\-\-passphrase -.RS 4 -Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing. -.RE -.sp -\-\-km\-key\-url -.RS 4 -Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable \f(CRCB_KM_KEY_URL\fP. For more on how to authenticate using the different -providers see \fBcbbackupmgr\-encryption\fP(7). -.sp -For AWS the expected key format is \f(CRawskms://\fP, for example -\f(CRawskms://alias\(rskeyAlias\fP. -.sp -For GCP the expected key format is \f(CRgcpkms://\fP, for example -\f(CRgcpkms://projects/project\-id/locations/location/keyRings/keyring/cryptoKeys/key\fP. -.sp -For Azure key vault the expected key format is \f(CRazurekeyvault://\fP -for example: -\f(CRazurekeyvault://vault\-name.vault.azure.net/object\-type/object\-name/object\-version\fP. -.sp -For HashiCorp Vault the expected format is \f(CRhashivaults:///\fP -for example: -\f(CRhashivaults://127.0.0.1:8200/keyName\fP. -.RE -.sp -\-\-km\-region -.RS 4 -Required when using AWS KMS, it allows you to set the key region. -.RE -.sp -\-\-km\-endpoint -.RS 4 -The host or address to use as your KMS. It will override the default SDK one. -.RE -.sp -\-\-km\-access\-key\-id -.RS 4 -The user ID used to connect to the key management service. It can also be -provided via \f(CRCB_KM_ACCESS_KEY_ID\fP environmental variable. Please refer -to \fBcbbackupmgr\-encryption\fP(7) for the required authentication for each -provider. -.RE -.sp -\-\-km\-secret\-access\-key -.RS 4 -The key used to connect to the key management service. It can also be -provided via the \f(CRCB_KM_SECRET_ACCESS_KEY\fP environmental variable. Please -refer to \fBcbbackupmgr\-encryption\fP(7) for the required authentication for -each provider. -.RE -.sp -\-\-km\-auth\-file -.RS 4 -The path to a file containing the authentication credentials for the key -management service. It can also be provided via the \f(CRCB_KM_AUTH_FILE\fP -environmental variable. Please refer to \fBcbbackupmgr\-encryption\fP(7) for the -required authentication for each provider. -.RE -.SH "HOST FORMATS" -.sp -When specifying a host/cluster for a command using the \f(CR\-c\fP/\f(CR\-\-cluster\fP flag, the following formats -are accepted: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CR:\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRhttp://:\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRhttps://:\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbase://:\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbases://:\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbase://\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbases://\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CR:,:\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CR://:,:\fP -.RE -.sp -The \f(CR\fP portion of the host format may be omitted, in which case the default port will be used -for the scheme provided. For example, \f(CRhttp://\fP and \f(CRcouchbase://\fP will both default to 8091 where -\f(CRhttps://\fP and \f(CRcouchbases://\fP will default to 18091. When connecting to a host/cluster using a -non\-default port, the \f(CR\fP portion of the host format must be specified. -.SS "Connection Strings (Multiple nodes)" -.sp -The \f(CR\-c\fP/\f(CR\-\-cluster\fP flag accepts multiple nodes in the format of a connection string; this is a -comma separated list of \f(CR:\fP strings where \f(CR\fP only needs to be specified once. -The main advantage of supplying multiple hosts is that in the event of a failure, the next host in -the list will be used. -.sp -For example, all of the following are valid connection strings: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRlocalhost,[::1]\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CR10.0.0.1,10.0.0.2\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRhttp://10.0.0.1,10.0.0.2\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRhttps://10.0.0.1:12345,10.0.0.2\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbase://10.0.0.1,10.0.0.2\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbases://10.0.0.1:12345,10.0.0.2:12345\fP -.RE -.SS "SRV Records" -.sp -The \f(CR\-c\fP/\f(CR\-\-cluster\fP flag accepts DNS SRV records in place of a host/cluster address where the SRV -record will be resolved into a valid connection string. There are a couple of rules which must be -followed when supplying an SRV record which are as follows: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -The \f(CR\fP portion must be either \f(CRcouchbase://\fP or \f(CRcouchbases://\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -The \f(CR\fP portion should be a hostname with no port -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -The \f(CR\fP portion must not be a valid IP address -.RE -.sp -For example, all of the following are valid connection string using an SRV record: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbase://hostname\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRcouchbases://hostname\fP -.RE -.SS "Alternate Addressing (CAO/K8S)" -.sp -Users of the CAO (Couchbase Autonomous Operator) or K8S may need to supply the -\f(CRnetwork=external\fP query parameter to force connection via the defined -alternate addressing. -.sp -For example, the following are valid connection strings: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRhttps://10.0.0.1:12345,10.0.0.2?network=default\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRhttps://10.0.0.1:12345,10.0.0.2?network=external\fP -.RE -.SH "CERTIFICATE AUTHENTICATION (MTLS AUTHENTICATION)" -.sp -This tool supports authenticating against a Couchbase Cluster by using certificate based authentication (mTLS -authentication). To use certificate based authentication a certificate/key must be supplied, there a currently -multiple ways this may be done. -.SS "PEM ENCODED CERTIFICATE/KEY" -.sp -An unencrypted PEM encoded certificate/key may be supplied by using: -\- \f(CR\-\-client\-cert \fP -\- \f(CR\-\-client\-key \fP -.sp -The file passed to \f(CR\-\-client\-cert\fP must contain the client certificate, and an optional chain required to authenticate -the client certificate. -.sp -The file passed to \f(CR\-\-client\-key\fP must contain at most one private key, the key can be in one of the following formats: -\- PKCS#1 -\- PKCS#8 -\- EC -.sp -Currently, only the following key types are supported: -\- RSA -\- ECDSA -\- ED25519 -.SS "PEM ENCODED CERTIFICATE/PEM OR DER ENCRYPTED PKCS#8 KEY" -.sp -An encrypted PKCS#8 formatted key may be provided using: -\- \f(CR\-\-client\-cert \fP -\- \f(CR\-\-client\-key \fP -\- \f(CR\-\-client\-key\-password \fP -.sp -The file passed to \f(CR\-\-client\-cert\fP must contain the client certificate, and an optional chain required to authenticate -the client certificate. -.sp -Currently, only the following key types are supported: -\- RSA -\- ECDSA -\- ED25519 -.SS "ENCRYPTED PKCS#12 CERTIFICATE/KEY" -.sp -An encrypted PKCS#12 certificate/key may be provided using: -\- \f(CR\-\-client\-cert \fP -\- \f(CR\-\-client\-cert\-password \fP -.sp -The file passed to \f(CR\-\-client\-cert\fP must contain the client certificate and exactly one private key. It may also contain -the chain required to authenticate the client certificate. -.sp -Currently, only the following key types are supported: -\- RSA -\- ECDSA -\- ED25519 -.SH "RBAC" -.sp -When performing a backup/restore with a user which is using RBAC, there are a -couple of things that should be taken into consideration each of which is -highlighted in this section. -.SS "Bucket Level" -.sp -Bucket level data may be backed up/restored using the \f(CRdata_backup\fP (Data -Backup & Restore) role. -.sp -The \f(CRdata_backup\fP role does not have access to cluster level data such as: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -Analytics Synonyms -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -Eventing Metadata -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -FTS Aliases -.RE -.sp -Backing up/restoring cluster level data with the \f(CRdata_backup\fP role will cause -permission errors like the one below. -.sp -.if n .RS 4 -.nf -.fam C -Error backing up cluster: {"message":"Forbidden. User needs one of the following permissions","permissions":["cluster.fts!read"]} -.fam -.fi -.if n .RE -.sp -When presented with an error message such as the one above, there\(cqs two clear -options. -.sp -The first option is to provide the user with the required credentials using -either the cli, REST API or Couchbase Server WebUI. This can be done by editing -the user and adding the required role. See \f(CRCluster Level\fP for more information -about the required roles. -.sp -Secondly, backing up/restoring the specific service can be disabled. For -backups this must be done when configuring the repository with the \f(CRconfig\fP -command using the \f(CR\-\-disable\fP style flags. For restore, these flags may be used -directly to disable one or more services. See the backup/restore documentation -for more information. -.SS "Cluster Level" -.sp -Backing up/restoring cluster level data requires additional RBAC roles, each of -which is highlighted below: -.sp -Analytics Synonyms -.RS 4 -\f(CRanalytics_admin\fP (Analytics Admin) -.RE -.sp -Eventing Metadata -.RS 4 -\f(CReventing_admin\fP (Eventing Full Admin) -.RE -.sp -FTS Aliases -.RS 4 -\f(CRfts_admin\fP (Search Admin) -.RE -.sp -These additional roles are required since this is cluster level data which may -encompass multiple buckets. -.SH "EXAMPLES" -.sp -The following command is used to take a backup of a Couchbase cluster. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-\-archive /data/backups \-\-repo example -$ cbbackupmgr backup \-a /data/backups \-r example \(rs - \-c couchbase://172.23.10.5 \-u Administrator \-p password -.fam -.fi -.if n .RE -.sp -Once the backup has finished there will be a new directory in the specified -backup repository containing the backed up data. You can see this new directory -using the \fBcbbackupmgr\-info\fP(1) command. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr info \-a /data/backups \-\-all - -Name | UUID | Size | # Repos | -example | 32c97d5f\-821a\-4284\-840b\-9ee7cf8733a3 | 55.56MB | 1 | - -* Name | Size | # Backups | -* Manchester | 55.56MB | 1 | - -+ Backup | Size | Type | Source | Cluster UUID | Range | Events | Aliases | Complete | -+ 2019\-03\-15T13_52_27.18301Z | 55.56MB | FULL | http://172.23.10.5 | c044f5eeb1dc16d0cd49dac29074b5f9 | N/A | 0 | 1 | true | - -\- Bucket | Size | Items | Mutations | Tombstones | Views | FTS | Indexes | CBAS | -\- beer\-sample | 6.85MB | 7303 | 7303 | 0 | 1 | 0 | 1 | 0 | -\- gamesim\-sample | 2.86MB | 586 | 586 | 0 | 1 | 0 | 1 | 0 | -\- travel\-sample | 42.72MB | 31591 | 31591 | 0 | 0 | 0 | 10 | 0 | -.fam -.fi -.if n .RE -.sp -If a backup fails then it is considered a partial backup and the backup client -will not be able to back up any new data until the user decides whether to -resume or purge the partial backup. This decision is made by specifying either -the \-\-resume or the \-\-purge flag on the next invocation of the backup command. -Below is an example of how this process works if the user wants to resume a -backup. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a /data/backups \-r example - -$ cbbackupmgr backup \-a /data/backups \-r example \(rs - \-c 172.23.10.5 \-u Administrator \-p password - -Error backing up cluster: Not all data was backed up due to connectivity -issues. Check to make sure there were no server side failures during -backup. See backup logs for more details on what wasn\*(Aqt backed up. - -$ cbbackupmgr backup \-a /data/backups \-r example \(rs - \-c 172.23.10.5 \-u Administrator \-p password - -Error backing up cluster: Partial backup error 2016\-02\-11T17:00:19.594970735\-08:00 - -$ cbbackupmgr backup \-a /data/backups \-r example \-c 172.23.10.5 \(rs - \-u Administrator \-p password \-\-resume - -Backup successfully completed -.fam -.fi -.if n .RE -.sp -To backup a cluster with a different amount of concurrent clients and decrease -the backup time you can specify the \-\-threads flag. Remember that specifying a -higher number of concurrent clients increases the amount of resources the -cluster uses to complete the backup. Below is an example of using 16 concurrent -clients. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a /data/backups \-r example - -$ cbbackupmgr backup \-a /data/backups \-r example \(rs - \-c 172.23.10.5 \-u Administrator \-p password \-t 16 -.fam -.fi -.if n .RE -.sp -To force the creation of a full backup, use the \f(CR\-\-full\-backup\fP flag. This will -result in cbbackupmgr streaming all the available data again. It\(cqs expected -that more disk space will be used and that the full backup will take longer -than performing an incremental backup (the default). -.sp -In the example below, the first backup is implicitly a full backup, the second -backup is an incremental (where no additional data needed to be backed up) and -the third is a forced full backup (note that we backed up the same amount of -items as the first backup). -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr backup \-a ~/Projects/couchbase\-archive \-r repo \-c 172.20.1.1 \-u Administrator \-p asdasd -Backing up to \*(Aq2021\-02\-09T13_35_15.426546996Z\*(Aq -Copied all data in 2.865028528s (Avg. 10.53MB/Sec) 31591 items / 21.06MB -[=====================================================================================================================================================] 100.00% -Backup completed successfully -Backed up bucket "travel\-sample" succeeded -Mutations backed up: 31591, Mutations failed to backup: 0 -Deletions backed up: 0, Deletions failed to backup: 0 - -$ cbbackupmgr backup \-a ~/Projects/couchbase\-archive \-r repo \-c 172.20.1.1 \-u Administrator \-p asdasd -Backing up to \*(Aq2021\-02\-09T13_35_21.580950579Z\*(Aq -Copied all data in 504.852625ms (Avg. 56.00KB/Sec) 0 items / 56.00KB -[=====================================================================================================================================================] 100.00% -Backup completed successfully -Backed up bucket "travel\-sample" succeeded -Mutations backed up: 0, Mutations failed to backup: 0 -Deletions backed up: 0, Deletions failed to backup: 0 -Skipped due to purge number or conflict resolution: Mutations: 0 Deletions: 0 - -$ cbbackupmgr backup \-a ~/Projects/couchbase\-archive \-r repo \-c 172.20.1.1 \-u Administrator \-p asdasd \-\-fullbackup -Backing up to \*(Aq2021\-02\-09T13_35_24.18890408Z\*(Aq -Copied all data in 2.85286061s (Avg. 10.53MB/Sec) 31591 items / 21.06MB -[=====================================================================================================================================================] 100.00% -Backup completed successfully -Backed up bucket "travel\-sample" succeeded -Mutations backed up: 31591, Mutations failed to backup: 0 -Deletions backed up: 0, Deletions failed to backup: 0 -Skipped due to purge number or conflict resolution: Mutations: 0 Deletions: 0 -.fam -.fi -.if n .RE -.SH "COLLECTION AWARE BACKUPS" -.sp -The 7.0.0 release of Couchbase Server introduced collections supports. The -following section will briefly discuss how this affects backups created by -cbbackupmgr. -.SS "What makes a backup collection aware?" -.sp -cbbackupmgr is not limited to only backing up the cluster version it was -released with; it may be used to backup previous versions of Couchbase -Server. This means that cbbackupmgr is able to create backups which are -collection aware or unaware. -.sp -The distinction between collection aware/unaware is simple; if you backup -a cluster prior to 7.0.0 your backup will be collection unaware. If you -backup a cluster version which supports collections (starting at 7.0.0) -your backup will be collection aware. -.sp -The default behavior of backup for the cluster versions which support -collections is to back up all available scopes/collections in a bucket. -.SS "What if I don\(cqt want to use collections?" -.sp -This is perfectly valid use case and is supported by Couchbase Server and -cbbackupmgr. When interacting with backups created of a collection aware -cluster, cbbackupmgr will only output collection aware information if the -backup contains a non\-default collection manifest (which implies the use of -collections). -.sp -This means you may continue using cbbackupmgr without needing to interact with -collections. Note that you may still need to update/change the flags being -passed to some sub\-commands. For example, when examining a backup, you will -still need to use the \f(CR\-\-collection\-string\fP flag. For example instead of -\f(CR\-\-bucket default\fP you should supply \f(CR\-\-collection\-string default\fP. -.SH "DISCUSSION" -.sp -This command always backs up data incrementally. By using the vbucket sequence -number that is associated with each item, the backup command is able to examine -previous backups in order to determine where the last backup finished. -.sp -When backing up a cluster, data for each bucket is backed up in the following -order: -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRBucket Settings\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRView Definitions\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRGlobal Secondary Index (GSI) Definitions\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRFull\-Text Index Definitions\fP -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -\f(CRKey\-Value Data\fP -.RE -.sp -The backup command will store everything that is persisted to disk on the -Couchbase Server nodes at the time the backup is started. Couchbase server is -consistent at a vBucket level and not across a whole bucket. The tool tries -to provide a strong consistency window by opening all connection to every -node at the same time. Being a distributed system there are times when this is -not possible such as when the cluster is under\-resourced or there are network -issues. These may affect the consistency of the backup across -the vBuckets. \f(CRcbbackupmgr backup\fP provides an Enterprise Edition, developer -preview feature that checks that the backup is inside a consistency window. -.SH "ROLLBACK" -.sp -During a backup, it\(cqs possible for cbbackupmgr to receive a rollback from the -cluster; this will be returned to you (the user) in the form of a message -similar to \f(CRclient received rollback, either purge this backup or create a new -backup repository\fP. -.sp -There are two sensible actions which can be taken at this point (which are -briefly alluded in the above message): -.sp -1) You can rerun the backup using the \f(CR\-\-purge\fP flag, this will remove the -failed backup creating a new incremental backup if possible, otherwise -falling back to creating a full backup. -2) You could create a new backup repository and begin creating backups there, -this method has the advantage of preserving any possible data which was -backed up prior to receiving the rollback. -.sp -In most cases, option one will be sufficient, however, if may be the case that -the partial backup taken by cbbackupmgr contains data which you don\(cqt want to -delete. In this case, option two allows you to retain this data which could be -restored using the \f(CR\-\-restore\-partial\-backups\fP flag. -.sp -Note that the \f(CR\-\-restore\-partial\-backups\fP flag is only supported for local -backups; backups stored in object store which failed due to a rollback must be -purged using the \f(CR\-\-purge\fP flag. -.SH "ENVIRONMENT AND CONFIGURATION VARIABLES" -.sp -CB_CLUSTER -.RS 4 -Specifies the hostname of the Couchbase cluster to connect to. If the -hostname is supplied as a command line argument then this value is -overridden. -.RE -.sp -CB_USERNAME -.RS 4 -Specifies the username for authentication to a Couchbase cluster. If the -username is supplied as a command line argument then this value is -overridden. -.RE -.sp -CB_PASSWORD -.RS 4 -Specifies the password for authentication to a Couchbase cluster. If the -password is supplied as a command line argument then this value is -overridden. -.RE -.sp -CB_CLIENT_CERT -.RS 4 -The path to a client certificate used to authenticate when connecting to a -cluster. May be supplied with \f(CRCB_CLIENT_KEY\fP as an alternative to the -\f(CRCB_USERNAME\fP and \f(CRCB_PASSWORD\fP variables. See the CERTIFICATE AUTHENTICATION -section for more information. -.RE -.sp -CB_CLIENT_CERT_PASSWORD -.RS 4 -The password for the certificate provided to the \f(CRCB_CLIENT_CERT\fP variable, -when using this variable, the certificate/key pair is expected to be in the -PKCS#12 format. See the CERTIFICATE AUTHENTICATION section for more -information. -.RE -.sp -CB_CLIENT_KEY -.RS 4 -The path to the client private key whose public key is contained in the -certificate provided to the \f(CRCB_CLIENT_CERT\fP variable. May be supplied with -\f(CRCB_CLIENT_CERT\fP as an alternative to the \f(CRCB_USERNAME\fP and \f(CRCB_PASSWORD\fP -variables. See the CERTIFICATE AUTHENTICATION section for more information. -.RE -.sp -CB_CLIENT_KEY_PASSWORD -.RS 4 -The password for the key provided to the \f(CRCB_CLIENT_KEY\fP variable, when using -this variable, the key is expected to be in the PKCS#8 format. See the -CERTIFICATE AUTHENTICATION section for more information. -.RE -.sp -CB_ARCHIVE_PATH -.RS 4 -Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden. -.RE -.sp -CB_OBJSTORE_STAGING_DIRECTORY -.RS 4 -Specifies the path to the staging directory. If the \f(CR\-\-obj\-staging\-dir\fP -argument is provided in the command line then this value is overridden. -.RE -.sp -CB_OBJSTORE_REGION -.RS 4 -Specifies the object store region. If the \f(CR\-\-obj\-region\fP argument is provided -in the command line then this value is overridden. -.RE -.sp -CB_OBJSTORE_ACCESS_KEY_ID -.RS 4 -Specifies the object store access key id. If the \f(CR\-\-obj\-access\-key\-id\fP -argument is provided in the command line this value is overridden. -.RE -.sp -CB_OBJSTORE_SECRET_ACCESS_KEY -.RS 4 -Specifies the object store secret access key. If the -\f(CR\-\-obj\-secret\-access\-key\fP argument is provided in the command line this value -is overridden. -.RE -.sp -CB_OBJSTORE_REFRESH_TOKEN -.RS 4 -Specifies the refresh token to use. If the \f(CR\-\-obj\-refresh\-token\fP argument is -provided in the command line, this value is overridden. -.RE -.sp -CB_AWS_ENABLE_EC2_METADATA -.RS 4 -By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint. -.RE -.sp -CB_ENCRYPTION_PASSPHRASE -.RS 4 -Specifies the passphrase used for encryption. -.RE -.sp -CB_KM_KEY_URL -.RS 4 -Specifies the URL identifying the encryption key on the KMS. See -\f(CR\-\-km\-key\-url\fP for the expected format and accepted KMSs. -.RE -.sp -CB_KM_ACCESS_ID -.RS 4 -Specifies the key/user ID used to connect to the KMS. -.RE -.sp -CB_KM_SECRET_ACCESS_KEY -.RS 4 -Specifies the secret key/token used to connect to the KMS. -.RE -.sp -CB_KM_AUTH_FILE -.RS 4 -Specifies a path to a file containing the required credentials to connect to -the KMS. -.RE -.SH "FILES" -.sp -restrictions.json -.RS 4 -Keeps a list of restrictions used to ensure data is not restored to the -place. -.RE -.sp -bucket\-config.json -.RS 4 -Stores the bucket configuration settings for a bucket. -.RE -.sp -views.json -.RS 4 -Stores the view definitions for a bucket. -.RE -.sp -gsi.json -.RS 4 -Stores the global secondary index (GSI) definitions for a bucket. -.RE -.sp -full\-text.json -.RS 4 -Stores the full\-text index definitions for a bucket. -.RE -.sp -shard\-*.fdb -.RS 4 -Stores the key\-value data for a bucket bucket. -.RE -.SH "SEE ALSO" -.sp -\fBcbbackupmgr\-config\fP(1), -\fBcbbackupmgr\-info\fP(1), -\fBcbbackupmgr\-strategies\fP(1), -\fBcbbackupmgr\-cloud\fP(7), -man:cbbackupmgr\-encryption, -\fBcbbackupmgr\-network\-filesystems\fP(7) -.SH "CBBACKUPMGR" -.sp -Part of the \fBcbbackupmgr\fP(1) suite -.SH "AUTHOR" -.sp -Couchbase \ No newline at end of file diff --git a/scripts/cbtools/share/man/man1/cbbackupmgr-collect-logs.1 b/scripts/cbtools/share/man/man1/cbbackupmgr-collect-logs.1 deleted file mode 100644 index 0a33d0ff..00000000 --- a/scripts/cbtools/share/man/man1/cbbackupmgr-collect-logs.1 +++ /dev/null @@ -1,383 +0,0 @@ -'\" t -.\" Title: cbbackupmgr-collect-logs -.\" Author: Couchbase -.\" Generator: Asciidoctor 2.0.16 -.\" Date: 2023-09-20 -.\" Manual: Couchbase Server Manual -.\" Source: Couchbase Server 7.2.2-6401 -.\" Language: English -.\" -.TH "CBBACKUPMGR\-COLLECT\-LOGS" "1" "2023-09-20" "Couchbase Server 7.2.2\-6401" "Couchbase Server Manual" -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.ss \n[.ss] 0 -.nh -.ad l -.de URL -\fI\\$2\fP <\\$1>\\$3 -.. -.als MTO URL -.if \n[.g] \{\ -. mso www.tmac -. am URL -. ad l -. . -. am MTO -. ad l -. . -. LINKSTYLE blue R < > -.\} -.SH "NAME" -cbbackupmgr-collect-logs \- Collects debugging information from a backup archive -.SH "SYNOPSIS" -.sp -.nf -cbbackupmgr collect\-logs [\-\-archive ] [\-\-output\-dir ] - [\-\-redact] [\-\-salt ] [\-\-upload\-host ] - [\-\-customer ] [\-\-upload\-proxy ] - [\-\-ticket ] [\-\-obj\-access\-key\-id ] - [\-\-obj\-cacert ] [\-\-obj\-endpoint ] - [\-\-obj\-no\-ssl\-verify] [\-\-obj\-region ] - [\-\-obj\-staging\-dir ] [\-\-obj\-secret\-access\-key ] - [\-\-s3\-force\-path\-style] -.fi -.br -.SH "DESCRIPTION" -.sp -Collects debugging information to make it easier to debug user issues. This -command collects the logs, the output of the info command, the range.json files -from each backup, the seqnos and failover logs from each backup, and the -backup_meta.json files from each repo. This data is put into a zip file that -contains the date that the collect\-info command was run. This command will only -collect meta from the backup archive. -.SH "OPTIONS" -.sp -Below are a list of parameters for the compact command. -.SS "Required" -.sp -\-a,\-\-archive -.RS 4 -The location of the archive directory. -.RE -.sp -\-o,\-\-output\-dir -.RS 4 -Specifies a path where the zip file that this command writes should be -written. If this flag is not used then the zip file is written to the logs -directory of the archive the command is run against. -.RE -.SS "Optional" -.sp -\-\-redact -.RS 4 -When the redact flag is specified a redacted copy of the logs will be -produced, in the same directory as the unredacted version. The redacted -zip directory will be prefixed by `redacted\-`. -.RE -.sp -\-\-salt -.RS 4 -Specifies the salt to used to hash the data. If possible the salt should be -the same used by Couchbase Server to allow the logs to be easily cross -referenced. The Couchbase Server salt is at the top of the couchbase.log in -the zip file generated by cbcollect_info. If no salt is specified it will be -auto\-generated. -.RE -.sp -\-\-upload\-host -.RS 4 -Specifies the fully\-qualified domain name of the host you want the logs -uploaded to. The protocol prefix of the domain name, https://, is optional. -It is the default\-only supported protocol. -.RE -.sp -\-\-customer -.RS 4 -Specifies the customer name. This value must be a string whose maximum length -is 50 characters. Only the following characters can be used: [A\-Za\-z0\-9_.\-]. -If any other characters are included, the request is rejected. -.RE -.sp -\-\-upload\-proxy -.RS 4 -Specifies a proxy for upload. -.RE -.sp -\-\-ticket -.RS 4 -Specifies the Couchbase Support ticket\-number. The value must be a string -with a maximum length of 7 characters, containing only digits in the range of -0\-9. -.RE -.SS "Cloud integration" -.sp -Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0. -.sp -Multiple cloud providers are supported, see the list below for more information. -.sp -.RS 4 -.ie n \{\ -\h'-04' 1.\h'+01'\c -.\} -.el \{\ -. sp -1 -. IP " 1." 4.2 -.\} -Supported -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -AWS S3 (\f(CRs3://\fP) -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -GCP Google Storage (\f(CRgs://\fP) -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -Azure Blob Storage in 7.1.2+ (\f(CRaz://\fP) -.RE -.RE -.SS "Required" -.sp -\-\-obj\-staging\-dir -.RS 4 -When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it\(cqs not treated as a persistent store) and is -only used during the backup. NOTE: Do not use \f(CR/tmp\fP as the \f(CRobj\-staging\-dir\fP. -See \f(CRDisk requirements\fP in \fBcbbackupmgr\-cloud\fP(7) for more information. -.RE -.SS "Optional" -.sp -\-\-obj\-access\-key\-id -.RS 4 -The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -\f(CRCB_OBJSTORE_ACCESS_KEY_ID\fP environment variable. -.sp -When using AWS, this option expects an access key id. See -\c -.URL "https://docs.aws.amazon.com/general/latest/gr/aws\-sec\-cred\-types.html#access\-keys\-and\-secret\-access\-keys" "" -for more information. -.sp -When using Azure, this option expects an account name. See -\c -.URL "https://docs.microsoft.com/en\-us/azure/storage/common/storage\-account\-overview#storage\-account\-endpoints" "" -for more information. -.sp -When using GCP, this option expects a client id. See -.URL "https://cloud.google.com/storage/docs/authentication" "" "" -for more information. -.RE -.sp -\-\-obj\-cacert -.RS 4 -Specifies a CA certificate that will be used to verify the identity of the -object store being connected to. -.RE -.sp -\-\-obj\-endpoint -.RS 4 -The host/address of your object store. -.RE -.sp -\-\-obj\-no\-ssl\-verify -.RS 4 -Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man\-in\-the\-middle attack. -.RE -.sp -\-\-obj\-region -.RS 4 -The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information. -.RE -.sp -\-\-obj\-secret\-access\-key -.RS 4 -The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -\f(CRCB_OBJSTORE_SECRET_ACCESS_KEY\fP environment variable. -.sp -When using AWS, this option expects a secret access key. See -\c -.URL "https://docs.aws.amazon.com/general/latest/gr/aws\-sec\-cred\-types.html#access\-keys\-and\-secret\-access\-keys" "" -for more information. -.sp -When using Azure, this option expects an account key. See -\c -.URL "https://docs.microsoft.com/en\-us/azure/storage/common/storage\-account\-keys\-manage?tabs=azure\-portal" "" -for more information. -.sp -When using GCP, this option expects a client secret. See -.URL "https://cloud.google.com/storage/docs/authentication" "" "" -for more information. -.RE -.sp -\-\-obj\-log\-level -.RS 4 -Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below. -.sp -The valid options for the AWS SDK are \f(CRdebug\fP, \f(CRdebug\-with\-signing\fP, -\f(CRdebug\-with\-body\fP, \f(CRdebug\-with\-request\-retries\fP, \f(CRdebug\-with\-request\-errors\fP, -and \f(CRdebug\-with\-event\-stream\-body\fP. -.sp -The valid options for the Azure SDK are \f(CRinfo\fP, \f(CRdebug\fP, \f(CRdebug\-with\-request\-retries\fP and -\f(CRdebug\-with\-request\-retries\-and\-lro\fP. -.sp -The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future. -.RE -.sp -\-\-obj\-auth\-by\-instance\-metadata -.RS 4 -Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints. -.sp -By default, this option is disabled for AWS. -.sp -By default, this option is enabled for Azure. -.sp -By default, this option is enabled for GCP. -.RE -.sp -\-\-obj\-auth\-file -.RS 4 -GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The \f(CR\-\-obj\-auth\-file\fP flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers. -.RE -.sp -\-\-obj\-refresh\-token -.RS 4 -GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage. -.RE -.SS "AWS S3 Options" -.SS "Optional" -.sp -\-\-s3\-force\-path\-style -.RS 4 -By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores. -.RE -.SH "EXAMPLES" -.sp -To generate debugging information for an archive dir "/data/backup" and output -the file to "/data/output/logs" run the following command. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr collect\-logs \-a /data/backup -.fam -.fi -.if n .RE -.sp -To generate debugging information for an archive dir "/data/backup" and output -the file to "/tmp" run the following command. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr collect\-logs \-a /data/backup \-o /tmp -.fam -.fi -.if n .RE -.sp -To upload the logs directly to support use: -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr collect\-logs \-a /data/backup \-o /tmp \(rs - \-\-upload\-host uploads.couchbase.com \(rs -\-\-customer CUSTOMER_NAME \-\-ticket 378211 -.fam -.fi -.if n .RE -.SH "ENVIRONMENT AND CONFIGURATION VARIABLES" -.sp -CB_ARCHIVE_PATH -.RS 4 -Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden. -.RE -.sp -CB_OBJSTORE_STAGING_DIRECTORY -.RS 4 -Specifies the path to the staging directory. If the \f(CR\-\-obj\-staging\-dir\fP -argument is provided in the command line then this value is overridden. -.RE -.sp -CB_OBJSTORE_REGION -.RS 4 -Specifies the object store region. If the \f(CR\-\-obj\-region\fP argument is provided -in the command line then this value is overridden. -.RE -.sp -CB_OBJSTORE_ACCESS_KEY_ID -.RS 4 -Specifies the object store access key id. If the \f(CR\-\-obj\-access\-key\-id\fP -argument is provided in the command line this value is overridden. -.RE -.sp -CB_OBJSTORE_SECRET_ACCESS_KEY -.RS 4 -Specifies the object store secret access key. If the -\f(CR\-\-obj\-secret\-access\-key\fP argument is provided in the command line this value -is overridden. -.RE -.sp -CB_OBJSTORE_REFRESH_TOKEN -.RS 4 -Specifies the refresh token to use. If the \f(CR\-\-obj\-refresh\-token\fP argument is -provided in the command line, this value is overridden. -.RE -.sp -CB_AWS_ENABLE_EC2_METADATA -.RS 4 -By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint. -.RE -.SH "SEE ALSO" -.sp -\fBcbbackupmgr\-info\fP(1), -.SH "CBBACKUPMGR" -.sp -Part of the \fBcbbackupmgr\fP(1) suite -.SH "AUTHOR" -.sp -Couchbase \ No newline at end of file diff --git a/scripts/cbtools/share/man/man1/cbbackupmgr-compact.1 b/scripts/cbtools/share/man/man1/cbbackupmgr-compact.1 deleted file mode 100644 index c06c1a1b..00000000 --- a/scripts/cbtools/share/man/man1/cbbackupmgr-compact.1 +++ /dev/null @@ -1,155 +0,0 @@ -'\" t -.\" Title: cbbackupmgr-compact -.\" Author: Couchbase -.\" Generator: Asciidoctor 2.0.16 -.\" Date: 2023-09-20 -.\" Manual: Couchbase Server Manual -.\" Source: Couchbase Server 7.2.2-6401 -.\" Language: English -.\" -.TH "CBBACKUPMGR\-COMPACT" "1" "2023-09-20" "Couchbase Server 7.2.2\-6401" "Couchbase Server Manual" -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.ss \n[.ss] 0 -.nh -.ad l -.de URL -\fI\\$2\fP <\\$1>\\$3 -.. -.als MTO URL -.if \n[.g] \{\ -. mso www.tmac -. am URL -. ad l -. . -. am MTO -. ad l -. . -. LINKSTYLE blue R < > -.\} -.SH "NAME" -cbbackupmgr-compact \- Compacts a backup to free disk space -.SH "SYNOPSIS" -.sp -.nf -cbbackupmgr compact [\-\-archive ] [\-\-repo ] - [\-\-backup ] -.fi -.br -.SH "DESCRIPTION" -.sp -Compacting backups created with a 6.6.0+ version of \f(CRcbbackupmgr\fP is no longer -supported/required due to changes in the storage format which significantly -reduced backup fragmentation. -.sp -Compacts a backup in a backup repository in order to free up disk space. Should -be run on each backup after it completes. The amount of freed disk space is -displayed when the compaction completes. -.SH "OPTIONS" -.sp -Below are a list of parameters for the compact command. -.SS "Required" -.sp -\-a,\-\-archive -.RS 4 -The location of the archive directory. -.RE -.sp -\-r,\-\-repo -.RS 4 -The name of the backup repository where the backup to be compacted is -located. -.RE -.sp -\-\-backup -.RS 4 -The name of the backup in the backup repository to compact or an index -value which references an incremental backup to compact. Valid index values -are any positive integer, "oldest", and "latest". If a positive integer is -used then it should reference the index of the incremental backup to compact -starting from the oldest to the most recent backup. For example, "1" -corresponds to the oldest backup, "2" corresponds to the second oldest backup, -and so on. Specifying "oldest" means that the index of the oldest backup -should be used and specifying "latest" means the index of the most recent -backup should be used. -.RE -.sp -\-t,\-\-threads -.RS 4 -Specify the number of files to be compacted in parallel. By default it is set -to 1, but it is recommended to set to match the number of CPUs in the system -although it can vary depending on the available RAM. -.RE -.SH "EXAMPLES" -.sp -The compact command is used to compact backups in order to free disk space and -should be run after each backup. Below is an example of how to run the -compaction command. The /data/backup directory is used as the archive directory -and the backup repository is named "example". -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a /data/backup \-r example -.fam -.fi -.if n .RE -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr backup \-a /data/backup \-r example \(rs - \-c 127.0.0.1:8091 \-u Administrator \-p password -.fam -.fi -.if n .RE -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr compact \-a /data/backup \-r example \(rs - \-\-backup 2016\-02\-12T12_45_33.329408761\-08_00 -.fam -.fi -.if n .RE -.sp -.if n .RS 4 -.nf -.fam C -Compaction succeeded, 58589184 bytes freed -.fam -.fi -.if n .RE -.SH "DISCUSSION" -.sp -The compact command can be run after a backup to reclaim any fragmented disk -space in the backup data files. It should be noted however that compaction may -be run during the backup itself if the fragmentation in the data file begins to -grow too big. Although the compaction can run during the backup it is not -guaranteed to be run once the backup has completed, hence the reason for the -compact command. -.SH "ENVIRONMENT AND CONFIGURATION VARIABLES" -.sp -CB_ARCHIVE_PATH -.RS 4 -Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden. -.RE -.SH "FILES" -.sp -shard\-*.fdb -.RS 4 -The compact command will compact all shard\-*.fdb files in the backup -specified by the compact command. -.RE -.SH "SEE ALSO" -.sp -\fBcbbackupmgr\-backup\fP(1), -\fBcbbackupmgr\-info\fP(1), -\fBcbbackupmgr\-strategies\fP(1) -.SH "CBBACKUPMGR" -.sp -Part of the \fBcbbackupmgr\fP(1) suite -.SH "AUTHOR" -.sp -Couchbase \ No newline at end of file diff --git a/scripts/cbtools/share/man/man1/cbbackupmgr-config.1 b/scripts/cbtools/share/man/man1/cbbackupmgr-config.1 deleted file mode 100644 index 8ceda64f..00000000 --- a/scripts/cbtools/share/man/man1/cbbackupmgr-config.1 +++ /dev/null @@ -1,629 +0,0 @@ -'\" t -.\" Title: cbbackupmgr-config -.\" Author: Couchbase -.\" Generator: Asciidoctor 2.0.16 -.\" Date: 2023-09-20 -.\" Manual: Couchbase Server Manual -.\" Source: Couchbase Server 7.2.2-6401 -.\" Language: English -.\" -.TH "CBBACKUPMGR\-CONFIG" "1" "2023-09-20" "Couchbase Server 7.2.2\-6401" "Couchbase Server Manual" -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.ss \n[.ss] 0 -.nh -.ad l -.de URL -\fI\\$2\fP <\\$1>\\$3 -.. -.als MTO URL -.if \n[.g] \{\ -. mso www.tmac -. am URL -. ad l -. . -. am MTO -. ad l -. . -. LINKSTYLE blue R < > -.\} -.SH "NAME" -cbbackupmgr-config \- Creates and configures a new backup repository -.SH "SYNOPSIS" -.sp -.nf -cbbackupmgr config [\-\-archive ] [\-\-repo ] - [\-\-include\-data ] - [\-\-exclude\-data ] - [\-\-disable\-bucket\-config] [\-\-disable\-views] [\-\-disable\-eventing] - [\-\-disable\-gsi\-indexes] [\-\-disable\-ft\-indexes] - [\-\-disable\-data] [\-\-vbucket\-filter ] - [\-\-disable\-cluster\-analytics] [\-\-disable\-analytics] [\-\-disable\-ft\-alias] - [\-\-disable\-bucket\-query] [\-\-disable\-cluster\-query] - [\-\-obj\-access\-key\-id ] [\-\-obj\-cacert ] - [\-\-obj\-endpoint ] [\-\-obj\-no\-ssl\-verify] - [\-\-obj\-region ] [\-\-obj\-staging\-dir ] - [\-\-obj\-secret\-access\-key ] - [\-\-s3\-force\-path\-style] [\-\-s3\-log\-level ] - [\-\-readme\-author ] [\-\-point\-in\-time] - [\-\-encrypted] [\-\-passphrase ] - [\-\-encryption\-algo ] [\-\-derivation\-algo ] - [\-\-km\-key\-url ] [\-\-km\-endpoint ] - [\-\-km\-region ] [\-\-km\-access\-key\-id ] - [\-\-km\-secret\-access\-key ] [\-\-km\-auth\-file ] -.fi -.br -.SH "DESCRIPTION" -.sp -Creates a new backup repository with a new backup configuration. The -configuration created is used for all backups in this backup repository. -.sp -By default a backup configuration is created that will backup all buckets in the -cluster. Each bucket will have its bucket configuration, views definitions, gsi -index definitions, full\-text index definitions, and data backed up. Specifying -various flags, the config command can modify the configuration to backup a -subset of the data. -.sp -Once a backup repository is created its configuration cannot be changed. -.SH "OPTIONS" -.sp -Below are a list of required and optional parameters for the config command. -.SS "Required" -.sp -\-a,\-\-archive -.RS 4 -The directory where the new backup repository will be created. If it does -not already exist, an attempt will be made to create it. When configuring -an archive in S3 you must prefix the archive path with \f(CRs3://${BUCKET_NAME}/\fP. -.RE -.sp -\-r,\-\-repo -.RS 4 -The name of the new backup repository. -.RE -.SS "Optional" -.sp -\-\-include\-data -.RS 4 -Modifies the repository configuration to backup only the data specified in -the . This flag takes a comma separated list of -collection strings and can\(cqt be specified at the same time as -\f(CR\-\-exclude\-data\fP. Note that including data at the scope/collection level is -an Enterprise Edition feature. -.RE -.sp -\-\-exclude\-data -.RS 4 -Modifies the repository configuration to skip restoring the data specified in -the . This flag takes a comma separated list of -collection strings and can\(cqt be specified at the same time as -\f(CR\-\-include\-data\fP. Note that excluding data at the scope/collection level is -an Enterprise Edition feature. -.RE -.sp -\-\-disable\-views -.RS 4 -Modifies the repository configuration to disable backing up view definitions -for all buckets. -.RE -.sp -\-\-disable\-gsi\-indexes -.RS 4 -Modifies the repository configuration to disable backing up gsi index -definitions for all buckets. -.RE -.sp -\-\-disable\-ft\-indexes -.RS 4 -Modifies the repository configuration to disable backing up full\-text index -definitions for all buckets. -.RE -.sp -\-\-disable\-ft\-alias -.RS 4 -Modifies the repository configuration to disable backing up full\-text alias -definitions. -.RE -.sp -\-\-disable\-data -.RS 4 -Modifies the repository configuration to disable backing up all key\-value -data for all buckets. -.RE -.sp -\-\-disable\-cluster\-analytics -.RS 4 -Modifies the repository configuration to disable backing up the cluster level -analytics data. For example, the Synonyms are backed up at the cluster level. -.RE -.sp -\-\-disable\-analytics -.RS 4 -Modifies the repository configuration to disable backing up analytics data. -.RE -.sp -\-\-disable\-eventing -.RS 4 -Modifies the repository configuration to disable backing up the Eventing -service metadata. -.RE -.sp -\-\-disable\-bucket\-query -.RS 4 -Modifies the repository configuration to disable backing up Query Service -metadata for all buckets. -.RE -.sp -\-\-disable\-cluster\-query -.RS 4 -Modifies the repository configuration to disable backing up the cluster level -Query Service metadata. -.RE -.sp -\-\-vbucket\-filter -.RS 4 -Specifies a list of VBuckets that should be backed up in the repo being -created. VBuckets are specified as a comma separated list of integers. If -this parameter is not set then all vbuckets are backed up. The comma -separated list can also take sequences denoted as e.g: 0\-5 this is equivalent -to 0,1,2,3,4,5. -.RE -.sp -\-\-readme\-author -.RS 4 -Specifies the name of the creator of the repository. This information -will be recorded in a README file at the top\-level of the repository. -.RE -.sp -\-\-point\-in\-time -.RS 4 -(BETA) This enables Point in Time feature. Which is currently in Beta and -is not supported, this should only be used in test environments. When it\(cqs -enabled cbbackupmgr will get historical data from the cluster. This will -allow restoring to a single point in time. This will increase the amount of -space needed to store the backup. The cluster also has to have Point In -Time configured to use this feature. -.RE -.SS "Cloud integration" -.sp -Native cloud integration is an Enterprise Edition feature which was introduced -in Couchbase Server 6.6.0. -.sp -Multiple cloud providers are supported, see the list below for more information. -.sp -.RS 4 -.ie n \{\ -\h'-04' 1.\h'+01'\c -.\} -.el \{\ -. sp -1 -. IP " 1." 4.2 -.\} -Supported -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -AWS S3 (\f(CRs3://\fP) -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -GCP Google Storage (\f(CRgs://\fP) -.RE -.sp -.RS 4 -.ie n \{\ -\h'-04'\(bu\h'+03'\c -.\} -.el \{\ -. sp -1 -. IP \(bu 2.3 -.\} -Azure Blob Storage in 7.1.2+ (\f(CRaz://\fP) -.RE -.RE -.SS "Required" -.sp -\-\-obj\-staging\-dir -.RS 4 -When performing an operation on an archive which is located in the cloud such -as AWS, the staging directory is used to store local meta data files. This -directory can be temporary (it\(cqs not treated as a persistent store) and is -only used during the backup. NOTE: Do not use \f(CR/tmp\fP as the \f(CRobj\-staging\-dir\fP. -See \f(CRDisk requirements\fP in \fBcbbackupmgr\-cloud\fP(7) for more information. -.RE -.SS "Optional" -.sp -\-\-obj\-access\-key\-id -.RS 4 -The access key id which has access to your chosen object store. This option -can be omitted when using the shared config functionality provided by your -chosen object store. Can alternatively be provided using the -\f(CRCB_OBJSTORE_ACCESS_KEY_ID\fP environment variable. -.sp -When using AWS, this option expects an access key id. See -\c -.URL "https://docs.aws.amazon.com/general/latest/gr/aws\-sec\-cred\-types.html#access\-keys\-and\-secret\-access\-keys" "" -for more information. -.sp -When using Azure, this option expects an account name. See -\c -.URL "https://docs.microsoft.com/en\-us/azure/storage/common/storage\-account\-overview#storage\-account\-endpoints" "" -for more information. -.sp -When using GCP, this option expects a client id. See -.URL "https://cloud.google.com/storage/docs/authentication" "" "" -for more information. -.RE -.sp -\-\-obj\-cacert -.RS 4 -Specifies a CA certificate that will be used to verify the identity of the -object store being connected to. -.RE -.sp -\-\-obj\-endpoint -.RS 4 -The host/address of your object store. -.RE -.sp -\-\-obj\-no\-ssl\-verify -.RS 4 -Skips the SSL verification phase when connecting to the object store. -Specifying this flag will allow a connection using SSL encryption, but you -are vulnerable to a man\-in\-the\-middle attack. -.RE -.sp -\-\-obj\-region -.RS 4 -The region in which your bucket/container resides. For AWS this option may be -omitted when using the shared config functionality. See the AWS section of -the cloud documentation for more information. -.RE -.sp -\-\-obj\-secret\-access\-key -.RS 4 -The secret access key which has access to you chosen object store. This -option can be omitted when using the shared config functionality provided by -your chosen object store. Can alternatively be provided using the -\f(CRCB_OBJSTORE_SECRET_ACCESS_KEY\fP environment variable. -.sp -When using AWS, this option expects a secret access key. See -\c -.URL "https://docs.aws.amazon.com/general/latest/gr/aws\-sec\-cred\-types.html#access\-keys\-and\-secret\-access\-keys" "" -for more information. -.sp -When using Azure, this option expects an account key. See -\c -.URL "https://docs.microsoft.com/en\-us/azure/storage/common/storage\-account\-keys\-manage?tabs=azure\-portal" "" -for more information. -.sp -When using GCP, this option expects a client secret. See -.URL "https://cloud.google.com/storage/docs/authentication" "" "" -for more information. -.RE -.sp -\-\-obj\-log\-level -.RS 4 -Set the log level for the cloud providers SDK. By default logging will be -disabled. Valid options are cloud provider specific and are listed below. -.sp -The valid options for the AWS SDK are \f(CRdebug\fP, \f(CRdebug\-with\-signing\fP, -\f(CRdebug\-with\-body\fP, \f(CRdebug\-with\-request\-retries\fP, \f(CRdebug\-with\-request\-errors\fP, -and \f(CRdebug\-with\-event\-stream\-body\fP. -.sp -The valid options for the Azure SDK are \f(CRinfo\fP, \f(CRdebug\fP, \f(CRdebug\-with\-request\-retries\fP and -\f(CRdebug\-with\-request\-retries\-and\-lro\fP. -.sp -The Google Storage SDK does not expose advanced logging configuration meaning -this option is explicitly ignored, however, this behavior may change in the -future. -.RE -.sp -\-\-obj\-auth\-by\-instance\-metadata -.RS 4 -Depending on the cloud provider, using instance metadata for authentication -is disabled by default. Supplying this flag will allow the fetching -credentials/auth tokens from (VM) internal instance metadata endpoints. -.sp -By default, this option is disabled for AWS. -.sp -By default, this option is enabled for Azure. -.sp -By default, this option is enabled for GCP. -.RE -.sp -\-\-obj\-auth\-file -.RS 4 -GCP offers the ability to use a file which contains credentials which will be -used to perform authentication. The \f(CR\-\-obj\-auth\-file\fP flag accepts a path to -an authentication file. This flag is unsupported for the AWS/Azure cloud -providers. -.RE -.sp -\-\-obj\-refresh\-token -.RS 4 -GCP requires a refresh token when using static credentials, this will be used -to refresh oauth2 tokens when accessing remote storage. -.RE -.SS "AWS S3 Options" -.SS "Optional" -.sp -\-\-s3\-force\-path\-style -.RS 4 -By default the updated virtual style paths will be used when interfacing with -AWS S3. This option will force the AWS SDK to use the alternative path style -URLs which are often required by S3 compatible object stores. -.RE -.SS "Encryption (Developer Preview)" -.sp -\-\-encrypted -.RS 4 -Enables configuring an encrypted repository. -.RE -.sp -\-\-encryption\-algo -.RS 4 -The encryption algorithm to use. Currently supported are -[AES256CBC, AES256GCM]. Defaults to AES256GCM. -.RE -.sp -\-\-derivation\-algo -.RS 4 -The derivation algorithm used when running with passphrase that creates -the repository encryption key. Currently supported [ARGON2ID, PBKDF2]. -Defaults to ARGON2ID. -.RE -.sp -\-\-passphrase -.RS 4 -Passphrase can be used instead of an external key manager. This is not -supported on production and should only be used in development or testing. -.RE -.sp -\-\-km\-key\-url -.RS 4 -Provides the Key Identifier in the external Key Management system. Currently -supported KMSs are AWS KMS, GCP KMS, Azure KeyVault, HashiCorp Vault Transit -secrets engine. The option can also be provided using the environmental -variable \f(CRCB_KM_KEY_URL\fP. For more on how to authenticate using the different -providers see \fBcbbackupmgr\-encryption\fP(7). -.sp -For AWS the expected key format is \f(CRawskms://\fP, for example -\f(CRawskms://alias\(rskeyAlias\fP. -.sp -For GCP the expected key format is \f(CRgcpkms://\fP, for example -\f(CRgcpkms://projects/project\-id/locations/location/keyRings/keyring/cryptoKeys/key\fP. -.sp -For Azure key vault the expected key format is \f(CRazurekeyvault://\fP -for example: -\f(CRazurekeyvault://vault\-name.vault.azure.net/object\-type/object\-name/object\-version\fP. -.sp -For HashiCorp Vault the expected format is \f(CRhashivaults:///\fP -for example: -\f(CRhashivaults://127.0.0.1:8200/keyName\fP. -.RE -.sp -\-\-km\-region -.RS 4 -Required when using AWS KMS, it allows you to set the key region. -.RE -.sp -\-\-km\-endpoint -.RS 4 -The host or address to use as your KMS. It will override the default SDK one. -.RE -.sp -\-\-km\-access\-key\-id -.RS 4 -The user ID used to connect to the key management service. It can also be -provided via \f(CRCB_KM_ACCESS_KEY_ID\fP environmental variable. Please refer -to \fBcbbackupmgr\-encryption\fP(7) for the required authentication for each -provider. -.RE -.sp -\-\-km\-secret\-access\-key -.RS 4 -The key used to connect to the key management service. It can also be -provided via the \f(CRCB_KM_SECRET_ACCESS_KEY\fP environmental variable. Please -refer to \fBcbbackupmgr\-encryption\fP(7) for the required authentication for -each provider. -.RE -.sp -\-\-km\-auth\-file -.RS 4 - The path to a file containing the authentication credentials for the key -management service. It can also be provided via the \f(CRCB_KM_AUTH_FILE\fP -environmental variable. Please refer to \fBcbbackupmgr\-encryption\fP(7) for the -required authentication for each provider. -:!config: -.RE -.SH "EXAMPLES" -.sp -The config command is used to create a backup repository and define the -repositories backup configuration. In the examples below, the backup archive is -located at /data/backups. Since this is the first backup repository we are -creating in a new backup archive, we need to ensure that /data/backups is an -empty directory. Archives are created automatically if an archive doesn\(cqt -already exist at the archive path, but are only created if the directory at that -path is empty. In order to create a backup repository called "example" with the -default configuration use the following command: -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a /data/backups \-r example -.fam -.fi -.if n .RE -.sp -Upon creation of a new backup repository there will be a new directory in the -backup archive containing a backup configuration. You can see this new directory -using the cbbackupmgr\-info[1] command. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr info \-a ~/data/backups -Name | UUID | Size | # Repos | -backups | 0db3337c\-96b0\-4b3a\-a7fb\-bbfd53790e5f | 0B | 1 | - -* Name | Size | # Backups | -* example | 0B | 0 | -.fam -.fi -.if n .RE -.sp -Using the optional parameters of the create command, you can modify the backup -configuration settings. To create a backup repository with a backup -configuration that only backs up the buckets "airline_data" and "ticket_prices" -and does not back up bucket configuration data, you can run the following: -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a /data/backups \-r example \(rs - \-\-include\-data airline_data,ticket_prices \-\-disable\-bucket\-config -.fam -.fi -.if n .RE -.sp -To create a backup repository with a backup configuration that backs up only -data on all buckets, you can run the following command: -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a /data/backups \-r example \-\-disable\-bucket\-config \(rs - \-\-disable\-views \-\-disable\-gsi\-indexes \-\-disable\-ft\-indexes -.fam -.fi -.if n .RE -.sp -When creating an archive in an object store such as S3 due care must be taken -to ensure that the provided directory is empty and can be managed by cbbackupmgr. -.sp -.if n .RS 4 -.nf -.fam C -$ cbbackupmgr config \-a s3://bucket/archive \-r repo \-\-obj\-staging\-dir /staging -.fam -.fi -.if n .RE -.SH "DISCUSSION" -.sp -Though not required, it is recommended that there is a single backup repository -per cluster. Backup repositories are managed so that all backups can be taken -incrementally and merged together as backup data ages. Backing up in this manner -allows backups to transfer the least amount of data necessary which reduces back -up time and cluster resource usage. For more details on backup strategies see -cbbackupmgr\-strategies[7]. -.sp -When a backup repository is created, it should only be modified by the -cbbackupmgr utility. Any modifications done outside of this utility can cause -corruption of backup files. -.SH "ENVIRONMENT AND CONFIGURATION VARIABLES" -.sp -CB_ARCHIVE_PATH -.RS 4 -Specifies the path to the backup archive. If the archive path is supplied as -a command line argument then this value is overridden. -.RE -.sp -CB_OBJSTORE_STAGING_DIRECTORY -.RS 4 -Specifies the path to the staging directory. If the \f(CR\-\-obj\-staging\-dir\fP -argument is provided in the command line then this value is overridden. -.RE -.sp -CB_OBJSTORE_REGION -.RS 4 -Specifies the object store region. If the \f(CR\-\-obj\-region\fP argument is provided -in the command line then this value is overridden. -.RE -.sp -CB_OBJSTORE_ACCESS_KEY_ID -.RS 4 -Specifies the object store access key id. If the \f(CR\-\-obj\-access\-key\-id\fP -argument is provided in the command line this value is overridden. -.RE -.sp -CB_OBJSTORE_SECRET_ACCESS_KEY -.RS 4 -Specifies the object store secret access key. If the -\f(CR\-\-obj\-secret\-access\-key\fP argument is provided in the command line this value -is overridden. -.RE -.sp -CB_OBJSTORE_REFRESH_TOKEN -.RS 4 -Specifies the refresh token to use. If the \f(CR\-\-obj\-refresh\-token\fP argument is -provided in the command line, this value is overridden. -.RE -.sp -CB_AWS_ENABLE_EC2_METADATA -.RS 4 -By default cbbackupmgr will disable fetching EC2 instance metadata. Setting -this environment variable to true will allow the AWS SDK to fetch metadata -from the EC2 instance endpoint. -.RE -.sp -CB_ENCRYPTION_PASSPHRASE -.RS 4 -Specifies the passphrase used for encryption. -.RE -.sp -CB_KM_KEY_URL -.RS 4 -Specifies the URL identifying the encryption key on the KMS. See -\f(CR\-\-km\-key\-url\fP for the expected format and accepted KMSs. -.RE -.sp -CB_KM_ACCESS_ID -.RS 4 -Specifies the key/user ID used to connect to the KMS. -.RE -.sp -CB_KM_SECRET_ACCESS_KEY -.RS 4 -Specifies the secret key/token used to connect to the KMS. -.RE -.sp -CB_KM_AUTH_FILE -.RS 4 -Specifies a path to a file containing the required credentials to connect to -the KMS. -.RE -.SH "FILES" -.sp -backup\-meta.json -.RS 4 -The config command creates a backup configuration file in the backup -repository called backup\-meta.json. This file contains the backup -configuration for all backups run in the backup repository it was created -in. It should never be modified and treated as a read\-only file. -.RE -.SH "SEE ALSO" -.sp -\fBcbbackupmgr\-info\fP(1), -\fBcbbackupmgr\-strategies\fP(1), -\fBcbbackupmgr\-cloud\fP(7), -\fBcbbackupmgr\-encryption\fP(7) -.SH "CBBACKUPMGR" -.sp -Part of the \fBcbbackupmgr\fP(1) suite -.SH "AUTHOR" -.sp -Couchbase \ No newline at end of file diff --git a/scripts/cbtools/share/man/man1/cbbackupmgr-examine.1 b/scripts/cbtools/share/man/man1/cbbackupmgr-examine.1 deleted file mode 100644 index 937d4910..00000000 --- a/scripts/cbtools/share/man/man1/cbbackupmgr-examine.1 +++ /dev/null @@ -1,873 +0,0 @@ -'\" t -.\" Title: cbbackupmgr-examine -.\" Author: Couchbase -.\" Generator: Asciidoctor 2.0.16 -.\" Date: 2023-09-20 -.\" Manual: Couchbase Server Manual -.\" Source: Couchbase Server 7.2.2-6401 -.\" Language: English -.\" -.TH "CBBACKUPMGR\-EXAMINE" "1" "2023-09-20" "Couchbase Server 7.2.2\-6401" "Couchbase Server Manual" -.ie \n(.g .ds Aq \(aq -.el .ds Aq ' -.ss \n[.ss] 0 -.nh -.ad l -.de URL -\fI\\$2\fP <\\$1>\\$3 -.. -.als MTO URL -.if \n[.g] \{\ -. mso www.tmac -. am URL -. ad l -. . -. am MTO -. ad l -. . -. LINKSTYLE blue R < > -.\} -.SH "NAME" -cbbackupmgr-examine \- Searches one or more backups by key for a specific document -.SH "SYNOPSIS" -.sp -.nf -cbbackupmgr examine [\-\-archive ] [\-\-repo ] - [\-\-collection\-string ] [\-\-key ] - [\-\-start start] [\-\-end end] [\-\-search\-partial\-backups] - [\-\-json] [\-\-no\-meta] [\-\-no\-xattrs] [\-\-no\-value] - [\-\-hex\-meta] [\-\-hex\-xattrs] [\-\-hex\-value] - [\-\-obj\-access\-key\-id ] - [\-\-obj\-cacert ] [\-\-obj\-endpoint ] - [\-\-obj\-no\-ssl\-verify] [\-\-obj\-region ] - [\-\-obj\-staging\-dir ] - [\-\-obj\-secret\-access\-key ] - [\-\-s3\-force\-path\-style] [\-\-s3\-log\-level ] - [\-\-passphrase ] [\-\-km\-key\-url ] - [\-\-km\-endpoint ] [\-\-km\-region ] - [\-\-km\-access\-key\-id ] [\-\-km\-secret\-access\-key ] - [\-\-km\-auth\-file ] [\-\-point\-in\-time