Merge pull request #16 from cloudfoundry/develop

Promote Develop

CONTRIBUTING.md

@@ -0,0 +1,22 @@
+# Contributing
+
+## Submitting an Issue
+We use the [GitHub issue tracker](https://github.com/cloudfoundry/bosh-gcscli/issues) to track bugs and features.
+Before submitting a bug report or feature request, check to make sure it hasn't already been submitted. You can indicate
+support for an existing issue by voting it up. When submitting a bug report, please include a
+[Gist](http://gist.github.com/) that includes a stack trace and any details that may be necessary to reproduce the bug,
+including your gem version, Ruby version, and operating system. Ideally, a bug report should include a pull request with failing specs.
+
+## Submitting a Pull Request
+You can add a feature or bug-fix via pull request.
+1. Fork the project
+1. Create a branch for your feature or fix from the `develop` branch. Replace `your-feature-name` with a description of your feature or fix:
+   ```
+   git checkout -b your-feature-name develop
+   ```
+1. Implement your feature or bug fix
+1. Commit and push your changes
+1. Submit a pull request to the `develop` branch of the [bosh-gcscli] repository. PRs to the master branch are not accepted.
+1. Unit tests and a BOSH release are created for each PR. You should see the status of your PR change to "pending" within a few minutes of submitting it, and then to "passed" or "failed" within 10 minutes.
+
+[bosh-gcscli]: https://github.com/cloudfoundry/bosh-gcscli/

Makefile

@@ -91,15 +91,16 @@ prep-gcs: regional-bucket multiregional-bucket public-bucket
 
 # Remove all buckets listed in $StorageClass.lock files.
 clean-gcs:
-	test -s "multiregional.lock" && \
-	test -s "regional.lock" && \
-	test -s "public.lock"
+	@test -s "multiregional.lock" && test -s "regional.lock" && test -s "public.lock"
+	@gsutil rm "gs://$$(cat regional.lock)/*" || true
 	@gsutil rb "gs://$$(cat regional.lock)"
-	rm regional.lock
+	@rm regional.lock
+	@gsutil rm "gs://$$(cat multiregional.lock)/*" || true
 	@gsutil rb "gs://$$(cat multiregional.lock)"
-	rm multiregional.lock
+	@rm multiregional.lock
+	@gsutil rm "gs://$$(cat public.lock)/*" || true
 	@gsutil rb "gs://$$(cat public.lock)"
-	rm public.lock
+	@rm public.lock
 
 # Perform only unit tests
 test-unit: get-deps clean fmt lint vet build

README.md

@@ -1,87 +1,85 @@
-## GCS CLI
+## bosh-gcscli
 
-A CLI for uploading, fetching and deleting content to/from
-the [GCS blobstore](https://cloud.google.com/storage/). This is **not**
-an official Google Product.
+[![GoDoc](https://godoc.org/github.com/cloudfoundry/bosh-gcscli?status.svg)](https://godoc.org/github.com/cloudfoundry/bosh-gcscli)
 
-## Installation
 
-```
-go get github.com/cloudfoundry/bosh-gcscli
-```
+A Golang CLI for uploading, fetching and deleting content to/from [Google Cloud Storage](https://cloud.google.com/storage/). 
+This tool exists to work with the [bosh-cli](https://github.com/cloudfoundry/bosh-cli) and [director](https://github.com/cloudfoundry/bosh).
 
-## Usage
-
-Given a JSON config file (`config.json`)...
-
-``` json
-{
-  "bucket_name":         "name of GCS bucket (required)",
-
-  "credentials_source":  "flag for credentials
-                          (optional, defaults to Application Default Credentials)
-                          (can be "static" for json_key),
-                          (can be "none" for explicitly no credentials)"
-  "storage_class":       "storage class for objects
-                          (optional, defaults to bucket settings)",
-  "json_key":            "JSON Service Account File
-                          (optional, required for static credentials)",
-  "encryption_key":      "Base64 encoded 32 byte Customer-Supplied
-                          encryption key used to encrypt objects 
-                          (optional)"
-}
-```
+This is **not** an official Google Product.
 
+## Installation
 
-Empty `credentials_source` implies attempting to use Application Default
-Credentials. `none` as `credentials_source` specifies no read-only scope
-with explicitly no credentials. `static` as `credentials_source` specifies to
-use the [Service Account File](https://developers.google.com/identity/protocols/OAuth2ServiceAccount) included
-in `json_key`.
+```bash
+go get github.com/cloudfoundry/bosh-gcscli
+```
 
-Empty `storage_class` implies using the default for the bucket.
+## Commands
 
-``` bash
-# Usage
+### Usage
+```bash
 bosh-gcscli --help
-
-# Command: "put"
-# Upload a blob to the GCS blobstore.
+```
+### Upload an object
+```bash
 bosh-gcscli -c config.json put <path/to/file> <remote-blob>
-
-# Command: "get"
-# Fetch a blob from the GCS blobstore.
-# Destination file will be overwritten if exists.
+```
+### Fetch an object
+```bash
 bosh-gcscli -c config.json get <remote-blob> <path/to/file>
-
-# Command: "delete"
-# Remove a blob from the GCS blobstore.
+```
+### Delete an object
+```bash
 bosh-gcscli -c config.json delete <remote-blob>
-
-# Command: "exists"
-# Checks if blob exists in the GCS blobstore.
-bosh-gcscli -c config.json exists <remote-blob>
 ```
+### Check if an object exists
+```bash
+bosh-gcscli -c config.json exists <remote-blob>```
+```
+
+## Configuration
+The command line tool expects a JSON configuration file. Run `bosh-gcscli --help` for details.
+
+### Authentication Methods (`credentials_source`)
+* `static`: A [service account](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) key will be provided via the `json_key` field.
+* `none`: No credentials are provided. The client is reading from a public bucket.
+* &lt;empty&gt;: [Application Default Credentials](https://developers.google.com/identity/protocols/application-default-credentials)
+  will be used if they exist (either through `gcloud auth application-default login` or a [service account](https://cloud.google.com/iam/docs/understanding-service-accounts)).
+  If they don't exist the client will fall back to `none` behavior.
+
+## Running Integration Tests
+
+1. Ensure [gcloud](https://cloud.google.com/sdk/downloads) is installed and you have authenticated (`gcloud auth login`).
+   These credentials will be used by the Makefile to create/destroy Google Cloud Storage buckets for testing.
+1. Set the Google Cloud project: `gcloud config set project <your project>`
+1. Generate a service account with the `Storage Admin` role for your project and set the contents as 
+    the environment variable `GOOGLE_APPLICATION_CREDENTIALS`, for example:
+   ```bash
+   export project_id=$(gcloud config get-value project)
+
+   export service_account_name=bosh-gcscli-integration-tests
+   export service_account_email=${service_account_name}@${project_id}.iam.gserviceaccount.com
+   credentials_file=$(mktemp)
 
-Alternatively, this package's underlying client can be used to access GCS,
-see the [godoc](https://godoc.org/github.com/cloudfoundry/bosh-gcscli)
-for more information.
+   gcloud config set project ${project_id}
+   gcloud iam service-accounts create ${service_account_name} --display-name "Integration Test Access for bosh-gcscli"
+   gcloud iam service-accounts keys create ${credentials_file} --iam-account ${service_account_email}
+   gcloud project add-iam-policy-binding ${project_id} --member serviceAccount:${service_account_email} --role roles/storage.admin
+  
+   export GOOGLE_APPLICATION_CREDENTIALS="$(cat ${credentials_file})"
+   ```
+1. Run the unit and fast integration tests: `make test-fast-int`
+1. Clean up buckets: `make clean-gcs`
 
-## Tooling
+## Development
 
-A Makefile is provided for ease of development. Targets are annotated
-with descriptions.
+* A Makefile is provided that automates integration testing. Try `make help` to get started.
+* [gvt](https://godoc.org/github.com/FiloSottile/gvt) is used for vendoring.
 
-gvt is used for vendoring. For full usage, see the [manual at godoc](https://godoc.org/github.com/FiloSottile/gvt).
+## Contributing
 
-Integration tests expect to be run from a host with [Application Default
-Credentials](https://developers.google.com/identity/protocols/application-default-credentials)
-available which has permissions to create and delete buckets.
-Application Default Credentials are present on any GCE instance and inherit
-the permisions of the [service account](https://cloud.google.com/iam/docs/service-accounts)
-assigned to the instance.
+For details on how to contribute to this project - including filing bug reports and contributing code changes - please see [CONTRIBUTING.md](./CONTRIBUTING.md).
 
 ## License
 
-This library is licensed under Apache 2.0. Full license text is
-available in [LICENSE](LICENSE).
+This tool is licensed under Apache 2.0. Full license text is available in [LICENSE](LICENSE).

ci/.gitignore

@@ -0,0 +1 @@
+credentials.yml

ci/pipeline.yml

@@ -65,10 +65,6 @@ jobs:
           params: {file: out-linux/bosh-gcscli-*-linux-amd64}
         - put: release-bucket-windows
           params: {file: out-windows/bosh-gcscli-*-windows-amd64.exe}
-      - put: gcscli-src
-        resource: bosh-gcscli-out
-        params: {repository: gcscli-src, tag: version-semver/number, tag_prefix: v}
-
 
 resources:
   - name: bosh-gcscli-src-in
@@ -82,6 +78,7 @@ resources:
     source:
       initial_version: 0.0.1
       key: current-version
+      endpoint: storage.googleapis.com
       bucket: {{gcscli_release_bucket}}
       access_key_id: {{gcscli_release_bucket_access_key}}
       secret_access_key: {{gcscli_release_bucket_secret_key}}
@@ -100,11 +97,4 @@ resources:
       regexp: bosh-gcscli-(.*)-windows-amd64\.exe
       bucket: {{gcscli_release_bucket}}
       access_key_id: {{gcscli_release_bucket_access_key}}
-      secret_access_key: {{gcscli_release_bucket_secret_key}}
-
-  - name: bosh-gcscli-out
-    type: git
-    source:
-      uri: [email protected]:cloudfoundry/bosh-gcscli.git
-      branch: master
-      private_key: {{github_deployment_key_gcscli}}
+      secret_access_key: {{gcscli_release_bucket_secret_key}}

ci/tasks/build.sh

@@ -1,14 +1,12 @@
 #!/usr/bin/env bash
 
-set -e
+set -ue
 
 my_dir="$( cd $(dirname $0) && pwd )"
-release_dir="$( cd ${my_dir} && cd ../.. && pwd )"
-workspace_dir="$( cd ${release_dir} && cd ../../../.. && pwd )"
-
-source ${release_dir}/ci/tasks/utils.sh
-export GOPATH=${workspace_dir}
-export PATH=${GOPATH}/bin:${PATH}
+pushd ${my_dir} > /dev/null
+    source utils.sh
+    set_env
+popd > /dev/null
 
 # inputs
 semver_dir="${workspace_dir}/version-semver"

ci/tasks/run-fast-int.sh

@@ -1,32 +1,15 @@
 #!/usr/bin/env bash
 
-set -e
+set -ue
 
 my_dir="$( cd $(dirname $0) && pwd )"
-release_dir="$( cd ${my_dir} && cd ../.. && pwd )"
-workspace_dir="$( cd ${release_dir} && cd ../../../.. && pwd )"
-
-pushd ${release_dir} > /dev/null
-
-source ci/tasks/utils.sh
-
+pushd ${my_dir} > /dev/null
+    source utils.sh
+    set_env
+    gcloud_login
 popd > /dev/null
 
-check_param google_project
-check_param google_json_key_data
-
-gcloud config set project $google_project
-
-echo $google_json_key_data > key.json
-gcloud auth activate-service-account --key-file=key.json
-
-export GOPATH=${workspace_dir}
-export PATH=${GOPATH}/bin:${PATH}
-
-pushd ${release_dir} > /dev/null
-
-GOOGLE_SERVICE_ACCOUNT=$google_json_key_data make test-fast-int
-
-make clean-gcs
-
-popd > /dev/null
+pushd ${release_dir}
+    trap clean_gcs EXIT
+    GOOGLE_SERVICE_ACCOUNT=$google_json_key_data make test-fast-int
+popd

ci/tasks/run-int.sh

@@ -1,32 +1,15 @@
 #!/usr/bin/env bash
 
-set -e
+set -ue
 
 my_dir="$( cd $(dirname $0) && pwd )"
-release_dir="$( cd ${my_dir} && cd ../.. && pwd )"
-workspace_dir="$( cd ${release_dir} && cd ../../../.. && pwd )"
-
-pushd ${release_dir} > /dev/null
-
-source ci/tasks/utils.sh
-
+pushd ${my_dir} > /dev/null
+    source utils.sh
+    set_env
+    gcloud_login
 popd > /dev/null
 
-check_param google_project
-check_param google_json_key_data
-
-gcloud config set project $google_project
-
-echo $google_json_key_data > key.json
-gcloud auth activate-service-account --key-file=key.json
-
-export GOPATH=${workspace_dir}
-export PATH=${GOPATH}/bin:${PATH}
-
-pushd ${release_dir} > /dev/null
-
-GOOGLE_SERVICE_ACCOUNT=$google_json_key_data make test-int
-
-make clean-gcs
-
-popd > /dev/null
+pushd ${release_dir}
+    trap clean_gcs EXIT
+    GOOGLE_SERVICE_ACCOUNT=$google_json_key_data make test-int
+popd

ci/tasks/run-unit.sh

@@ -1,17 +1,13 @@
 #!/usr/bin/env bash
 
-set -e
-
+set -ue
 
 my_dir="$( cd $(dirname $0) && pwd )"
-release_dir="$( cd ${my_dir} && cd ../.. && pwd )"
-workspace_dir="$( cd ${release_dir} && cd ../../../.. && pwd )"
-
-export GOPATH=${workspace_dir}
-export PATH=${GOPATH}/bin:${PATH}
-
-pushd ${release_dir} > /dev/null
-
-make test-unit
+pushd ${my_dir} > /dev/null
+    source utils.sh
+    set_env
+popd > /dev/null
 
+pushd ${release_dir}
+    make test-unit
 popd > /dev/null