Skip to content

Commit

Permalink
Merge pull request #1056 from mozilla/develop
Browse files Browse the repository at this point in the history
v3.7.3
  • Loading branch information
ajvb authored May 9, 2022
2 parents 86f500d + 1fb588e commit e1edc05
Show file tree
Hide file tree
Showing 20 changed files with 1,348 additions and 296 deletions.
11 changes: 9 additions & 2 deletions .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -18,15 +18,22 @@ jobs:
resource_class: large
steps:
- checkout
- setup_remote_docker
- setup_remote_docker:
version: 20.10.11
- run:
name: Build containers
command: |
docker build -t mozilla/sops .
docker tag mozilla/sops "mozilla/sops:$CIRCLE_SHA1"
- run:
name: Build containers (alpine)
command: |
# Just to ensure the container can be built.
docker build -f Dockerfile.alpine -t mozilla/sops:alpine .
push:
machine: true
machine:
image: ubuntu-2004:202111-02
resource_class: large
steps:
- checkout
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ jobs:
run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
- name: Set RELEASE_NUMBER
run: echo "RELEASE_NUMBER=$(echo $RELEASE_VERSION | cut -c2-)" >> $GITHUB_ENV
- name: Build linux amd64 binary
run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64 go.mozilla.org/sops/v3/cmd/sops && cp dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64 dist/sops-${{ env.RELEASE_VERSION }}.linux
- name: Build linux arm64 binary
run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.linux.arm64 go.mozilla.org/sops/v3/cmd/sops
- name: Build darwin amd64 binary
Expand All @@ -44,8 +46,6 @@ jobs:
run: GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin.arm64 go.mozilla.org/sops/v3/cmd/sops
- name: Build windows binary
run: GOOS=windows CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.exe go.mozilla.org/sops/v3/cmd/sops
- name: Copy already built linux binary
run: cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux && cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64
- name: Create release
uses: "mozilla/action-automatic-releases@latest"
with:
Expand Down
16 changes: 16 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
@@ -1,6 +1,22 @@
Changelog
=========

3.7.3
-----
Changes:

* Upgrade dependencies (#1024, #1045)
* Build alpine container in CI (#1018, #1032, #1025)
* keyservice: accept KeyServiceServer in LocalClient (#1035)
* Add support for GCP Service Account within `GOOGLE_CREDENTIALS` (#953)

Bug fixes:

* Upload the correct binary for the linux amd64 build (#1026)
* Fix bug when specifying multiple age recipients (#966)
* Allow for empty yaml maps (#908)
* Limit AWS role names to 64 characters (#1037)

3.7.2
-----
Changes:
Expand Down
1 change: 0 additions & 1 deletion Dockerfile.alpine
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
FROM golang:1.12-alpine3.10 AS builder
FROM golang:1.17-alpine3.15 AS builder

RUN apk --no-cache add make
Expand Down
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ download-index:
bash make_download_page.sh

mock:
go get github.com/vektra/mockery/.../
go install github.com/vektra/mockery/.../
mockery -dir vendor/github.com/aws/aws-sdk-go/service/kms/kmsiface/ -name KMSAPI -output kms/mocks

.PHONY: all test generate clean vendor functional-tests mock
3 changes: 3 additions & 0 deletions age/keys.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
# created: 2020-07-18T03:16:47-07:00
# public key: age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw
AGE-SECRET-KEY-1NJT5YCS2LWU4V4QAJQ6R4JNU7LXPDX602DZ9NUFANVU5GDTGUWCQ5T59M6
# created: 2021-12-12T01:39:30+01:00
# public key: age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep
AGE-SECRET-KEY-1T0Z66WSXS6RMNCPSL7P2E8N4Q7SUD8VMG9ND27S08JL7Y2XAU9EQECHDS7
6 changes: 3 additions & 3 deletions age/keysource.go
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,7 @@ func MasterKeysFromRecipients(commaSeparatedRecipients string) ([]*MasterKey, er
var keys []*MasterKey

for _, recipient := range recipients {
key, err := MasterKeyFromRecipient(recipient)
key, err := masterKeyFromRecipient(recipient)

if err != nil {
return nil, err
Expand All @@ -196,8 +196,8 @@ func MasterKeysFromRecipients(commaSeparatedRecipients string) ([]*MasterKey, er
return keys, nil
}

// MasterKeyFromRecipient takes a Bech32-encoded public key and returns a new MasterKey.
func MasterKeyFromRecipient(recipient string) (*MasterKey, error) {
// masterKeyFromRecipient takes a Bech32-encoded public key and returns a new MasterKey.
func masterKeyFromRecipient(recipient string) (*MasterKey, error) {
recipient = strings.TrimSpace(recipient)
parsedRecipient, err := parseRecipient(recipient)

Expand Down
95 changes: 75 additions & 20 deletions age/keysource_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,76 +21,131 @@ func TestMasterKeysFromRecipientsEmpty(t *testing.T) {
assert.Equal(recipients, make([]*MasterKey, 0))
}

func TestMasterKeyFromRecipientWithLeadingAndTrailingSpaces(t *testing.T) {
func TestMasterKeyFromRecipientWithLeadingAndTrailingSpacesSingle(t *testing.T) {
assert := assert.New(t)

key, err := MasterKeyFromRecipient(" age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw ")
commaSeparatedRecipients := " age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw "
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

assert.NoError(err)

assert.Equal(key.Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
assert.Equal(len(keys), 1)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
}

func TestAge(t *testing.T) {
func TestMasterKeyFromRecipientWithLeadingAndTrailingSpacesMultiple(t *testing.T) {
assert := assert.New(t)

key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
commaSeparatedRecipients := " age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw , age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep "
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

assert.NoError(err)
assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString())

dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456")
assert.Equal(len(keys), 2)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
}

func TestMasterKeysFromRecipientsWithSingle(t *testing.T) {
assert := assert.New(t)

commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw"
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

err = key.Encrypt(dataKey)
assert.NoError(err)

_, filename, _, _ := runtime.Caller(0)
err = os.Setenv(SopsAgeKeyFileEnv, path.Join(path.Dir(filename), "keys.txt"))
assert.Equal(len(keys), 1)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
}

func TestMasterKeysFromRecipientsWithMultiple(t *testing.T) {
assert := assert.New(t)

commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

assert.NoError(err)

decryptedKey, err := key.Decrypt()
assert.Equal(len(keys), 2)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
}

func TestAge(t *testing.T) {
assert := assert.New(t)

commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

assert.NoError(err)
assert.Equal(dataKey, decryptedKey)

assert.Equal(len(keys), 2)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")

dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456")

for _, key := range keys {
err = key.Encrypt(dataKey)
assert.NoError(err)

_, filename, _, _ := runtime.Caller(0)
err = os.Setenv("SOPS_AGE_KEY_FILE", path.Join(path.Dir(filename), "keys.txt"))
assert.NoError(err)

decryptedKey, err := key.Decrypt()
assert.NoError(err)
assert.Equal(dataKey, decryptedKey)
}

}

func TestAgeDotEnv(t *testing.T) {
assert := assert.New(t)

key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

assert.NoError(err)
assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString())

assert.Equal(len(keys), 2)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")

dotenv := `IMAGE_PREFIX=repo/service-
APPLICATION_KEY=K6pfAWuUVND9Fz5SC7jmA6pfAWuUVND9Fz5SC7jmA
KEY_ID=003683d721f2ae683d721f2a1
DOMAIN=files.127.0.0.1.nip.io`
dataKey := []byte(dotenv)

err = key.Encrypt(dataKey)
err = keys[0].Encrypt(dataKey)
assert.NoError(err)

_, filename, _, _ := runtime.Caller(0)
err = os.Setenv(SopsAgeKeyFileEnv, path.Join(path.Dir(filename), "keys.txt"))
defer os.Unsetenv(SopsAgeKeyFileEnv)
assert.NoError(err)

decryptedKey, err := key.Decrypt()
decryptedKey, err := keys[0].Decrypt()
assert.NoError(err)
assert.Equal(dataKey, decryptedKey)
}

func TestAgeEnv(t *testing.T) {
assert := assert.New(t)

key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)

assert.NoError(err)
assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString())

assert.Equal(len(keys), 2)
assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")

dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456")

err = key.Encrypt(dataKey)
err = keys[0].Encrypt(dataKey)
assert.NoError(err)

_, filename, _, _ := runtime.Caller(0)
Expand All @@ -100,7 +155,7 @@ func TestAgeEnv(t *testing.T) {
defer os.Unsetenv(SopsAgeKeyEnv)
assert.NoError(err)

decryptedKey, err := key.Decrypt()
decryptedKey, err := keys[0].Decrypt()
assert.NoError(err)
assert.Equal(dataKey, decryptedKey)
}
6 changes: 4 additions & 2 deletions cmd/sops/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -442,12 +442,14 @@ func main() {
group = append(group, k)
}
for _, recipient := range ageRecipients {
k, err := age.MasterKeyFromRecipient(recipient)
keys, err := age.MasterKeysFromRecipients(recipient)
if err != nil {
log.WithError(err).Error("Failed to add key")
continue
}
group = append(group, k)
for _, key := range keys {
group = append(group, key)
}
}
return groups.Add(groups.AddOpts{
InputPath: c.String("file"),
Expand Down
6 changes: 4 additions & 2 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -153,11 +153,13 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[
for _, group := range cRule.KeyGroups {
var keyGroup sops.KeyGroup
for _, k := range group.Age {
key, err := age.MasterKeyFromRecipient(k)
keys, err := age.MasterKeysFromRecipients(k)
if err != nil {
return nil, err
}
keyGroup = append(keyGroup, key)
for _, key := range keys {
keyGroup = append(keyGroup, key)
}
}
for _, k := range group.PGP {
keyGroup = append(keyGroup, pgp.NewMasterKeyFromFingerprint(k))
Expand Down
28 changes: 22 additions & 6 deletions gcpkms/keysource.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,16 +3,16 @@ package gcpkms //import "go.mozilla.org/sops/v3/gcpkms"
import (
"encoding/base64"
"fmt"
"google.golang.org/api/option"
"os"
"regexp"
"strings"
"time"

"go.mozilla.org/sops/v3/logging"

"golang.org/x/net/context"
"golang.org/x/oauth2/google"

"github.com/sirupsen/logrus"
"golang.org/x/net/context"
cloudkms "google.golang.org/api/cloudkms/v1"
)

Expand Down Expand Up @@ -131,12 +131,15 @@ func (key MasterKey) createCloudKMSService() (*cloudkms.Service, error) {
}

ctx := context.Background()
client, err := google.DefaultClient(ctx, cloudkms.CloudPlatformScope)
if err != nil {
var options []option.ClientOption

if credentials, err := getGoogleCredentials(); err != nil {
return nil, err
} else if len(credentials) > 0 {
options = append(options, option.WithCredentialsJSON(credentials))
}

cloudkmsService, err := cloudkms.New(client)
cloudkmsService, err := cloudkms.NewService(ctx, options...)
if err != nil {
return nil, err
}
Expand All @@ -151,3 +154,16 @@ func (key MasterKey) ToMap() map[string]interface{} {
out["created_at"] = key.CreationDate.UTC().Format(time.RFC3339)
return out
}

// getGoogleCredentials looks for a GCP Service Account in the environment
// variable: GOOGLE_CREDENTIALS, set as either a path to a credentials file or directly as the
// variable's value in JSON format.
//
// If not set, will default to use GOOGLE_APPLICATION_CREDENTIALS
func getGoogleCredentials() ([]byte, error) {
defaultCredentials := os.Getenv("GOOGLE_CREDENTIALS")
if _, err := os.Stat(defaultCredentials); err == nil {
return os.ReadFile(defaultCredentials)
}
return []byte(defaultCredentials), nil
}
Loading

0 comments on commit e1edc05

Please sign in to comment.