Merge pull request #17 from prebid/tcf20-refactor

First stage of adding additional TCF2.0 support

.travis.yml

@@ -1,10 +1,22 @@
 language: go
 go:
- - "1.9"
- - "1.10"
+ - "1.14"
+ - "1.12"
+ - "1.13"
 
 script:
+ - go test -timeout 30s github.com/prebid/go-gdpr/bitutils
  - go test -timeout 30s github.com/prebid/go-gdpr/vendorconsent
+ - go test -timeout 30s github.com/prebid/go-gdpr/vendorconsent/tcf1
+ - go test -timeout 30s github.com/prebid/go-gdpr/vendorconsent/tcf2
  - go test -timeout 30s github.com/prebid/go-gdpr/vendorlist
- - go tool vet -source vendorconsent
- - go tool vet -source vendorlist
+ - go test -timeout 30s github.com/prebid/go-gdpr/vendorlist2
+ - go vet -source github.com/prebid/go-gdpr/api
+ - go vet -source github.com/prebid/go-gdpr/bitutils
+ - go vet -source github.com/prebid/go-gdpr/consentconstants
+ - go vet -source github.com/prebid/go-gdpr/consentconstants/tcf2
+ - go vet -source github.com/prebid/go-gdpr/vendorconsent
+ - go vet -source github.com/prebid/go-gdpr/vendorconsent/tcf1
+ - go vet -source github.com/prebid/go-gdpr/vendorconsent/tcf2
+ - go vet -source github.com/prebid/go-gdpr/vendorlist
+ - go vet -source github.com/prebid/go-gdpr/vendorlist2

api/vendorlist.go

@@ -22,10 +22,14 @@ type VendorList interface {
 type Vendor interface {
  // Purpose returns true if this vendor claims to use data for the given purpose, or false otherwise
  Purpose(purposeID consentconstants.Purpose) bool
+ // PurposeStrict checks only for the primary purpose, not considering flex purposes.
+ PurposeStrict(purposeID consentconstants.Purpose) bool
 
- // LegitimateInterest retursn true if this vendor claims a "Legitimate Interest" to
+ // LegitimateInterest returns true if this vendor claims a "Legitimate Interest" to
  // use data for the given purpose.
  //
  // For an explanation of legitimate interest, see https://www.gdpreu.org/the-regulation/key-concepts/legitimate-interest/
  LegitimateInterest(purposeID consentconstants.Purpose) bool
+ // LegitimateInterestStrict checks only for the primary legitimate, not considering flex purposes.
+ LegitimateInterestStrict(purposeID consentconstants.Purpose) (hasLegitimateInterest bool)
 }

bitutils/bitutils.go

@@ -0,0 +1,96 @@
+package bitutils
+
+import (
+ "encoding/binary"
+ "fmt"
+)
+
+// ParseByte4 parses 4 bits of data from the data array, starting at the given index
+func ParseByte4(data []byte, bitStartIndex uint) (byte, error) {
+ startByte := bitStartIndex / 8
+ bitStartOffset := bitStartIndex % 8
+ if bitStartOffset < 5 {
+ if uint(len(data)) < (startByte + 1) {
+ return 0, fmt.Errorf("ParseByte4 expected 4 bits to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+ return (data[startByte] & (0xf0 >> bitStartOffset)) >> (4 - bitStartOffset), nil
+ }
+ if uint(len(data)) < (startByte+2) && bitStartOffset > 4 {
+ return 0, fmt.Errorf("ParseByte4 expected 4 bits to start at bit %d, but the consent string was only %d bytes long (needs second byte)", bitStartIndex, len(data))
+ }
+
+ leftBits := (data[startByte] & (0xf0 >> bitStartOffset)) << (bitStartOffset - 4)
+ bitsConsumed := 8 - bitStartOffset
+ overflow := 4 - bitsConsumed
+ rightBits := (data[startByte+1] & (0xf0 << (4 - overflow))) >> (8 - overflow)
+ return leftBits | rightBits, nil
+}
+
+// ParseByte8 parses 8 bits of data from the data array, starting at the given index
+func ParseByte8(data []byte, bitStartIndex uint) (byte, error) {
+ startByte := bitStartIndex / 8
+ bitStartOffset := bitStartIndex % 8
+ if bitStartOffset == 0 {
+ if uint(len(data)) < (startByte + 1) {
+ return 0, fmt.Errorf("ParseByte8 expected 8 bits to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+ return data[startByte], nil
+ }
+ if uint(len(data)) < (startByte + 2) {
+ return 0, fmt.Errorf("ParseByte8 expected 8 bitst to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+
+ leftBits := (data[startByte] & (0xff >> bitStartOffset)) << bitStartOffset
+ shiftComplement := 8 - bitStartOffset
+ rightBits := (data[startByte+1] & (0xff << shiftComplement)) >> shiftComplement
+ return leftBits | rightBits, nil
+}
+
+// ParseUInt12 parses 12 bits of data fromt the data array, starting at the given index
+func ParseUInt12(data []byte, bitStartIndex uint) (uint16, error) {
+ startByte := bitStartIndex / 8
+ bitStartOffset := bitStartIndex % 8
+ if bitStartOffset < 4 {
+ if uint(len(data)) < (startByte + 2) {
+ return 0, fmt.Errorf("ParseUInt12 expected a 12-bit int to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+ }
+ if uint(len(data)) < (startByte+3) && bitStartOffset > 3 {
+ return 0, fmt.Errorf("ParseUInt12 expected a 12-bit int to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+
+ leftByte, err := ParseByte4(data, bitStartIndex)
+ if err != nil {
+ return 0, fmt.Errorf("ParseUInt12 error on left byte: %s", err)
+ }
+ rightByte, err := ParseByte8(data, bitStartIndex+4)
+ if err != nil {
+ return 0, fmt.Errorf("ParseUInt12 error on right byte: %s", err)
+ }
+ return binary.BigEndian.Uint16([]byte{leftByte, rightByte}), nil
+}
+
+// ParseUInt16 parses a 16-bit integer from the data array, starting at the given index
+func ParseUInt16(data []byte, bitStartIndex uint) (uint16, error) {
+ startByte := bitStartIndex / 8
+ bitStartOffset := bitStartIndex % 8
+ if bitStartOffset == 0 {
+ if uint(len(data)) < (startByte + 2) {
+ return 0, fmt.Errorf("ParseUInt16 expected a 16-bit int to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+ return binary.BigEndian.Uint16(data[startByte : startByte+2]), nil
+ }
+ if uint(len(data)) < (startByte + 3) {
+ return 0, fmt.Errorf("ParseUInt16 expected a 16-bit int to start at bit %d, but the consent string was only %d bytes long", bitStartIndex, len(data))
+ }
+
+ leftByte, err := ParseByte8(data, bitStartIndex)
+ if err != nil {
+ return 0, fmt.Errorf("ParseUInt16 error on left byte: %s", err)
+ }
+ rightByte, err := ParseByte8(data, bitStartIndex+8)
+ if err != nil {
+ return 0, fmt.Errorf("ParseUInt16 error on right byte: %s", err)
+ }
+ return binary.BigEndian.Uint16([]byte{leftByte, rightByte}), nil
+}

bitutils/bitutils_test.go

@@ -0,0 +1,159 @@
+package bitutils
+
+import (
+ "testing"
+)
+
+// Define some test data
+
+// 0000 0100 1010 0010 0000 0011 1011 0001 0000 0000 0010 1011
+
+var testdata = []byte{0x04, 0xa2, 0x03, 0xb1, 0x00, 0x2b}
+
+type testDefinition struct {
+ data []byte // The data to feed the function
+ offset uint // The bit offset in the byte slice to start
+ value uint64 // The value we expect the function to return (64 bit to allow for future functions that extract larger ints)
+}
+
+var test4Bits = []testDefinition{
+ {testdata, 21, 7}, // testdata duplicate of Offset which involves flowing over to a second byte
+ {testdata, 12, 2}, // testdata duplicate of Offset which aligns with a nibble and doesn't span over multiple bytes
+ {testdata, 44, 11}, // testdata duplicate of Offset which aligns with a nibble and doesn't span over multiple bytes
+ {testdata, 6, 2}, // testdata duplicate of Offset which involves flowing over to a second byte
+ {[]byte{0x10}, 0, 1}, // No offset
+ {[]byte{0x92}, 4, 2}, // Offset which aligns with a nibble and doesn't span over multiple bytes
+ {[]byte{0x99}, 1, 3}, // Offset which doesn't align with a nibble.
+ {[]byte{0x01, 0xe0}, 7, 15}, // Offset which involves flowing over to a second byte
+}
+
+func TestParseByte4(t *testing.T) {
+ b, err := ParseByte4(testdata, 46)
+ assertStringsEqual(t, "ParseByte4 expected 4 bits to start at bit 46, but the consent string was only 6 bytes long (needs second byte)", err.Error())
+
+ b, err = ParseByte4(testdata, 80)
+ assertStringsEqual(t, "ParseByte4 expected 4 bits to start at bit 80, but the consent string was only 6 bytes long", err.Error())
+
+ for _, test := range test4Bits {
+ b, err = ParseByte4(test.data, test.offset)
+ assertNilError(t, err)
+ assertBytesEqual(t, byte(test.value), b)
+ }
+}
+
+// Used https://cryptii.com/ to convert 8 bit sequeces to integers
+var test8Bits = []testDefinition{
+ {testdata, 4, 0x4a}, // Offset that alligns to a nibble
+ {testdata, 7, 81}, // Odd Offset
+ {testdata, 26, 196}, // Even offset that does not align to a nibble
+ {testdata, 6, 40}, // Second even offset that does not align to a nibble
+ {testdata, 8, 162}, // Zero offset
+}
+
+func TestParseByte8(t *testing.T) {
+ b, err := ParseByte8([]byte{0x44, 0x76}, 11)
+ assertStringsEqual(t, "ParseByte8 expected 8 bitst to start at bit 11, but the consent string was only 2 bytes long", err.Error())
+
+ b, err = ParseByte8([]byte{0x44, 0x76}, 18)
+ assertStringsEqual(t, "ParseByte8 expected 8 bitst to start at bit 18, but the consent string was only 2 bytes long", err.Error())
+
+ for _, test := range test8Bits {
+ b, err = ParseByte8(test.data, test.offset)
+ assertNilError(t, err)
+ assertBytesEqual(t, byte(test.value), b)
+ }
+}
+
+var test12Bits = []testDefinition{
+ {testdata, 10, 2176}, // Even Offset that does not align to a nibble, but fits 2 bytes
+ {testdata, 16, 59}, // Zero Offset
+ {testdata, 19, 472}, // Odd Offset that overflows to 3rd byte
+ {testdata, 1, 148}, // Odd offset that fits 2 bytes
+ {testdata, 22, 3780}, // Another even unaligned offset that overflows to 3rd byte
+ {testdata, 4, 1186}, // Offset that aligns to a nibble (these can never overflow)
+}
+
+func TestParseUInt12(t *testing.T) {
+ i, err := ParseUInt12(testdata, 44)
+ assertStringsEqual(t, "ParseUInt12 expected a 12-bit int to start at bit 44, but the consent string was only 6 bytes long", err.Error())
+
+ i, err = ParseUInt12(testdata, 40)
+ assertStringsEqual(t, "ParseUInt12 expected a 12-bit int to start at bit 40, but the consent string was only 6 bytes long", err.Error())
+
+ for _, test := range test12Bits {
+ i, err = ParseUInt12(test.data, test.offset)
+ assertNilError(t, err)
+ assertUInt16sEqual(t, uint16(test.value), i)
+ }
+}
+
+var test16Bits = []testDefinition{
+ {testdata, 10, 34830}, // Even offset that does not align to a nibble
+ {testdata, 16, 945}, // Zero offset
+ {testdata, 19, 7560}, // Odd offset
+ {testdata, 1, 2372}, // Odd offset
+ {testdata, 22, 60480}, // Second even offset that does not align to a nibble
+ {testdata, 4, 18976}, // Nibble aligned offset
+}
+
+func TestParseUInt16(t *testing.T) {
+ i, err := ParseUInt16(testdata, 44)
+ assertStringsEqual(t, "ParseUInt16 expected a 16-bit int to start at bit 44, but the consent string was only 6 bytes long", err.Error())
+
+ i, err = ParseUInt16(testdata, 40)
+ assertStringsEqual(t, "ParseUInt16 expected a 16-bit int to start at bit 40, but the consent string was only 6 bytes long", err.Error())
+
+ for _, test := range test16Bits {
+ i, err = ParseUInt16(test.data, test.offset)
+ assertNilError(t, err)
+ assertUInt16sEqual(t, uint16(test.value), i)
+ }
+}
+
+func assertNilError(t *testing.T, err error) {
+ if err != nil {
+ t.Fatalf("Unexpected error: %v", err)
+ }
+}
+
+func assertStringsEqual(t *testing.T, expected string, actual string) {
+ t.Helper()
+ if actual != expected {
+ t.Errorf("Strings were not equal. Expected %s, actual %s", expected, actual)
+ }
+}
+
+func assertBytesEqual(t *testing.T, expected byte, actual byte) {
+ t.Helper()
+ if actual != expected {
+ t.Errorf("bytes were not equal. Expected %d, actual %d", expected, actual)
+ }
+}
+
+func assertUInt8sEqual(t *testing.T, expected uint8, actual uint8) {
+ t.Helper()
+ if actual != expected {
+ t.Errorf("Ints were not equal. Expected %d, actual %d", expected, actual)
+ }
+}
+
+func assertUInt16sEqual(t *testing.T, expected uint16, actual uint16) {
+ t.Helper()
+ if actual != expected {
+ t.Errorf("Ints were not equal. Expected %d, actual %d", expected, actual)
+ }
+}
+
+func assertIntsEqual(t *testing.T, expected int, actual int) {
+ t.Helper()
+ if actual != expected {
+ t.Errorf("Ints were not equal. Expected %d, actual %d", expected, actual)
+ }
+}
+
+func assertBoolsEqual(t *testing.T, expected bool, actual bool) {
+ t.Helper()
+ if actual != expected {
+ t.Errorf("Bools were not equal. Expected %t, actual %t", expected, actual)
+ }
+}

vendorconsent/consent20_test.go

@@ -50,10 +50,10 @@ func TestInvalidConsentStrings20(t *testing.T) {
  assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQAeAA", "a BitField for 60 vendors requires a consent string of 36 bytes. This consent string had 30")
 
  // Bad RangeSections
- assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwA", "vendor consent strings using RangeSections require at least 31 bytes. Got 30")  // This encodes 184 bits
- assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAQQ", "rangeSection expected a 16-bit vendorID to start at bit 243, but the consent string was only 31 bytes long") // 1 single vendor, too few bits
- assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAYQAC", "rangeSection expected a 16-bit vendorID to start at bit 259, but the consent string was only 33 bytes long") // 1 vendor range, too few bits
- assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAgABA", "rangeSection expected a 16-bit vendorID to start at bit 260, but the consent string was only 33 bytes long") // 2 single vendors, too few bits
+ assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwA", "vendor consent strings using RangeSections require at least 31 bytes. Got 30") // This encodes 184 bits
+ assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAQQ", "ParseUInt16 expected a 16-bit int to start at bit 243, but the consent string was only 31 bytes long") // 1 single vendor, too few bits
+ assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAYQAC", "ParseUInt16 expected a 16-bit int to start at bit 259, but the consent string was only 33 bytes long") // 1 vendor range, too few bits
+ assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAgABA", "ParseUInt16 expected a 16-bit int to start at bit 260, but the consent string was only 33 bytes long") // 2 single vendors, too few bits
  assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAgAAAAA", "bit 242 range entry excludes vendor 0, but only vendors [1, 3] are valid")
  assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAgACAAA", "bit 242 range entry excludes vendor 4, but only vendors [1, 3] are valid")
  assertInvalid20(t, "CONciguONcjGKADACHENAOCIAC0ta__AACiQABwAgABAAAA", "bit 259 range entry excludes vendor 0, but only vendors [1, 3] are valid")
@@ -62,9 +62,9 @@ func TestInvalidConsentStrings20(t *testing.T) {
 }
 
 func TestParseValidString20(t *testing.T) {
- parsed, err := ParseString("CONciguONcjGKADACHENAOCIAC0ta__AACiQABgAAYA")
+ parsed, err := ParseString("COyiILmOyiILmADACHENAPCAAAAAAAAAAAAAE5QBgALgAqgD8AQACSwEygJyAAAAAA")
  assertNilError(t, err)
- assertUInt16sEqual(t, 14, parsed.VendorListVersion())
+ assertUInt16sEqual(t, 15, parsed.VendorListVersion())
 }
 
 func TestParseValidString20MaxVendorID0(t *testing.T) {

vendorconsent/tcf2/bitfield.go

@@ -4,51 +4,34 @@ import (
  "fmt"
 )
 
-func parseBitField(data consentMetadata) (*consentBitField, error) {
- vendorBitsRequired := data.MaxVendorID()
+func parseBitField(metadata ConsentMetadata, vendorBitsRequired uint16, startbit uint) (*consentBitField, uint, error) {
+ data := metadata.data
 
- // BitFields start at bit 230. This means the last three bits of byte 28 are part of the bitfield.
- // In this case "others" will never be used, and we don't risk an index-out-of-bounds by using it.
- if vendorBitsRequired <= 3 {
- return &consentBitField{
- consentMetadata: data,
- firstTwo: data[28],
- others: nil,
- }, nil
- }
-
- otherBytesRequired := (vendorBitsRequired - 3) / 8
- if (vendorBitsRequired-3)%8 > 0 {
- otherBytesRequired = otherBytesRequired + 1
- }
- dataLengthRequired := 28 + otherBytesRequired
- if uint(len(data)) < uint(dataLengthRequired) {
- return nil, fmt.Errorf("a BitField for %d vendors requires a consent string of %d bytes. This consent string had %d", vendorBitsRequired, dataLengthRequired, len(data))
+ bytesRequired := (uint(vendorBitsRequired) + startbit) / 8
+ if uint(len(data)) < bytesRequired {
+ return nil, 0, fmt.Errorf("a BitField for %d vendors requires a consent string of %d bytes. This consent string had %d", vendorBitsRequired, bytesRequired, len(data))
  }
 
  return &consentBitField{
- consentMetadata: data,
- firstTwo:  data[28],
- others: data[29:],
- }, nil
+ data:  data,
+ startbit: startbit,
+ maxVendorID: vendorBitsRequired,
+ }, startbit + uint(vendorBitsRequired), nil
 }
 
 // A BitField has len(MaxVendorID()) entries, with one bit for every vendor in the range.
 type consentBitField struct {
- consentMetadata
- firstTwo byte
- others []byte
+ data []byte
+ startbit uint
+ maxVendorID uint16
 }
 
 func (f *consentBitField) VendorConsent(id uint16) bool {
- if id < 1 || id > f.MaxVendorID() {
+ if id < 1 || id > f.maxVendorID {
  return false
  }
  // Careful here... vendor IDs start at index 1...
- if id <= 3 {
- return byteToBool(f.firstTwo & (0x04 >> id))
- }
- return isSet(f.others, uint(id-3))
+ return isSet(f.data, f.startbit+uint(id)-1)
 }
 
 // byteToBool returns false if val is 0, and true otherwise