Unique now works on maps and arrays #2068

mikefarah · Jun 15, 2024 · 42120e1 · 42120e1
1 parent 1147113
commit 42120e1
Show file tree

Hide file tree

Showing 3 changed files with 84 additions and 9 deletions.
diff --git a/pkg/yqlib/doc/operators/unique.md b/pkg/yqlib/doc/operators/unique.md
@@ -63,7 +63,29 @@ will output
 - ~
 ```
 
-## Unique array object fields
+## Unique array objects
+Given a sample.yml file of:
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+- name: harry
+  pet: cat
+```
+then
+```bash
+yq 'unique' sample.yml
+```
+will output
+```yaml
+- name: harry
+  pet: cat
+- name: billy
+  pet: dog
+```
+
+## Unique array of objects by a field
 Given a sample.yml file of:
 ```yaml
 - name: harry
@@ -85,3 +107,25 @@ will output
   pet: dog
 ```
 
+## Unique array of arrays
+Given a sample.yml file of:
+```yaml
+- - cat
+  - dog
+- - cat
+  - sheep
+- - cat
+  - dog
+```
+then
+```bash
+yq 'unique' sample.yml
+```
+will output
+```yaml
+- - cat
+  - dog
+- - cat
+  - sheep
+```
+
diff --git a/pkg/yqlib/operator_unique.go b/pkg/yqlib/operator_unique.go
@@ -23,7 +23,7 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 		candidate := el.Value.(*CandidateNode)
 
 		if candidate.Kind != SequenceNode {
-			return Context{}, fmt.Errorf("Only arrays are supported for unique")
+			return Context{}, fmt.Errorf("only arrays are supported for unique")
 		}
 
 		var newMatches = orderedmap.NewOrderedMap()
@@ -34,12 +34,9 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 				return Context{}, err
 			}
 
-			keyValue := "null"
-
-			if rhs.MatchingNodes.Len() > 0 {
-				first := rhs.MatchingNodes.Front()
-				keyCandidate := first.Value.(*CandidateNode)
-				keyValue = keyCandidate.Value
+			keyValue, err := getUniqueKeyValue(rhs)
+			if err != nil {
+				return Context{}, err
 			}
 
 			_, exists := newMatches.Get(keyValue)
@@ -59,3 +56,21 @@ func uniqueBy(d *dataTreeNavigator, context Context, expressionNode *ExpressionN
 	return context.ChildContext(results), nil
 
 }
+
+func getUniqueKeyValue(rhs Context) (string, error) {
+	keyValue := "null"
+
+	if rhs.MatchingNodes.Len() > 0 {
+		first := rhs.MatchingNodes.Front()
+		keyCandidate := first.Value.(*CandidateNode)
+		keyValue = keyCandidate.Value
+		if keyCandidate.Kind != ScalarNode {
+			var err error
+			keyValue, err = encodeToString(keyCandidate, encoderPreferences{YamlFormat, 0})
+			if err != nil {
+				return "", err
+			}
+		}
+	}
+	return keyValue, nil
+}
diff --git a/pkg/yqlib/operator_unique_test.go b/pkg/yqlib/operator_unique_test.go
@@ -33,13 +33,29 @@ var uniqueOperatorScenarios = []expressionScenario{
 		},
 	},
 	{
-		description: "Unique array object fields",
+		description: "Unique array objects",
+		document:    `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: cat}]`,
+		expression:  `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n",
+		},
+	},
+	{
+		description: "Unique array of objects by a field",
 		document:    `[{name: harry, pet: cat}, {name: billy, pet: dog}, {name: harry, pet: dog}]`,
 		expression:  `unique_by(.name)`,
 		expected: []string{
 			"D0, P[], (!!seq)::[{name: harry, pet: cat}, {name: billy, pet: dog}]\n",
 		},
 	},
+	{
+		description: "Unique array of arrays",
+		document:    `[[cat,dog], [cat, sheep], [cat,dog]]`,
+		expression:  `unique`,
+		expected: []string{
+			"D0, P[], (!!seq)::[[cat, dog], [cat, sheep]]\n",
+		},
+	},
 	{
 		skipDoc:    true,
 		document:   `[{name: harry, pet: cat}, {pet: fish}, {name: harry, pet: dog}]`,