From 6edb219fed7b7f14a174389f9fe63617d618c91f Mon Sep 17 00:00:00 2001 From: Rowan Seymour Date: Fri, 18 Aug 2023 13:18:46 -0500 Subject: [PATCH] Pass env into CollateEquals and CollateTransform --- assets/location.go | 2 +- envs/collate.go | 13 ++++++++++ envs/location_test.go | 28 ++++++++++---------- envs/locations.go | 39 ++++++++++++++++------------ flows/environment.go | 14 +++++----- flows/environment_test.go | 2 +- flows/field.go | 6 ++--- flows/routers/cases/tests.go | 50 +++++++++++++++++------------------- 8 files changed, 85 insertions(+), 69 deletions(-) create mode 100644 envs/collate.go diff --git a/assets/location.go b/assets/location.go index b7903c97f..0b16bbd93 100644 --- a/assets/location.go +++ b/assets/location.go @@ -40,5 +40,5 @@ import "github.com/nyaruka/goflow/envs" // @asset location type LocationHierarchy interface { FindByPath(path envs.LocationPath) *envs.Location - FindByName(name string, level envs.LocationLevel, parent *envs.Location) []*envs.Location + FindByName(env envs.Environment, name string, level envs.LocationLevel, parent *envs.Location) []*envs.Location } diff --git a/envs/collate.go b/envs/collate.go new file mode 100644 index 000000000..2eeb32dc4 --- /dev/null +++ b/envs/collate.go @@ -0,0 +1,13 @@ +package envs + +import "strings" + +// CollateEquals returns true if the given strings are equal in the given environment's collation +func CollateEquals(env Environment, s, t string) bool { + return CollateTransform(env, s) == CollateTransform(env, t) +} + +// CollateTransform transforms the given string into it's form to be used for collation. +func CollateTransform(env Environment, s string) string { + return strings.ToLower(s) +} diff --git a/envs/location_test.go b/envs/location_test.go index 49de58fd7..777dbff69 100644 --- a/envs/location_test.go +++ b/envs/location_test.go @@ -57,7 +57,9 @@ var locationHierarchyJSON = ` }` func TestLocationHierarchy(t *testing.T) { - hierarchy, err := envs.ReadLocationHierarchy(json.RawMessage(locationHierarchyJSON)) + env := envs.NewBuilder().Build() + + hierarchy, err := envs.ReadLocationHierarchy(env, json.RawMessage(locationHierarchyJSON)) assert.NoError(t, err) rwanda := hierarchy.Root() @@ -90,18 +92,18 @@ func TestLocationHierarchy(t *testing.T) { assert.Equal(t, gasabo, ndera.Parent()) assert.Equal(t, 0, len(ndera.Children())) - assert.Equal(t, []*envs.Location{rwanda}, hierarchy.FindByName("RWaNdA", envs.LocationLevel(0), nil)) - assert.Equal(t, []*envs.Location{kigali}, hierarchy.FindByName("kigari", envs.LocationLevel(1), nil)) - assert.Equal(t, []*envs.Location{kigali}, hierarchy.FindByName("rwanda > kigali city", envs.LocationLevel(1), nil)) - assert.Equal(t, []*envs.Location{kigali}, hierarchy.FindByName("kigari", envs.LocationLevel(1), rwanda)) - assert.Equal(t, []*envs.Location{gasabo}, hierarchy.FindByName("GASABO", envs.LocationLevel(2), nil)) - assert.Equal(t, []*envs.Location{gasabo}, hierarchy.FindByName("GASABO", envs.LocationLevel(2), kigali)) - assert.Equal(t, []*envs.Location{ndera}, hierarchy.FindByName("RWANDA > kigali city > gasabo > ndera", envs.LocationLevel(3), nil)) - - assert.Equal(t, []*envs.Location{}, hierarchy.FindByName("boston", envs.LocationLevel(1), nil)) // no such name - assert.Equal(t, []*envs.Location{}, hierarchy.FindByName("kigari", envs.LocationLevel(8), nil)) // no such level - assert.Equal(t, []*envs.Location{}, hierarchy.FindByName("kigari", envs.LocationLevel(2), nil)) // wrong level - assert.Equal(t, []*envs.Location{}, hierarchy.FindByName("kigari", envs.LocationLevel(2), gasabo)) // wrong parent + assert.Equal(t, []*envs.Location{rwanda}, hierarchy.FindByName(env, "RWaNdA", envs.LocationLevel(0), nil)) + assert.Equal(t, []*envs.Location{kigali}, hierarchy.FindByName(env, "kigari", envs.LocationLevel(1), nil)) + assert.Equal(t, []*envs.Location{kigali}, hierarchy.FindByName(env, "rwanda > kigali city", envs.LocationLevel(1), nil)) + assert.Equal(t, []*envs.Location{kigali}, hierarchy.FindByName(env, "kigari", envs.LocationLevel(1), rwanda)) + assert.Equal(t, []*envs.Location{gasabo}, hierarchy.FindByName(env, "GASABO", envs.LocationLevel(2), nil)) + assert.Equal(t, []*envs.Location{gasabo}, hierarchy.FindByName(env, "GASABO", envs.LocationLevel(2), kigali)) + assert.Equal(t, []*envs.Location{ndera}, hierarchy.FindByName(env, "RWANDA > kigali city > gasabo > ndera", envs.LocationLevel(3), nil)) + + assert.Equal(t, []*envs.Location{}, hierarchy.FindByName(env, "boston", envs.LocationLevel(1), nil)) // no such name + assert.Equal(t, []*envs.Location{}, hierarchy.FindByName(env, "kigari", envs.LocationLevel(8), nil)) // no such level + assert.Equal(t, []*envs.Location{}, hierarchy.FindByName(env, "kigari", envs.LocationLevel(2), nil)) // wrong level + assert.Equal(t, []*envs.Location{}, hierarchy.FindByName(env, "kigari", envs.LocationLevel(2), gasabo)) // wrong parent assert.Equal(t, rwanda, hierarchy.FindByPath(envs.LocationPath("RWANDA"))) assert.Equal(t, kigali, hierarchy.FindByPath("RWANDA > KIGALI CITY")) diff --git a/envs/locations.go b/envs/locations.go index ab7939c9e..61ba25afd 100644 --- a/envs/locations.go +++ b/envs/locations.go @@ -16,8 +16,8 @@ type LocationPath string // LocationResolver is used to resolve locations from names or hierarchical paths type LocationResolver interface { - FindLocations(string, LocationLevel, *Location) []*Location - FindLocationsFuzzy(string, LocationLevel, *Location) []*Location + FindLocations(Environment, string, LocationLevel, *Location) []*Location + FindLocationsFuzzy(Environment, string, LocationLevel, *Location) []*Location LookupLocation(LocationPath) *Location } @@ -118,12 +118,14 @@ func (p locationPathLookup) lookup(path LocationPath) *Location { return p[path. // location names aren't always unique in a given level - i.e. you can have two wards with the same name, but different parents type locationNameLookup map[string][]*Location -func (n locationNameLookup) addLookup(name string, location *Location) { - name = strings.ToLower(name) +func (n locationNameLookup) addLookup(env Environment, name string, location *Location) { + name = CollateTransform(env, name) n[name] = append(n[name], location) } -func (n locationNameLookup) lookup(name string) []*Location { return n[strings.ToLower(name)] } +func (n locationNameLookup) lookup(env Environment, name string) []*Location { + return n[CollateTransform(env, name)] +} // LocationHierarchy is a hierarical tree of locations type LocationHierarchy struct { @@ -135,14 +137,14 @@ type LocationHierarchy struct { } // NewLocationHierarchy cretes a new location hierarchy -func NewLocationHierarchy(root *Location, numLevels int) *LocationHierarchy { +func NewLocationHierarchy(env Environment, root *Location, numLevels int) *LocationHierarchy { h := &LocationHierarchy{} - h.initializeFromRoot(root, numLevels) + h.initializeFromRoot(env, root, numLevels) return h } // NewLocationHierarchy cretes a new location hierarchy -func (h *LocationHierarchy) initializeFromRoot(root *Location, numLevels int) { +func (h *LocationHierarchy) initializeFromRoot(env Environment, root *Location, numLevels int) { h.root = root h.levelLookups = make([]locationNameLookup, numLevels) h.pathLookup = make(locationPathLookup) @@ -160,17 +162,17 @@ func (h *LocationHierarchy) initializeFromRoot(root *Location, numLevels int) { } h.pathLookup.addLookup(location.path, location) - h.addNameLookups(location) + h.addNameLookups(env, location) }) } -func (h *LocationHierarchy) addNameLookups(location *Location) { +func (h *LocationHierarchy) addNameLookups(env Environment, location *Location) { lookups := h.levelLookups[int(location.level)] - lookups.addLookup(location.name, location) + lookups.addLookup(env, location.name, location) // include any aliases as names too for _, alias := range location.aliases { - lookups.addLookup(alias, location) + lookups.addLookup(env, alias, location) } } @@ -180,7 +182,7 @@ func (h *LocationHierarchy) Root() *Location { } // FindByName looks for all locations in the hierarchy with the given level and name or alias -func (h *LocationHierarchy) FindByName(name string, level LocationLevel, parent *Location) []*Location { +func (h *LocationHierarchy) FindByName(env Environment, name string, level LocationLevel, parent *Location) []*Location { // try it as a path first if it looks possible if level == 0 || IsPossibleLocationPath(name) { @@ -191,7 +193,7 @@ func (h *LocationHierarchy) FindByName(name string, level LocationLevel, parent } if int(level) < len(h.levelLookups) { - matches := h.levelLookups[int(level)].lookup(name) + matches := h.levelLookups[int(level)].lookup(env, name) if matches != nil { // if a parent is specified, filter the matches by it if parent != nil { @@ -221,8 +223,11 @@ func (h *LocationHierarchy) UnmarshalJSON(data []byte) error { return err } + // TODO this method is only used to load static assets and they're only used for testing.. but this isn't ideal + env := NewBuilder().Build() + root := locationFromEnvelope(&le, LocationLevel(0), nil) - h.initializeFromRoot(root, 4) + h.initializeFromRoot(env, root, 4) return nil } @@ -253,7 +258,7 @@ func locationFromEnvelope(envelope *locationEnvelope, currentLevel LocationLevel } // ReadLocationHierarchy reads a location hierarchy from the given JSON -func ReadLocationHierarchy(data json.RawMessage) (*LocationHierarchy, error) { +func ReadLocationHierarchy(env Environment, data json.RawMessage) (*LocationHierarchy, error) { var le locationEnvelope if err := utils.UnmarshalAndValidate(data, &le); err != nil { return nil, err @@ -261,5 +266,5 @@ func ReadLocationHierarchy(data json.RawMessage) (*LocationHierarchy, error) { root := locationFromEnvelope(&le, LocationLevel(0), nil) - return NewLocationHierarchy(root, 4), nil + return NewLocationHierarchy(env, root, 4), nil } diff --git a/flows/environment.go b/flows/environment.go index da4be8f99..7356a7c42 100644 --- a/flows/environment.go +++ b/flows/environment.go @@ -38,8 +38,8 @@ type assetLocationResolver struct { } // FindLocations returns locations with the matching name (case-insensitive), level and parent (optional) -func (r *assetLocationResolver) FindLocations(name string, level envs.LocationLevel, parent *envs.Location) []*envs.Location { - return r.locations.FindByName(name, level, parent) +func (r *assetLocationResolver) FindLocations(env envs.Environment, name string, level envs.LocationLevel, parent *envs.Location) []*envs.Location { + return r.locations.FindByName(env, name, level, parent) } // FindLocationsFuzzy returns matching locations like FindLocations but attempts the following strategies @@ -48,15 +48,15 @@ func (r *assetLocationResolver) FindLocations(name string, level envs.LocationLe // 2. Match with punctuation removed // 3. Split input into words and try to match each word // 4. Try to match pairs of words -func (r *assetLocationResolver) FindLocationsFuzzy(text string, level envs.LocationLevel, parent *envs.Location) []*envs.Location { +func (r *assetLocationResolver) FindLocationsFuzzy(env envs.Environment, text string, level envs.LocationLevel, parent *envs.Location) []*envs.Location { // try matching name exactly - if locations := r.FindLocations(text, level, parent); len(locations) > 0 { + if locations := r.FindLocations(env, text, level, parent); len(locations) > 0 { return locations } // try with punctuation removed stripped := strings.TrimSpace(regexp.MustCompile(`[\s\p{P}]+`).ReplaceAllString(text, "")) - if locations := r.FindLocations(stripped, level, parent); len(locations) > 0 { + if locations := r.FindLocations(env, stripped, level, parent); len(locations) > 0 { return locations } @@ -64,7 +64,7 @@ func (r *assetLocationResolver) FindLocationsFuzzy(text string, level envs.Locat re := regexp.MustCompile(`[\p{L}\d]+(-[\p{L}\d]+)*`) words := re.FindAllString(text, -1) for _, word := range words { - if locations := r.FindLocations(word, level, parent); len(locations) > 0 { + if locations := r.FindLocations(env, word, level, parent); len(locations) > 0 { return locations } } @@ -72,7 +72,7 @@ func (r *assetLocationResolver) FindLocationsFuzzy(text string, level envs.Locat // try with each pair of words for i := 0; i < len(words)-1; i++ { wordPair := strings.Join(words[i:i+2], " ") - if locations := r.FindLocations(wordPair, level, parent); len(locations) > 0 { + if locations := r.FindLocations(env, wordPair, level, parent); len(locations) > 0 { return locations } } diff --git a/flows/environment_test.go b/flows/environment_test.go index 3724ccd2b..1b81f17a4 100644 --- a/flows/environment_test.go +++ b/flows/environment_test.go @@ -89,7 +89,7 @@ func TestAssetsEnvironment(t *testing.T) { kigali := aenv.LocationResolver().LookupLocation("Rwanda > Kigali City") assert.Equal(t, "Kigali City", kigali.Name()) - matches := aenv.LocationResolver().FindLocationsFuzzy("gisozi town", flows.LocationLevelWard, nil) + matches := aenv.LocationResolver().FindLocationsFuzzy(env, "gisozi town", flows.LocationLevelWard, nil) assert.Equal(t, 1, len(matches)) assert.Equal(t, "Gisozi", matches[0].Name()) } diff --git a/flows/field.go b/flows/field.go index dac140449..cd477436a 100644 --- a/flows/field.go +++ b/flows/field.go @@ -233,15 +233,15 @@ func (f FieldValues) Parse(env envs.Environment, fields *FieldAssets, field *Fie if field.Type() == assets.FieldTypeWard { parent := f.getFirstLocationValue(env, fields, assets.FieldTypeDistrict) if parent != nil { - matchingLocations = locations.FindLocationsFuzzy(rawValue, LocationLevelWard, parent) + matchingLocations = locations.FindLocationsFuzzy(env, rawValue, LocationLevelWard, parent) } } else if field.Type() == assets.FieldTypeDistrict { parent := f.getFirstLocationValue(env, fields, assets.FieldTypeState) if parent != nil { - matchingLocations = locations.FindLocationsFuzzy(rawValue, LocationLevelDistrict, parent) + matchingLocations = locations.FindLocationsFuzzy(env, rawValue, LocationLevelDistrict, parent) } } else if field.Type() == assets.FieldTypeState { - matchingLocations = locations.FindLocationsFuzzy(rawValue, LocationLevelState, nil) + matchingLocations = locations.FindLocationsFuzzy(env, rawValue, LocationLevelState, nil) } if len(matchingLocations) > 0 { diff --git a/flows/routers/cases/tests.go b/flows/routers/cases/tests.go index ec57cd8b6..71ded8d87 100644 --- a/flows/routers/cases/tests.go +++ b/flows/routers/cases/tests.go @@ -199,7 +199,7 @@ func HasGroup(env envs.Environment, args ...types.XValue) types.XValue { // // @test has_phrase(text, phrase) func HasPhrase(env envs.Environment, text types.XText, test types.XText) types.XValue { - return testStringTokens(env, text, test, hasPhraseTest) + return testTextTokens(env, text, test, hasPhraseTest) } // HasAllWords tests whether all the `words` are contained in `text` @@ -212,7 +212,7 @@ func HasPhrase(env envs.Environment, text types.XText, test types.XText) types.X // // @test has_all_words(text, words) func HasAllWords(env envs.Environment, text types.XText, test types.XText) types.XValue { - return testStringTokens(env, text, test, hasAllWordsTest) + return testTextTokens(env, text, test, hasAllWordsTest) } // HasAnyWord tests whether any of the `words` are contained in the `text` @@ -225,7 +225,7 @@ func HasAllWords(env envs.Environment, text types.XText, test types.XText) types // // @test has_any_word(text, words) func HasAnyWord(env envs.Environment, text types.XText, test types.XText) types.XValue { - return testStringTokens(env, text, test, hasAnyWordTest) + return testTextTokens(env, text, test, hasAnyWordTest) } // HasOnlyPhrase tests whether the `text` contains only `phrase` @@ -241,7 +241,7 @@ func HasAnyWord(env envs.Environment, text types.XText, test types.XText) types. // // @test has_only_phrase(text, phrase) func HasOnlyPhrase(env envs.Environment, text types.XText, test types.XText) types.XValue { - return testStringTokens(env, text, test, hasOnlyPhraseTest) + return testTextTokens(env, text, test, hasOnlyPhraseTest) } // HasText tests whether there the text has any characters in it @@ -608,7 +608,7 @@ func HasState(env envs.Environment, text types.XText) types.XValue { return types.NewXErrorf("can't find locations in environment which is not location enabled") } - states := locations.FindLocationsFuzzy(text.Native(), flows.LocationLevelState, nil) + states := locations.FindLocationsFuzzy(env, text.Native(), flows.LocationLevelState, nil) if len(states) > 0 { return NewTrueResult(types.NewXText(string(states[0].Path()))) } @@ -643,9 +643,9 @@ func HasDistrict(env envs.Environment, args ...types.XValue) types.XValue { } } - states := locations.FindLocationsFuzzy(stateText.Native(), flows.LocationLevelState, nil) + states := locations.FindLocationsFuzzy(env, stateText.Native(), flows.LocationLevelState, nil) if len(states) > 0 { - districts := locations.FindLocationsFuzzy(text.Native(), flows.LocationLevelDistrict, states[0]) + districts := locations.FindLocationsFuzzy(env, text.Native(), flows.LocationLevelDistrict, states[0]) if len(districts) > 0 { return NewTrueResult(types.NewXText(string(districts[0].Path()))) } @@ -653,7 +653,7 @@ func HasDistrict(env envs.Environment, args ...types.XValue) types.XValue { // try without a parent state - it's ok as long as we get a single match if stateText.Empty() { - districts := locations.FindLocationsFuzzy(text.Native(), flows.LocationLevelDistrict, nil) + districts := locations.FindLocationsFuzzy(env, text.Native(), flows.LocationLevelDistrict, nil) if len(districts) == 1 { return NewTrueResult(types.NewXText(string(districts[0].Path()))) } @@ -700,11 +700,11 @@ func HasWard(env envs.Environment, args ...types.XValue) types.XValue { } } - states := locations.FindLocationsFuzzy(stateText.Native(), flows.LocationLevelState, nil) + states := locations.FindLocationsFuzzy(env, stateText.Native(), flows.LocationLevelState, nil) if len(states) > 0 { - districts := locations.FindLocationsFuzzy(districtText.Native(), flows.LocationLevelDistrict, states[0]) + districts := locations.FindLocationsFuzzy(env, districtText.Native(), flows.LocationLevelDistrict, states[0]) if len(districts) > 0 { - wards := locations.FindLocationsFuzzy(text.Native(), flows.LocationLevelWard, districts[0]) + wards := locations.FindLocationsFuzzy(env, text.Native(), flows.LocationLevelWard, districts[0]) if len(wards) > 0 { return NewTrueResult(types.NewXText(string(wards[0].Path()))) } @@ -713,7 +713,7 @@ func HasWard(env envs.Environment, args ...types.XValue) types.XValue { // try without a parent district - it's ok as long as we get a single match if districtText.Empty() { - wards := locations.FindLocationsFuzzy(text.Native(), flows.LocationLevelWard, nil) + wards := locations.FindLocationsFuzzy(env, text.Native(), flows.LocationLevelWard, nil) if len(wards) == 1 { return NewTrueResult(types.NewXText(string(wards[0].Path()))) } @@ -726,20 +726,16 @@ func HasWard(env envs.Environment, args ...types.XValue) types.XValue { // Text Test Functions //------------------------------------------------------------------------------------------ -func textEquals(s, t string) bool { - return strings.EqualFold(s, t) -} - -type stringTokenTest func(hayTokens []string, pinTokens []string) types.XValue +type stringTokenTest func(env envs.Environment, hayTokens []string, pinTokens []string) types.XValue -func testStringTokens(env envs.Environment, str types.XText, testStr types.XText, testFunc stringTokenTest) types.XValue { +func testTextTokens(env envs.Environment, str types.XText, testStr types.XText, testFunc stringTokenTest) types.XValue { hays := utils.TokenizeString(strings.TrimSpace(str.Native())) needles := utils.TokenizeString(strings.TrimSpace(testStr.Native())) - return testFunc(hays, needles) + return testFunc(env, hays, needles) } -func hasPhraseTest(hays []string, pins []string) types.XValue { +func hasPhraseTest(env envs.Environment, hays []string, pins []string) types.XValue { if len(pins) == 0 { return NewTrueResult(types.XTextEmpty) } @@ -747,7 +743,7 @@ func hasPhraseTest(hays []string, pins []string) types.XValue { pinIdx := 0 matches := make([]string, len(pins)) for i, hay := range hays { - if textEquals(hay, pins[pinIdx]) { + if envs.CollateEquals(env, hay, pins[pinIdx]) { matches[pinIdx] = hays[i] pinIdx++ if pinIdx == len(pins) { @@ -765,14 +761,14 @@ func hasPhraseTest(hays []string, pins []string) types.XValue { return FalseResult } -func hasAllWordsTest(hays []string, pins []string) types.XValue { +func hasAllWordsTest(env envs.Environment, hays []string, pins []string) types.XValue { matches := make([]string, 0, len(pins)) pinMatches := make([]int, len(pins)) for i, hay := range hays { matched := false for j, pin := range pins { - if textEquals(hay, pin) { + if envs.CollateEquals(env, hay, pin) { matched = true pinMatches[j]++ } @@ -799,12 +795,12 @@ func hasAllWordsTest(hays []string, pins []string) types.XValue { return FalseResult } -func hasAnyWordTest(hays []string, pins []string) types.XValue { +func hasAnyWordTest(env envs.Environment, hays []string, pins []string) types.XValue { matches := make([]string, 0, len(pins)) for i, hay := range hays { matched := false for _, pin := range pins { - if textEquals(hay, pin) { + if envs.CollateEquals(env, hay, pin) { matched = true break } @@ -822,7 +818,7 @@ func hasAnyWordTest(hays []string, pins []string) types.XValue { return FalseResult } -func hasOnlyPhraseTest(hays []string, pins []string) types.XValue { +func hasOnlyPhraseTest(env envs.Environment, hays []string, pins []string) types.XValue { // must be same length if len(hays) != len(pins) { return FalseResult @@ -831,7 +827,7 @@ func hasOnlyPhraseTest(hays []string, pins []string) types.XValue { // and every token must match matches := make([]string, 0, len(pins)) for i := range hays { - if !textEquals(hays[i], pins[i]) { + if !envs.CollateEquals(env, hays[i], pins[i]) { return FalseResult } matches = append(matches, hays[i])