Skip to content

Commit

Permalink
Merge pull request #59 from dolthub/daylon/cd-permutations
Browse files Browse the repository at this point in the history 
Added permutation counting, variable definitions to parser test generation
  • Loading branch information
@Hydrocharged
Hydrocharged authored Dec 1, 2023
2 parents 85686ac + 4c15d74 commit 06567ef
Show file tree
Hide file tree
Showing 6 changed files with 196 additions and 229 deletions.
63 changes: 42 additions & 21 deletions testing/generation/command_docs/create_tests.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ const TestFooter = ` }
`

// GenerateTestsFromSynopses generates a test file in the output directory for each file in the synopses directory.
func GenerateTestsFromSynopses() (err error) {
func GenerateTestsFromSynopses(repetitionDisabled ...string) (err error) {
parentFolder, err := GetCommandDocsFolder()
if err != nil {
return err
Expand Down Expand Up @@ -114,7 +114,14 @@ FileLoop:
err = errors.Join(err, errors.New(sb.String()))
continue FileLoop
}
stmtGen, nErr := ParseTokens(tokens)
includeRepetition := len(repetitionDisabled) == 0 || repetitionDisabled[0] != "*"
for _, bans := range repetitionDisabled {
if strings.ToLower(bans) == strings.ToLower(prefix) {
includeRepetition = false
break
}
}
stmtGen, nErr := ParseTokens(tokens, includeRepetition)
if nErr != nil {
err = errors.Join(err, nErr)
continue FileLoop
Expand Down Expand Up @@ -148,9 +155,11 @@ FileLoop:
}

// ParseTokens parses the given tokens into a StatementGenerator.
func ParseTokens(tokens []Token) (StatementGenerator, error) {
func ParseTokens(tokens []Token, includeRepetition bool) (StatementGenerator, error) {
stack := NewStatementGeneratorStack()
var statements []StatementGenerator
variables := make(map[string]StatementGenerator)
currentVariable := ""
tokenReader := NewTokenReader(tokens)
ForLoop:
for {
Expand All @@ -164,27 +173,25 @@ ForLoop:
case TokenType_Variable:
stack.AddVariable(token.Literal)
case TokenType_VariableDefinition:
//TODO: implement variable definitions
break ForLoop
currentVariable = token.Literal
if token, _ = tokenReader.Next(); token.Type != TokenType_LongSpace {
return nil, fmt.Errorf("expected a long space after a variable definition declaration")
}
case TokenType_Or:
if err := stack.Or(); err != nil {
return nil, err
}
case TokenType_Repeat:
if err := stack.Repeat(false); err != nil {
return nil, err
}
case TokenType_CommaRepeat:
if err := stack.Repeat(true); err != nil {
return nil, err
if includeRepetition {
if err := stack.Repeat(); err != nil {
return nil, err
}
}
case TokenType_OptionalRepeat:
if err := stack.OptionalRepeat(false); err != nil {
return nil, err
}
case TokenType_OptionalCommaRepeat:
if err := stack.OptionalRepeat(true); err != nil {
return nil, err
if includeRepetition {
if err := stack.OptionalRepeat(token.Literal); err != nil {
return nil, err
}
}
case TokenType_ShortSpace, TokenType_MediumSpace:
return nil, fmt.Errorf("token reader should have removed all short and medium spaces")
Expand All @@ -196,7 +203,15 @@ ForLoop:
if newStatement == nil {
return nil, fmt.Errorf("long space encountered before writing to the stack")
}
statements = append(statements, newStatement)
if len(currentVariable) > 0 {
if _, ok = variables[currentVariable]; ok {
return nil, fmt.Errorf("multiple definitions for the same variable: %s", currentVariable)
}
variables[currentVariable] = newStatement
currentVariable = ""
} else {
statements = append(statements, newStatement)
}
if token.Type == TokenType_EOF {
break ForLoop
} else {
Expand Down Expand Up @@ -233,11 +248,17 @@ ForLoop:
}
if len(statements) == 0 {
return nil, fmt.Errorf("no statements were generated from the token stream")
} else if len(statements) == 1 {
return statements[0], nil
}
var finalStatementGenerator StatementGenerator
if len(statements) == 1 {
finalStatementGenerator = statements[0]
} else {
return Or(statements...), nil
finalStatementGenerator = Or(statements...)
}
if err = ApplyVariableDefinition(finalStatementGenerator, variables); err != nil {
return nil, err
}
return finalStatementGenerator, nil
}

// GetQueryResult runs the query against a Postgres server to validate that the query is syntactically valid. It then
Expand Down
48 changes: 20 additions & 28 deletions testing/generation/command_docs/generator_stack.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (sgs *StatementGeneratorStack) AddText(text string) {

// AddVariable creates a new VariableGen at the current depth.
func (sgs *StatementGeneratorStack) AddVariable(name string) {
sgs.stack.Peek().Append(Variable(name))
sgs.stack.Peek().Append(Variable(name, nil))
}

// Or will take all items from the current depth and add them to a parent OrGen. Either the previous depth is an OrGen,
Expand Down Expand Up @@ -209,43 +209,29 @@ func (sgs *StatementGeneratorStack) ExitParenScope() error {
return nil
}

// Repeat will add the last StatementGenerator at the current depth to a RepeatGen. By default, the limit is 2, however
// an optional parameter may be passed to specify a custom limit (only the first limit given is used).
func (sgs *StatementGeneratorStack) Repeat(includesComma bool, limit ...int) error {
// Repeat will add the last StatementGenerator at the current depth to an OptionalGen.
func (sgs *StatementGeneratorStack) Repeat() error {
current := sgs.stack.Peek()
lastGen := current.LastGenerator()
if lastGen == nil {
return fmt.Errorf("unable to repeat as no generators exist at the current depth")
}
actualLimit := 2
if len(limit) >= 1 {
actualLimit = limit[0]
}
if includesComma {
current.Append(Repeat(0, actualLimit, Collection(Text(","), lastGen.Copy())))
} else {
current.Append(Repeat(0, actualLimit, lastGen.Copy()))
}
current.Append(Optional(lastGen.Copy()))
return nil
}

// OptionalRepeat will add the last StatementGenerator at the current depth to a RepeatGen inside an OptionalGen. By
// default, the limit is 2, however an optional parameter may be passed to specify a custom limit (only the first limit
// given is used).
func (sgs *StatementGeneratorStack) OptionalRepeat(includesComma bool, limit ...int) error {
// OptionalRepeat will add the last StatementGenerator at the current depth to an OptionalGen. If a prefix is given,
// then it will be added as a TextGen before the repeated generator.
func (sgs *StatementGeneratorStack) OptionalRepeat(prefix string) error {
current := sgs.stack.Peek()
lastGen := current.LastGenerator()
if lastGen == nil {
return fmt.Errorf("unable to optionally repeat as no generators exist at the current depth")
}
actualLimit := 2
if len(limit) >= 1 {
actualLimit = limit[0]
}
if includesComma {
current.Append(Optional(Repeat(1, actualLimit, Collection(Text(","), lastGen.Copy()))))
if len(prefix) > 0 {
current.Append(Optional(Collection(Text(prefix), lastGen.Copy())))
} else {
current.Append(Optional(Repeat(1, actualLimit, lastGen.Copy())))
current.Append(Optional(lastGen.Copy()))
}
return nil
}
Expand All @@ -271,10 +257,16 @@ func (sgs *StatementGeneratorStack) Finish() (StatementGenerator, error) {
if len(currentDepth.generators) == 0 {
return nil, fmt.Errorf("internal bookkeeping error, stack has a depth with no generators")
}
if len(lastDepth) == 1 {
currentDepth.generators = append(currentDepth.generators, lastDepth[0])
} else if len(lastDepth) > 1 {
currentDepth.generators = append(currentDepth.generators, Collection(lastDepth...))
if lastGen := currentDepth.LastGenerator(); lastGen != nil {
if orGen, ok := lastGen.(*OrGen); ok {
if err := orGen.AddChildren(sgs.aggregate(lastDepth)); err != nil {
return nil, err
}
} else {
currentDepth.Append(sgs.aggregate(lastDepth))
}
} else {
currentDepth.Append(sgs.aggregate(lastDepth))
}
lastDepth = currentDepth.generators
}
Expand Down
92 changes: 28 additions & 64 deletions testing/generation/command_docs/scanner.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,73 +191,37 @@ ScannerLoop:
}
}
case ',':
// All comma-dot repetition blocks look like `, ...`
if peek, _ := scanner.Peek(); peek != ' ' {
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_Text,
Literal: ",",
})
case '[':
if scanner.PeekMatchOffset("[ ... ]", 0) {
scanner.AdvanceBy(6)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_OptionalRepeat,
Literal: "",
})
} else if scanner.PeekMatchOffset("[ , ... ]", 0) {
scanner.AdvanceBy(8)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_Text,
Type: TokenType_OptionalRepeat,
Literal: ",",
})
continue ScannerLoop
}
dotCount := 0
CommaDotRepetitionLoop:
for n := 2; true; n++ {
peek, _ := scanner.PeekBy(n)
switch peek {
case '.':
dotCount++
default:
// If the dot count is different from 3, then we treat the comma as text
if dotCount == 3 {
scanner.AdvanceBy(n - 1)
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_CommaRepeat})
} else {
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_Text,
Literal: ",",
})
}
break CommaDotRepetitionLoop
}
}

case '[':
commaCount := 0
dotCount := 0
// Optional repetition blocks generally look like either [...] or [ , ... ]
RepetitionLoop:
for n := 1; true; n++ {
peek, ok := scanner.PeekBy(n)
if !ok {
scanner.savedErr = fmt.Errorf("unexpected EOF when looking for potential repetition")
break ScannerLoop
}
switch peek {
case ' ':
// We ignore spaces here, as no optional repetition blocks have other forms of whitespace.
case '.':
dotCount++
case ',':
commaCount++
case ']':
// If the dot count is different from 3, then we'll ignore it
if dotCount == 3 {
// If the comma count is greater than 1, then we'll ignore it
if commaCount == 0 {
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalRepeat})
} else if commaCount == 1 {
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalCommaRepeat})
}
scanner.AdvanceBy(n)
break RepetitionLoop
}
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalOpen})
break RepetitionLoop
default:
// We've encountered something not present in normal repetition blocks
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalOpen})
break RepetitionLoop
}
} else if scanner.PeekMatchOffset("[ AND ... ]", 0) {
scanner.AdvanceBy(10)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_OptionalRepeat,
Literal: "AND",
})
} else if scanner.PeekMatchOffset("[ OR ... ]", 0) {
scanner.AdvanceBy(9)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_OptionalRepeat,
Literal: "OR",
})
} else {
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalOpen})
}
case ']':
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalClose})
Expand Down
Loading

0 comments on commit 06567ef

Please sign in to comment.