Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Added permutation counting, variable definitions to parser test generation #59

Merged
merged 1 commit into from
Dec 1, 2023
Merged

Added permutation counting, variable definitions to parser test generation #59

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 42 additions & 21 deletions testing/generation/command_docs/create_tests.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ const TestFooter = ` }
`

// GenerateTestsFromSynopses generates a test file in the output directory for each file in the synopses directory.
func GenerateTestsFromSynopses() (err error) {
func GenerateTestsFromSynopses(repetitionDisabled ...string) (err error) {
parentFolder, err := GetCommandDocsFolder()
if err != nil {
return err
Expand Down Expand Up @@ -114,7 +114,14 @@ FileLoop:
err = errors.Join(err, errors.New(sb.String()))
continue FileLoop
}
stmtGen, nErr := ParseTokens(tokens)
includeRepetition := len(repetitionDisabled) == 0 || repetitionDisabled[0] != "*"
for _, bans := range repetitionDisabled {
if strings.ToLower(bans) == strings.ToLower(prefix) {
includeRepetition = false
break
}
}
stmtGen, nErr := ParseTokens(tokens, includeRepetition)
if nErr != nil {
err = errors.Join(err, nErr)
continue FileLoop
Expand Down Expand Up @@ -148,9 +155,11 @@ FileLoop:
}

// ParseTokens parses the given tokens into a StatementGenerator.
func ParseTokens(tokens []Token) (StatementGenerator, error) {
func ParseTokens(tokens []Token, includeRepetition bool) (StatementGenerator, error) {
stack := NewStatementGeneratorStack()
var statements []StatementGenerator
variables := make(map[string]StatementGenerator)
currentVariable := ""
tokenReader := NewTokenReader(tokens)
ForLoop:
for {
Expand All @@ -164,27 +173,25 @@ ForLoop:
case TokenType_Variable:
stack.AddVariable(token.Literal)
case TokenType_VariableDefinition:
//TODO: implement variable definitions
break ForLoop
currentVariable = token.Literal
if token, _ = tokenReader.Next(); token.Type != TokenType_LongSpace {
return nil, fmt.Errorf("expected a long space after a variable definition declaration")
}
case TokenType_Or:
if err := stack.Or(); err != nil {
return nil, err
}
case TokenType_Repeat:
if err := stack.Repeat(false); err != nil {
return nil, err
}
case TokenType_CommaRepeat:
if err := stack.Repeat(true); err != nil {
return nil, err
if includeRepetition {
if err := stack.Repeat(); err != nil {
return nil, err
}
}
case TokenType_OptionalRepeat:
if err := stack.OptionalRepeat(false); err != nil {
return nil, err
}
case TokenType_OptionalCommaRepeat:
if err := stack.OptionalRepeat(true); err != nil {
return nil, err
if includeRepetition {
if err := stack.OptionalRepeat(token.Literal); err != nil {
return nil, err
}
}
case TokenType_ShortSpace, TokenType_MediumSpace:
return nil, fmt.Errorf("token reader should have removed all short and medium spaces")
Expand All @@ -196,7 +203,15 @@ ForLoop:
if newStatement == nil {
return nil, fmt.Errorf("long space encountered before writing to the stack")
}
statements = append(statements, newStatement)
if len(currentVariable) > 0 {
if _, ok = variables[currentVariable]; ok {
return nil, fmt.Errorf("multiple definitions for the same variable: %s", currentVariable)
}
variables[currentVariable] = newStatement
currentVariable = ""
} else {
statements = append(statements, newStatement)
}
if token.Type == TokenType_EOF {
break ForLoop
} else {
Expand Down Expand Up @@ -233,11 +248,17 @@ ForLoop:
}
if len(statements) == 0 {
return nil, fmt.Errorf("no statements were generated from the token stream")
} else if len(statements) == 1 {
return statements[0], nil
}
var finalStatementGenerator StatementGenerator
if len(statements) == 1 {
finalStatementGenerator = statements[0]
} else {
return Or(statements...), nil
finalStatementGenerator = Or(statements...)
}
if err = ApplyVariableDefinition(finalStatementGenerator, variables); err != nil {
return nil, err
}
return finalStatementGenerator, nil
}

// GetQueryResult runs the query against a Postgres server to validate that the query is syntactically valid. It then
Expand Down
48 changes: 20 additions & 28 deletions testing/generation/command_docs/generator_stack.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ func (sgs *StatementGeneratorStack) AddText(text string) {

// AddVariable creates a new VariableGen at the current depth.
func (sgs *StatementGeneratorStack) AddVariable(name string) {
sgs.stack.Peek().Append(Variable(name))
sgs.stack.Peek().Append(Variable(name, nil))
}

// Or will take all items from the current depth and add them to a parent OrGen. Either the previous depth is an OrGen,
Expand Down Expand Up @@ -209,43 +209,29 @@ func (sgs *StatementGeneratorStack) ExitParenScope() error {
return nil
}

// Repeat will add the last StatementGenerator at the current depth to a RepeatGen. By default, the limit is 2, however
// an optional parameter may be passed to specify a custom limit (only the first limit given is used).
func (sgs *StatementGeneratorStack) Repeat(includesComma bool, limit ...int) error {
// Repeat will add the last StatementGenerator at the current depth to an OptionalGen.
func (sgs *StatementGeneratorStack) Repeat() error {
current := sgs.stack.Peek()
lastGen := current.LastGenerator()
if lastGen == nil {
return fmt.Errorf("unable to repeat as no generators exist at the current depth")
}
actualLimit := 2
if len(limit) >= 1 {
actualLimit = limit[0]
}
if includesComma {
current.Append(Repeat(0, actualLimit, Collection(Text(","), lastGen.Copy())))
} else {
current.Append(Repeat(0, actualLimit, lastGen.Copy()))
}
current.Append(Optional(lastGen.Copy()))
return nil
}

// OptionalRepeat will add the last StatementGenerator at the current depth to a RepeatGen inside an OptionalGen. By
// default, the limit is 2, however an optional parameter may be passed to specify a custom limit (only the first limit
// given is used).
func (sgs *StatementGeneratorStack) OptionalRepeat(includesComma bool, limit ...int) error {
// OptionalRepeat will add the last StatementGenerator at the current depth to an OptionalGen. If a prefix is given,
// then it will be added as a TextGen before the repeated generator.
func (sgs *StatementGeneratorStack) OptionalRepeat(prefix string) error {
current := sgs.stack.Peek()
lastGen := current.LastGenerator()
if lastGen == nil {
return fmt.Errorf("unable to optionally repeat as no generators exist at the current depth")
}
actualLimit := 2
if len(limit) >= 1 {
actualLimit = limit[0]
}
if includesComma {
current.Append(Optional(Repeat(1, actualLimit, Collection(Text(","), lastGen.Copy()))))
if len(prefix) > 0 {
current.Append(Optional(Collection(Text(prefix), lastGen.Copy())))
} else {
current.Append(Optional(Repeat(1, actualLimit, lastGen.Copy())))
current.Append(Optional(lastGen.Copy()))
}
return nil
}
Expand All @@ -271,10 +257,16 @@ func (sgs *StatementGeneratorStack) Finish() (StatementGenerator, error) {
if len(currentDepth.generators) == 0 {
return nil, fmt.Errorf("internal bookkeeping error, stack has a depth with no generators")
}
if len(lastDepth) == 1 {
currentDepth.generators = append(currentDepth.generators, lastDepth[0])
} else if len(lastDepth) > 1 {
currentDepth.generators = append(currentDepth.generators, Collection(lastDepth...))
if lastGen := currentDepth.LastGenerator(); lastGen != nil {
if orGen, ok := lastGen.(*OrGen); ok {
if err := orGen.AddChildren(sgs.aggregate(lastDepth)); err != nil {
return nil, err
}
} else {
currentDepth.Append(sgs.aggregate(lastDepth))
}
} else {
currentDepth.Append(sgs.aggregate(lastDepth))
}
lastDepth = currentDepth.generators
}
Expand Down
92 changes: 28 additions & 64 deletions testing/generation/command_docs/scanner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -191,73 +191,37 @@ ScannerLoop:
}
}
case ',':
// All comma-dot repetition blocks look like `, ...`
if peek, _ := scanner.Peek(); peek != ' ' {
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_Text,
Literal: ",",
})
case '[':
if scanner.PeekMatchOffset("[ ... ]", 0) {
scanner.AdvanceBy(6)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_OptionalRepeat,
Literal: "",
})
} else if scanner.PeekMatchOffset("[ , ... ]", 0) {
scanner.AdvanceBy(8)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_Text,
Type: TokenType_OptionalRepeat,
Literal: ",",
})
continue ScannerLoop
}
dotCount := 0
CommaDotRepetitionLoop:
for n := 2; true; n++ {
peek, _ := scanner.PeekBy(n)
switch peek {
case '.':
dotCount++
default:
// If the dot count is different from 3, then we treat the comma as text
if dotCount == 3 {
scanner.AdvanceBy(n - 1)
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_CommaRepeat})
} else {
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_Text,
Literal: ",",
})
}
break CommaDotRepetitionLoop
}
}

case '[':
commaCount := 0
dotCount := 0
// Optional repetition blocks generally look like either [...] or [ , ... ]
RepetitionLoop:
for n := 1; true; n++ {
peek, ok := scanner.PeekBy(n)
if !ok {
scanner.savedErr = fmt.Errorf("unexpected EOF when looking for potential repetition")
break ScannerLoop
}
switch peek {
case ' ':
// We ignore spaces here, as no optional repetition blocks have other forms of whitespace.
case '.':
dotCount++
case ',':
commaCount++
case ']':
// If the dot count is different from 3, then we'll ignore it
if dotCount == 3 {
// If the comma count is greater than 1, then we'll ignore it
if commaCount == 0 {
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalRepeat})
} else if commaCount == 1 {
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalCommaRepeat})
}
scanner.AdvanceBy(n)
break RepetitionLoop
}
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalOpen})
break RepetitionLoop
default:
// We've encountered something not present in normal repetition blocks
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalOpen})
break RepetitionLoop
}
} else if scanner.PeekMatchOffset("[ AND ... ]", 0) {
scanner.AdvanceBy(10)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_OptionalRepeat,
Literal: "AND",
})
} else if scanner.PeekMatchOffset("[ OR ... ]", 0) {
scanner.AdvanceBy(9)
scanner.tokens = append(scanner.tokens, Token{
Type: TokenType_OptionalRepeat,
Literal: "OR",
})
} else {
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalOpen})
}
case ']':
scanner.tokens = append(scanner.tokens, Token{Type: TokenType_OptionalClose})
Expand Down
Loading