resolves marianogappa#19 : include/exclude groups or databases to run

.settings.json.example

@@ -0,0 +1,34 @@
+{
+  "MaxAppServerConnections": 8,
+  "Databases": {
+    "local": {
+        "dbName": "test_database"
+    },
+    "staging": {
+        "appServer": "staging.server.com",
+        "dbName": "staging_database",
+        "dbServer": "staging.db.server.com",
+        "user": "root",
+        "pass": ""
+    },
+    "prod": {
+        "appServer": "prod.server.com",
+        "dbName": "prod_database",
+        "dbServer": "prod.db.server.com",
+        "user": "user",
+        "pass": "pass",
+        "sqlType": "mysql"
+    },
+    "postgresdb": {
+        "appServer": "pg.server.com",
+        "dbName": "pg_database",
+        "dbServer": "pg.db.server.com",
+        "user": "postgres",
+        "pass": "postgres",
+        "sqlType": "postgres"
+    }
+  },
+  "DatabaseGroups": {
+    "group1": [ "staging","local" ]
+  }
+}

README.md

@@ -35,10 +35,19 @@ $ compinit
 
 ## Configuration
 
-Create a `.databases.json` dotfile in your home folder or in any [XDG-compliant](https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html) directory. [This](.databases.json.example) is an example file.
+Create a `.settings.json` dotfile in your home folder or in any [XDG-compliant](https://standards.freedesktop.org/basedir-spec/basedir-spec-latest.html) directory. [This](.settings.json.example) is an example file.
 
 `sql` decides to execute with MySQL or PostgreSQL depending on the `sqlType` property set for a database, *defaulting to to MySQL if not set.*
 
+(The older `.databases.json` is used if `.settings.json` file is not found or contains no databases.)
+
+## Command line flags
+
+- gs (Group Select): A database group defined in `.settings.json`. If multiple `-gs {groupName}` arguments are specified, the union of the databases is queried.
+- gf (Group Filter): A database group defined in `.settings.json`. If multiple `-gf {groupName}` arguments are specified, the intersection of the databases is queried.
+- ge (Group Exclude): A database group defined in `.settings.json`. Databases in this group are not queried.
+- da (Database Exclude): This database is not queried.
+
 ## Example usages
 
 ```
@@ -47,6 +56,18 @@ cat query.sql | sql test_db
 sed 's/2015/2016/g' query_for_2015.sql | sql db1 db2 db3
 
 sql all "SELECT * FROM users WHERE name = 'John'"
+
+sql -gs group1 "SELECT * FROM users"
+  (All databases in group1)
+
+sql -gf group1 -gf group2 "SELECT * FROM users"
+  (All databases that are in group1 AND group2)
+
+sql -ge group2 all "SELECT * FROM users"
+  (All databases, except if they are in group2)
+
+sql -de db2 all "SELECT * FROM users"
+  (All databases, except db2)
 ```
 
 ## Notes
@@ -56,9 +77,13 @@ sql all "SELECT * FROM users WHERE name = 'John'"
 - `sql` assumes that you have correctly configured SSH keys on all servers you `ssh` to
 - `sql` will error if all targeted databases do not have the same sql type.
 
+## Settings
+
+- MaxAppServerConnections: Too many concurrent ssh connections to the same app server might be rejected by the host.
+
 ## Beware!
 
-- please note that `~/.databases.json` will contain your database credentials in plain text; if this is a problem for you, don't use `sql`!
+- please note that `~/.settings.json` will contain your database credentials in plain text; if this is a problem for you, don't use `sql`!
 - `sql` is meant for automation of one-time lightweight ad-hoc `SELECT`s on many databases at once; it's not recommended for mission critical bash scripts that do destructive operations on production servers!
 - If you close an ongoing `sql` operation, spawned `mysql` and `ssh`->`mysql` processes will soon follow to their deaths, but the underlying mysql server query thread will complete, as long as it takes! https://github.com/marianogappa/sql/issues/7
 

config.go

@@ -14,6 +14,7 @@ const DefaultMaxAppServerConnections = 5
 type settings struct {
 	MaxAppServerConnections int64
 	Databases               map[string]database
+	DatabaseGroups          map[string][]string
 }
 
 type database struct {

main.go

@@ -11,11 +11,36 @@ import (
 	"golang.org/x/sync/semaphore"
 )
 
+type arrayFlags []string
+
+func (i *arrayFlags) String() string {
+	result := ""
+	for _, v := range *i {
+		result = result + " " + v
+	}
+	return result
+}
+
+func (i *arrayFlags) Set(value string) error {
+	*i = append(*i, value)
+	return nil
+}
+
 func main() {
 	var (
-		flagHelp    = flag.Bool("help", false, "shows usage")
-		flagListDBs = flag.Bool("list-dbs", false, "List all available DBs (used for auto-completion)")
+		flagHelp       = flag.Bool("help", false, "shows usage")
+		flagListDBs    = flag.Bool("list-dbs", false, "List all available DBs (used for auto-completion)")
+		flagListGroups = flag.Bool("list-groups", false, "List all available Groups")
 	)
+	var flagGroupExclusion arrayFlags
+	var flagGroupFilter arrayFlags
+	var flagGroupSelector arrayFlags
+	var flagDbExclusion arrayFlags
+	flag.Var(&flagGroupExclusion, "ge", "DB group exclusion (can be repeated)")
+	flag.Var(&flagGroupFilter, "gf", "DB group filter (can be repeated, intersection)")
+	flag.Var(&flagGroupSelector, "gs", "DB group selector (can be repeated, union)")
+	flag.Var(&flagDbExclusion, "de", "DB exclusion (can be repeated)")
+
 	flag.BoolVar(flagHelp, "h", false, "shows usage")
 	flag.Parse()
 	if *flagHelp {
@@ -32,10 +57,16 @@ func main() {
 		os.Exit(0)
 	}
 
-	if len(os.Args[1:]) == 0 {
-		usage("Target database unspecified; where should I run the query?")
+	if *flagListGroups {
+		for groupName := range settings.DatabaseGroups {
+			fmt.Print(groupName, " ")
+		}
+		fmt.Println()
+		os.Exit(0)
 	}
 
+	nonFlagArgs := flag.Args()
+
 	var query string
 	var databasesArgs []string
 
@@ -45,39 +76,29 @@ func main() {
 	}
 	if (stat.Mode() & os.ModeCharDevice) != 0 {
 		// Stdin is a terminal. The last argument is the SQL.
-		if len(os.Args) < 3 {
+		if len(nonFlagArgs) == 0 {
 			usage("No SQL to run. Exiting.")
 		}
-		query = os.Args[len(os.Args)-1]
-		databasesArgs = os.Args[1 : len(os.Args)-1]
+		query = nonFlagArgs[len(nonFlagArgs)-1]
+		databasesArgs = nonFlagArgs[:len(nonFlagArgs)-1]
 	} else {
 		query = readQuery(os.Stdin)
-		databasesArgs = os.Args[1:]
+		databasesArgs = nonFlagArgs
 	}
 
 	if len(query) <= 3 {
 		usage("No SQL to run. Exiting.")
 	}
 
-	os.Exit(_main(settings, databasesArgs, query, newThreadSafePrintliner(os.Stdout).println))
-}
-
-func _main(settings *settings, databasesArgs []string, query string, println func(string)) int {
-	targetDatabases := []string{}
-	for _, k := range databasesArgs {
-		if _, ok := settings.Databases[k]; k != "all" && !ok {
-			usage("Target database unknown: [%v]", k)
-		}
-		if k == "all" {
-			targetDatabases = nil
-			for k := range settings.Databases {
-				targetDatabases = append(targetDatabases, k)
-			}
-			break
-		}
-		targetDatabases = append(targetDatabases, k)
+	targetDatabases := getTargetDatabases(settings, databasesArgs, flagGroupExclusion, flagGroupFilter, flagGroupSelector, flagDbExclusion)
+	if len(targetDatabases) == 0 {
+		usage("No database to run. Exiting.")
 	}
 
+	os.Exit(_main(settings, targetDatabases, query, newThreadSafePrintliner(os.Stdout).println))
+}
+
+func _main(settings *settings, targetDatabases []string, query string, println func(string)) int {
 	quitContext, cancel := context.WithCancel(context.Background())
 	go awaitSignal(cancel)
 

main_test.go

@@ -45,26 +45,9 @@ var baseTests = tests{
 			"db2\t3",
 		},
 	},
-	{
-		name:      "reads from all databases with the all keyword",
-		targetDBs: []string{"all"},
-		query:     "SELECT id FROM table1",
-		expected: []string{
-			"",
-			"db1\t1",
-			"db1\t2",
-			"db1\t3",
-			"db2\t1",
-			"db2\t2",
-			"db2\t3",
-			"db3\t1",
-			"db3\t2",
-			"db3\t3",
-		},
-	},
 	{
 		name:      "reads two fields from all databases",
-		targetDBs: []string{"all"},
+		targetDBs: []string{"db1", "db2", "db3"},
 		query:     "SELECT id, name FROM table1",
 		expected: []string{
 			"",
@@ -147,29 +130,9 @@ func Test_Mix_Mysql_PostgreSQL(t *testing.T) {
 					"db3\t3",
 				},
 			},
-			{
-				name:      "reads from all databases with the all keyword",
-				targetDBs: []string{"all"},
-				query:     "SELECT id FROM table1",
-				expected: []string{
-					"",
-					"db1\t1",
-					"db1\t2",
-					"db1\t3",
-					"db2\t1",
-					"db2\t2",
-					"db2\t3",
-					"db3\t1",
-					"db3\t2",
-					"db3\t3",
-					"db4\t1",
-					"db4\t2",
-					"db4\t3",
-				},
-			},
 			{
 				name:      "reads two fields from all databases",
-				targetDBs: []string{"all"},
+				targetDBs: []string{"db1", "db2", "db3", "db4"},
 				query:     "SELECT id, name FROM table1",
 				expected: []string{
 					"",

utils.go

@@ -7,6 +7,7 @@ import (
 	"io/ioutil"
 	"os"
 	"os/signal"
+	"sort"
 	"strings"
 	"sync"
 	"syscall"
@@ -48,3 +49,95 @@ func awaitSignal(cancel context.CancelFunc) {
 	<-signals
 	cancel()
 }
+
+func _mustGetGroupDatabases(settings *settings, groupName string) []string {
+	groupDatabases, ok := settings.DatabaseGroups[groupName]
+	if !ok {
+		usage("Selected group is unknown: [%v]", groupName)
+	}
+	if len(groupDatabases) == 0 {
+		usage("Selected group has no databases: [%v]", groupName)
+	}
+
+	for _, dbName := range groupDatabases {
+		if _, ok := settings.Databases[dbName]; !ok {
+			usage("Database [%v] in group [%v] unknown", dbName, groupName)
+		}
+	}
+	return groupDatabases
+}
+
+func _contains(s []string, str string) bool {
+	for _, v := range s {
+		if v == str {
+			return true
+		}
+	}
+
+	return false
+}
+
+func getTargetDatabases(settings *settings, databasesArgs []string, flagGroupExclusion []string, flagGroupFilter []string, flagGroupSelector []string, flagDbExclusion []string) []string {
+	targetDatabases := make(map[string]bool)
+
+	// add explicit databases from Args, or 'all' databases
+	for _, arg := range databasesArgs {
+		if _, ok := settings.Databases[arg]; arg != "all" && !ok {
+			usage("Target database unknown: [%v]", arg)
+		}
+		if arg == "all" {
+			for dbName := range settings.Databases {
+				targetDatabases[dbName] = true
+			}
+		} else {
+			targetDatabases[arg] = true
+		}
+	}
+
+	// add Selected Groups
+	for _, groupName := range flagGroupSelector {
+		for _, databaseName := range _mustGetGroupDatabases(settings, groupName) {
+			targetDatabases[databaseName] = true
+		}
+	}
+
+	// if no explicit databases, 'all' databases or Selected Groups has been set, use Filtered Group
+	if len(targetDatabases) == 0 {
+		if len(flagGroupFilter) == 0 {
+			usage("Must either specify 'all', group selectors, group filters or a list of databases.")
+		}
+		for _, databaseName := range _mustGetGroupDatabases(settings, flagGroupFilter[0]) {
+			targetDatabases[databaseName] = true
+		}
+	}
+
+	// remove databases that are not in filtered group
+	for _, groupName := range flagGroupFilter {
+		groupFilterDatabases := _mustGetGroupDatabases(settings, groupName)
+		for databaseName, _ := range targetDatabases {
+			if !_contains(groupFilterDatabases, databaseName) {
+				delete(targetDatabases, databaseName)
+			}
+		}
+	}
+
+	// remove databases of groups that are explicitly removed
+	for _, groupName := range flagGroupExclusion {
+		groupExclusionDatabases := _mustGetGroupDatabases(settings, groupName)
+		for _, databaseName := range groupExclusionDatabases {
+			delete(targetDatabases, databaseName)
+		}
+	}
+
+	// remove databases that are explicitly removed
+	for _, databaseName := range flagDbExclusion {
+		delete(targetDatabases, databaseName)
+	}
+
+	result := []string{}
+	for databaseName := range targetDatabases {
+		result = append(result, databaseName)
+	}
+	sort.Strings(result)
+	return result
+}