diff --git a/Changelog b/Changelog index 095c977f..2e849057 100644 --- a/Changelog +++ b/Changelog @@ -1,3 +1,18 @@ +1.4.0 28-Apr-2018 + NEW FEATURES: + - Added option --enable-mysqlx (MySQL 5.7.12+) + - Added options --enable-general-log and --init-general-log + - Added list of "reserverd-ports" to defaults + - Increased documentation inside command "usage" + - Added dbdeployer version and timestamp to sandbox descriprtion files. + - Added "mysqlsh" script to sandboxes 5.7.12+ with Xplugin enabled + - Added "show_log" script to all sandboxes + - Improved interface of show_binlog and show_relaylog + TESTING + - Added tests for reserved-ports + - Added test for mysqlsh and show_* creation + VARIOUS + - Updated documentation 1.3.0 21-Apr-2018 ADJUSTMENTS: - Added support for mysqlx plugin being enabled by default (MySQL 8.0.11+) diff --git a/README.md b/README.md index 56492929..95db8eb3 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [DBdeployer](https://github.com/datacharmer/dbdeployer) is a tool that deploys MySQL database servers easily. This is a port of [MySQL-Sandbox](https://github.com/datacharmer/mysql-sandbox), originally written in Perl, and re-designed from the ground up in [Go](https://golang.org). See the [features comparison](https://github.com/datacharmer/dbdeployer/blob/master/docs/features.md) for more detail. -Documentation updated for version 1.3.0 (20-Apr-2018 21:22 UTC) +Documentation updated for version 1.4.0 (28-Apr-2018 12:56 UTC) ## Installation @@ -13,7 +13,7 @@ Get the one for your O.S. from [dbdeployer releases](https://github.com/datachar For example: - $ VERSION=1.3.0 + $ VERSION=1.4.0 $ origin=https://github.com/datacharmer/dbdeployer/releases/download/$VERSION $ wget $origin/dbdeployer-$VERSION.linux.tar.gz $ tar -xzf dbdeployer-$VERSION.linux.tar.gz @@ -47,7 +47,7 @@ For example: The program doesn't have any dependencies. Everything is included in the binary. Calling *dbdeployer* without arguments or with ``--help`` will show the main help screen. $ dbdeployer --version - dbdeployer version 1.3.0 + dbdeployer version 1.4.0 $ dbdeployer -h @@ -140,10 +140,13 @@ The easiest command is ``deploy single``, which installs a single sandbox. -u, --db-user string database user (default "msandbox") --defaults strings Change defaults on-the-fly (--defaults=label:value) --disable-mysqlx Disable MySQLX plugin (8.0.11+) + --enable-general-log Enables general log for the sandbox (MySQL 5.1+) + --enable-mysqlx Enables MySQLX plugin (5.7.12+) --expose-dd-tables In MySQL 8.0+ shows data dictionary tables --force If a destination sandbox already exists, it will be overwritten --gtid enables GTID -h, --help help for deploy + --init-general-log uses general log during initialization (MySQL 5.1+) -i, --init-options strings mysqld options to run during initialization --keep-server-uuid Does not change the server UUID --my-cnf-file string Alternative source file for my.sandbox.cnf @@ -256,12 +259,49 @@ If you want to deploy several instances of the same version and the same type (f $ dbdeployer deploy single 8.0.4 # will deploy in msb_8_0_4 using port 8004 - $ dbdeployer deploy single 8.0.4 --sandbox-directory=msb2_8_0_4 --port=8005 - # will deploy in msb2_8_0_4 using port 8005 + $ dbdeployer deploy single 8.0.4 --sandbox-directory=msb2_8_0_4 + # will deploy in msb2_8_0_4 using port 8005 (which dbdeployer detects and uses) $ dbdeployer deploy replication 8.0.4 --sandbox-directory=rsandbox2_8_0_4 --base-port=18600 # will deploy replication in rsandbox2_8_0_4 using ports 18601, 18602, 18603 +## Ports management + +dbdeployer will try using the default port for each sandbox whenever possible. For single sandboxes, the port will be the version number without dots: 5.7.22 will deploy on port 5722. For multiple sandboxes, the port number is defined by using a prefix number (visible in the defaults: ``dbdeployer defaults list``) + the port number + the revision number (for some topologies multiplied by 100.) +For example, single-primary group replication with MySQL 8.0.11 will compute the ports like this: + + base port = 8011 (version number) + 13000 (prefix) + 11 (revision) * 100 = 22111 + node1 port = base port + 1 = 22112 + node2 port = base port + 2 = 22113 + node3 port = base port + 2 = 22114 + +For group replication we need to calculate the group port, and we use the ``group-port-delta`` (= 125) to obtain it from the regular port: + + node1 group port = 22112 + 125 = 22237 + node2 group port = 22113 + 125 = 22238 + node3 group port = 22114 + 125 = 22239 + +For MySQL 8.0.11+, we also need to assign a port for the XPlugin, and we compute that using the regular port + the ``mysqlx-port-delta`` (=10000). + +Thus, for MySQL 8.0.11 group replication deployments, you would see this listing: + + $ dbdeployer sandboxes --header + name type version ports + ---------------- ------- ------- ----- + group_msb_8_0_11 : group-multi-primary 8.0.11 [20023 20148 30023 20024 20149 30024 20025 20150 30025] + group_sp_msb_8_0_11 : group-single-primary 8.0.11 [22112 22237 32112 22113 22238 32113 22114 22239 32114] + +This method makes port clashes unlikely when using the same version in different deployments, but there is a risk of port clashes when deploying many multiple sandboxes of close-by versions. +However, dbdeployer doesn't let the clash happen. Thanks to its central catalog of sandboxes, it knows which ports were already used, and will search for free ones whenever a potential clash is detected. +Bear in mind that the concept of "used" is only related to sandboxes. dbdeployer does not know if ports may be used by other applications. +You can minimize risks, however, by telling dbdeployer which ports may be occupied. The defaults have a field ``reserved-ports``, containing the ports that should not be used. You can add to that list by modifying the defaults. For example, if you want to exclude port 7001, 10000, and 15000 from being used, you can run + + dbdeployer defaults update reserved-ports '7001,10000,15000' + +or, if you want to preserve the ones that are reserved by default: + + dbdeployer defaults update reserved-ports '1186,3306,33060,7001,10000,15000' + ## Concurrent deployment and deletion Starting with version 0.3.0, dbdeployer can deploy groups of sandboxes (``deploy replication``, ``deploy multiple``) with the flag ``--concurrent``. When this flag is used, dbdeployed will run operations concurrently. @@ -328,6 +368,25 @@ Similarly, for group replication WARNING: running sandboxes with ``--skip-start`` is provided for advanced users and is not recommended. If the purpose of skipping the start is to inspect the server before the sandbox granting operations, you may consider using ``--pre-grants-sql`` and ``--pre-grants-sql-file`` to run the necessary SQL commands (see _Sandbox customization_ below.) +## MySQL Document store, mysqlsh, and defaults. + +MySQL 5.7.12+ introduces the XPlugin (a.k.a. _mysqlx_) which enables operations using a separate port (33060 by default) on special tables that can be treated as NoSQL collections. +In MySQL 8.0.11+ the XPlugin is enabled by default, giving dbdeployer the task of defining an additional port and socket for this service. When you deploy MySQL 8.0.11 or later, dbdeployer sets the ``mysqlx-port`` to the value of the regular port + ``mysqlx-delta-port`` (= 10000). + +If you want to avoid having the XPlugin enabled, you can deploy the sandbox with the option ``--disable-mysqlx``. + +For MySQL between 5.7.12 and 8.0.4, the approach is the opposite. By default, the XPlugin is disabled, and if you want to use it you will run the deployment using ``--enable-mysqlx``. In both cases the port and socket will be computed by dbdeployer. + +When the XPlugin is enabled, it makes sense to use [the MySQL shell](https://dev.mysql.com/doc/refman/8.0/en/mysql-shell.html) and dbdeployer will create a ``mysqlsh`` script for the sandboxes that use the plugin. Unfortunately, as of today (late April 2018) the MySQL shell is not released with the server tarball, and therefore we have to fix things manually. dbdeployer will look for ``mysqlsh`` in the same directory where the other clients are, so if you manually merge the mysql shell and the server tarballs, you will get the appropriate version of MySQL shell. If not, you will use the version of the shell that is available in ``$PATH``. If there is no MySQL shell available, you will get an error. + +## Logs management. + +Sometimes, when using sandboxes for testing, it makes sense to enable the general log, either during initialization or for regular operation. While you can do that with ``--my-cnf-options=general-log=1`` or ``--my-init-options=--general-log=1``, as of version 1.4.0 you have two easy boolean shortcuts: ``--init-general-log`` and ``--enable-general-log`` that will start the general log when requested. + +Additionally, each sandbox has a convenience script named ``show_log`` that can easily display either the error log or the general log. Run `./show_log -h` for usage info. + +For replication, you also have ``show_binlog`` and ``show_relaylog`` in every sandbox as a shortcut to display replication logs easily. + ## Sandbox customization There are several ways of changing the default behavior of a sandbox. @@ -371,7 +430,7 @@ Here's how: $ dbdeployer defaults show # Internal values: { - "version": "1.3.0", + "version": "1.4.0", "sandbox-home": "$HOME/sandboxes", "sandbox-binary": "$HOME/opt/mysql", "use-sandbox-catalog": true, @@ -382,6 +441,7 @@ Here's how: "all-masters-replication-base-port": 15000, "multiple-base-port": 16000, "group-port-delta": 125, + "mysqlx-port-delta": 10000, "master-name": "master", "master-abbr": "m", "node-prefix": "node", @@ -393,7 +453,13 @@ Here's how: "group-sp-prefix": "group_sp_msb_", "multiple-prefix": "multi_msb_", "fan-in-prefix": "fan_in_msb_", - "all-masters-prefix": "all_masters_msb_" + "all-masters-prefix": "all_masters_msb_", + "reserved-ports": [ + 1186, + 3306, + 33060 + ], + "timestamp": "Sat Apr 28 14:56:12 CEST 2018" } @@ -401,7 +467,7 @@ Here's how: # Updated master-slave-base-port -> "15000" # Configuration file: $HOME/.dbdeployer/config.json { - "version": "1.3.0", + "version": "1.4.0", "sandbox-home": "$HOME/sandboxes", "sandbox-binary": "$HOME/opt/mysql", "use-sandbox-catalog": true, @@ -412,6 +478,7 @@ Here's how: "all-masters-replication-base-port": 15000, "multiple-base-port": 16000, "group-port-delta": 125, + "mysqlx-port-delta": 10000, "master-name": "master", "master-abbr": "m", "node-prefix": "node", @@ -423,7 +490,13 @@ Here's how: "group-sp-prefix": "group_sp_msb_", "multiple-prefix": "multi_msb_", "fan-in-prefix": "fan_in_msb_", - "all-masters-prefix": "all_masters_msb_" + "all-masters-prefix": "all_masters_msb_", + "reserved-ports": [ + 1186, + 3306, + 33060 + ], + "timestamp": "Sat Apr 28 14:56:12 CEST 2018" } @@ -473,9 +546,22 @@ The command "usage" shows how to use the scripts that were installed with each s ./use -BN -e "select @@server_id" ./use -u root - "./clear" stops the server and removes everything from the data directory, + "./clear" stops the server and removes everything from the data directory, letting you ready to start from scratch. (Warning! It's irreversible!) + "./send_kill" does almost the same as "./stop", as it sends a SIGTERM (-15) kill + to shut down the server. Additionally, when the regular kill fails, it will + send an unfriendly SIGKILL (-9) to the unresponsive server. + + "./add_option" will add one or more options to my.sandbox.cnf, and restarts the + server to apply the changes. + + "init_db" and "load_grants" are used during the server initialization, and should not be used + in normal operations. They are nonetheless useful to see which operations were performed + to set up the server. + + "./show_binlog" and "./show_relaylog" will show the latest binary log or relay-log. + "./my" is a prefix script to invoke any command named "my*" from the MySQL /bin directory. It is important to use it rather than the corresponding globally installed tool, because this guarantees @@ -485,6 +571,11 @@ The command "usage" shows how to use the scripts that were installed with each s ./my sqldump db_name ./my sqlbinlog somefile + "./mysqlsh" invokes the mysql shell. Unlike other commands, this one only works + if mysqlsh was installed, with preference to the binaries found in "basedir". + This script is created only if the X plugin was enabled (5.7.12+ with --enable-mysqlx + or 8.0.11+ without --disable-mysqlx) + USING MULTIPLE SERVER SANDBOX On a replication sandbox, you have the same commands (run "dbdeployer usage single"), with an "_all" suffix, meaning that you propagate the command to all the members. @@ -494,16 +585,16 @@ The command "usage" shows how to use the scripts that were installed with each s In group sandboxes without a master slave relationship (group replication and multiple sandboxes) the nodes can be accessed by ./n1, ./n2, ./n3, and so on. - start_all - status_all - restart_all - stop_all - use_all - use_all_masters - use_all_slaves - clear_all - m - s1, s2, n1, n2 + start_all [options] > starts all nodes + status_all > get the status of all nodes + restart_all [options] > restarts all nodes + stop_all > stops all nodes + use_all "SQL" > runs a SQL statement in all nodes + use_all_masters "SQL" > runs a SQL statement in all masters + use_all_slaves "SQL" > runs a SQL statement in all slaves + clear_all > stops all nodes and removes all data + m > invokes MySQL client in the master + s1, s2, n1, n2 > invokes MySQL client in slave 1, 2, node 1, 2 The scripts "check_slaves" or "check_nodes" give the status of replication in the sandbox. @@ -583,18 +674,18 @@ Should you need to compile your own binaries for dbdeployer, follow these steps: 2. Run ``go get github.com/datacharmer/dbdeployer``. This will import all the code that is needed to build dbdeployer. 3. Change directory to ``$GOPATH/src/github.com/datacharmer/dbdeployer``. 4. From the folder ``./pflag``, copy the file ``string_slice.go`` to ``$GOPATH/src/github.com/spf13/pflag``. -5. Run ``./build.sh {linux|OSX} 1.3.0`` -6. If you need the docs enabled binaries (see the section "Generating additional documentation") run ``MKDOCS=1 ./build.sh {linux|OSX} 1.3.0`` +5. Run ``./build.sh {linux|OSX} 1.4.0`` +6. If you need the docs enabled binaries (see the section "Generating additional documentation") run ``MKDOCS=1 ./build.sh {linux|OSX} 1.4.0`` ## Generating additional documentation Between this file and [the API API list](https://github.com/datacharmer/dbdeployer/blob/master/docs/API-1.1.md), you have all the existing documentation for dbdeployer. Should you need additional formats, though, dbdeployer is able to generate them on-the-fly. Tou will need the docs-enabled binaries: in the distribution list, you will find: -* dbdeployer-1.3.0-docs.linux.tar.gz -* dbdeployer-1.3.0-docs.osx.tar.gz -* dbdeployer-1.3.0.linux.tar.gz -* dbdeployer-1.3.0.osx.tar.gz +* dbdeployer-1.4.0-docs.linux.tar.gz +* dbdeployer-1.4.0-docs.osx.tar.gz +* dbdeployer-1.4.0.linux.tar.gz +* dbdeployer-1.4.0.osx.tar.gz The executables containing ``-docs`` in their name have the same capabilities of the regular ones, but in addition they can run the *hidden* command ``tree``, with alias ``docs``. diff --git a/cmd/deploy.go b/cmd/deploy.go index c7f11c70..c0dfa886 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -35,11 +35,14 @@ func init() { deployCmd.PersistentFlags().Bool("force", false, "If a destination sandbox already exists, it will be overwritten") deployCmd.PersistentFlags().Bool("skip-start", false, "Does not start the database server") deployCmd.PersistentFlags().Bool("disable-mysqlx", false, "Disable MySQLX plugin (8.0.11+)") + deployCmd.PersistentFlags().Bool("enable-mysqlx", false, "Enables MySQLX plugin (5.7.12+)") deployCmd.PersistentFlags().Bool("skip-load-grants", false, "Does not load the grants") deployCmd.PersistentFlags().Bool("skip-report-host", false, "Does not include report host in my.sandbox.cnf") deployCmd.PersistentFlags().Bool("skip-report-port", false, "Does not include report port in my.sandbox.cnf") deployCmd.PersistentFlags().Bool("expose-dd-tables", false, "In MySQL 8.0+ shows data dictionary tables") deployCmd.PersistentFlags().Bool("concurrent", false, "Runs multiple sandbox deployments concurrently") + deployCmd.PersistentFlags().Bool("enable-general-log", false, "Enables general log for the sandbox (MySQL 5.1+)") + deployCmd.PersistentFlags().Bool("init-general-log", false, "uses general log during initialization (MySQL 5.1+)") set_pflag(deployCmd,"remote-access", "", "", "127.%", "defines the database access ", false) set_pflag(deployCmd,"bind-address", "", "", "127.0.0.1", "defines the database bind-address ", false) diff --git a/cmd/single.go b/cmd/single.go index 4762ab50..0b4b171e 100644 --- a/cmd/single.go +++ b/cmd/single.go @@ -93,6 +93,9 @@ func FillSdef(cmd *cobra.Command, args []string) sandbox.SandboxDef { sd.SandboxDir, _ = flags.GetString("sandbox-home") common.CheckSandboxDir(sd.SandboxDir) sd.InstalledPorts = common.GetInstalledPorts(sd.SandboxDir) + for _, p := range defaults.Defaults().ReservedPorts { + sd.InstalledPorts = append(sd.InstalledPorts, p) + } sd.LoadGrants = true sd.SkipStart, _ = flags.GetBool("skip-start") skip_load_grants, _ := flags.GetBool("skip-load-grants") @@ -102,6 +105,7 @@ func FillSdef(cmd *cobra.Command, args []string) sandbox.SandboxDef { sd.SkipReportHost, _ = flags.GetBool("skip-report-host") sd.SkipReportPort, _ = flags.GetBool("skip-report-port") sd.DisableMysqlX, _ = flags.GetBool("disable-mysqlx") + sd.EnableMysqlX, _ = flags.GetBool("enable-mysqlx") sd.DbUser, _ = flags.GetString("db-user") sd.DbPassword, _ = flags.GetString("db-password") sd.RplUser, _ = flags.GetString("rpl-user") @@ -120,7 +124,13 @@ func FillSdef(cmd *cobra.Command, args []string) sandbox.SandboxDef { sd.KeepUuid, _ = flags.GetBool("keep-server-uuid") sd.Force, _ = flags.GetBool("force") sd.ExposeDdTables, _ = flags.GetBool("expose-dd-tables") + sd.InitGeneralLog, _ = flags.GetBool("init-general-log") + sd.EnableGeneralLog, _ = flags.GetBool("enable-general-log") + if sd.DisableMysqlX && sd.EnableMysqlX { + fmt.Printf("flags --enable-mysqlx and --disable-mysqlx cannot be used together\n") + os.Exit(1) + } sd.RunConcurrently, _ = flags.GetBool("concurrent") if os.Getenv("RUN_CONCURRENTLY") != "" { sd.RunConcurrently = true diff --git a/cmd/usage.go b/cmd/usage.go index 48e086ed..6a631957 100644 --- a/cmd/usage.go +++ b/cmd/usage.go @@ -46,9 +46,22 @@ Example: ./use -BN -e "select @@server_id" ./use -u root -"./clear" stops the server and removes everything from the data directory, +"./clear" stops the server and removes everything from the data directory, letting you ready to start from scratch. (Warning! It's irreversible!) +"./send_kill" does almost the same as "./stop", as it sends a SIGTERM (-15) kill +to shut down the server. Additionally, when the regular kill fails, it will +send an unfriendly SIGKILL (-9) to the unresponsive server. + +"./add_option" will add one or more options to my.sandbox.cnf, and restarts the +server to apply the changes. + +"init_db" and "load_grants" are used during the server initialization, and should not be used +in normal operations. They are nonetheless useful to see which operations were performed +to set up the server. + +"./show_binlog" and "./show_relaylog" will show the latest binary log or relay-log. + "./my" is a prefix script to invoke any command named "my*" from the MySQL /bin directory. It is important to use it rather than the corresponding globally installed tool, because this guarantees @@ -57,6 +70,11 @@ Examples: ./my sqldump db_name ./my sqlbinlog somefile + +"./mysqlsh" invokes the mysql shell. Unlike other commands, this one only works +if mysqlsh was installed, with preference to the binaries found in "basedir". +This script is created only if the X plugin was enabled (5.7.12+ with --enable-mysqlx +or 8.0.11+ without --disable-mysqlx) ` const multiple_usage string = ` USING MULTIPLE SERVER SANDBOX On a replication sandbox, you have the same commands (run "dbdeployer usage single"), @@ -67,16 +85,16 @@ the slaves (and "s3", "s4" ... if you define more). In group sandboxes without a master slave relationship (group replication and multiple sandboxes) the nodes can be accessed by ./n1, ./n2, ./n3, and so on. -start_all -status_all -restart_all -stop_all -use_all -use_all_masters -use_all_slaves -clear_all -m -s1, s2, n1, n2 +start_all [options] > starts all nodes +status_all > get the status of all nodes +restart_all [options] > restarts all nodes +stop_all > stops all nodes +use_all "SQL" > runs a SQL statement in all nodes +use_all_masters "SQL" > runs a SQL statement in all masters +use_all_slaves "SQL" > runs a SQL statement in all slaves +clear_all > stops all nodes and removes all data +m > invokes MySQL client in the master +s1, s2, n1, n2 > invokes MySQL client in slave 1, 2, node 1, 2 The scripts "check_slaves" or "check_nodes" give the status of replication in the sandbox. ` diff --git a/common/fileutil.go b/common/fileutil.go index 865d30ed..4f5df3ba 100644 --- a/common/fileutil.go +++ b/common/fileutil.go @@ -27,6 +27,7 @@ import ( "os/exec" "path/filepath" "regexp" + "time" ) type SandboxUser struct { @@ -43,6 +44,8 @@ type SandboxDescription struct { Port []int `json:"port"` Nodes int `json:"nodes"` NodeNum int `json:"node_num"` + DbDeployerVersion string `json:"dbdeployer-version"` + Timestamp string `json:"timestamp"` } type keyvalue struct { @@ -89,6 +92,8 @@ func ParseConfigFile(filename string) configOptions { } func WriteSandboxDescription(destination string, sd SandboxDescription) { + sd.DbDeployerVersion = VersionDef + sd.Timestamp = time.Now().Format(time.UnixDate) b, err := json.MarshalIndent(sd, " ", "\t") if err != nil { fmt.Println("error encoding sandbox description: ", err) @@ -206,6 +211,11 @@ func ExecExists(filename string) bool { return err == nil } +func FindInPath(filename string) string { + path, _ := exec.LookPath(filename) + return path +} + func Run_cmd_with_args(c string, args []string) (error, string) { cmd := exec.Command(c, args...) //var out bytes.Buffer diff --git a/common/strutils.go b/common/strutils.go index 1e9a6ad9..d6035803 100644 --- a/common/strutils.go +++ b/common/strutils.go @@ -19,6 +19,9 @@ import ( "fmt" "os" "regexp" + "sort" + "strings" + "strconv" ) // Given a path starting at the HOME directory @@ -81,3 +84,65 @@ func Includes(main_string, contained string) bool { return re.MatchString(main_string) } + +// Given a list of version strings (in the format x.x.x) +// this function returns an ordered list, taking into account the +// components of the versions, so that 5.6.2 sorts lower than 5.6.11 +// while a text sort would put 5.6.11 before 5.6.2 +func SortVersions(versions []string ) (sorted []string) { + type version_list struct { + text string + maj_min_rev []int + } + var vlist []version_list + for _, line := range versions { + vl := VersionToList(line) + rec := version_list { + text : line, + maj_min_rev : vl, + } + if vl[0] > 0 { + vlist = append(vlist, rec) + } + } + sort.Slice(vlist, func(a, b int) bool { + maj_a := vlist[a].maj_min_rev[0] + min_a := vlist[a].maj_min_rev[1] + rev_a := vlist[a].maj_min_rev[2] + maj_b := vlist[b].maj_min_rev[0] + min_b := vlist[b].maj_min_rev[1] + rev_b := vlist[b].maj_min_rev[2] + return maj_a < maj_b || + ( maj_a == maj_b && min_a < min_b) || + ( maj_a == maj_b && min_a == min_b && rev_a < rev_b) + }) + for _, v := range vlist { + sorted = append(sorted, v.text) + } + return +} + +func LatestVersion(versions []string ) string { + sorted := SortVersions(versions) + return sorted[0] +} + +func Atoi(val string) int { + numvalue, err := strconv.Atoi(val) + if err != nil { + fmt.Printf("Not a valid number: %s (%s)\n", val, err) + os.Exit(1) + } + return numvalue +} + + +func StringToIntSlice(val string) (num_list []int) { + list := strings.Split(val, ",") + for _, item := range list { + num_list = append(num_list, Atoi(strings.TrimSpace(item))) + } + return num_list +} + + diff --git a/common/version.go b/common/version.go index f8716b42..c4775c10 100644 --- a/common/version.go +++ b/common/version.go @@ -15,9 +15,9 @@ package common -var VersionDef string = "1.3.0" // 2018-04-15 +var VersionDef string = "1.4.0" // 2018-04-22 // Compatible version is the version used to mark compatible archives (templates, configuration). // It is usually major.minor.0, except when we are at version 0.x, when // every revision may bring incompatibility -var CompatibleVersion string = "1.3.0" // 2018-04-15 +var CompatibleVersion string = "1.4.0" // 2018-04-22 diff --git a/defaults/catalog.go b/defaults/catalog.go index 27f680cf..e16cfb0d 100644 --- a/defaults/catalog.go +++ b/defaults/catalog.go @@ -31,6 +31,8 @@ type SandboxItem struct { Port []int `json:"port"` Nodes []string `json:"nodes"` Destination string `json:"destination"` + DbDeployerVersion string `json:"dbdeployer-version"` + Timestamp string `json:"timestamp"` } type SandboxCatalog map[string]SandboxItem @@ -103,6 +105,8 @@ func ReadCatalog() (sc SandboxCatalog) { } func UpdateCatalog(sb_name string, details SandboxItem) { + details.DbDeployerVersion = common.VersionDef + details.Timestamp = time.Now().Format(time.UnixDate) if !enable_catalog_management { return } diff --git a/defaults/defaults.go b/defaults/defaults.go index f068db19..9bb0d641 100644 --- a/defaults/defaults.go +++ b/defaults/defaults.go @@ -41,6 +41,7 @@ type DbdeployerDefaults struct { // PXCBasePort int `json:"pxc-base-port"` // NdbBasePort int `json:"ndb-base-port"` GroupPortDelta int `json:"group-port-delta"` + MysqlXPortDelta int `json:"mysqlx-port-delta"` MasterName string `json:"master-name"` MasterAbbr string `json:"master-abbr"` NodePrefix string `json:"node-prefix"` @@ -53,9 +54,11 @@ type DbdeployerDefaults struct { MultiplePrefix string `json:"multiple-prefix"` FanInPrefix string `json:"fan-in-prefix"` AllMastersPrefix string `json:"all-masters-prefix"` + ReservedPorts []int `json:"reserved-ports"` // GaleraPrefix string `json:"galera-prefix"` // PxcPrefix string `json:"pxc-prefix"` // NdbPrefix string `json:"ndb-prefix"` + Timestamp string `json:"timestamp"` } const ( @@ -92,6 +95,7 @@ var ( // PxcBasePort: 18000, // NdbBasePort: 19000, GroupPortDelta: 125, + MysqlXPortDelta: 10000, MasterName: "master", MasterAbbr: "m", NodePrefix: "node", @@ -104,9 +108,11 @@ var ( MultiplePrefix: "multi_msb_", FanInPrefix: "fan_in_msb_", AllMastersPrefix: "all_masters_msb_", + ReservedPorts: []int{1186, 3306, 33060}, // GaleraPrefix: "galera_msb_", // NdbPrefix: "ndb_msb_", // PxcPrefix: "pxc_msb_", + Timestamp: time.Now().Format(time.UnixDate), } currentDefaults DbdeployerDefaults ) @@ -200,6 +206,7 @@ func ValidateDefaults(nd DbdeployerDefaults) bool { // check_int("pxc-base-port", nd.PxcBasePort, min_port_value, max_port_value) && // check_int("ndb-base-port", nd.NdbBasePort, min_port_value, max_port_value) && check_int("group-port-delta", nd.GroupPortDelta, 101, 299) + check_int("mysqlx-port-delta", nd.MysqlXPortDelta, 2000, 15000) if !all_ints { return false } @@ -324,6 +331,8 @@ func UpdateDefaults(label, value string, store_defaults bool) { // new_defaults.PxcBasePort = a_to_i(value) case "group-port-delta": new_defaults.GroupPortDelta = a_to_i(value) + case "mysqlx-port-delta": + new_defaults.MysqlXPortDelta = a_to_i(value) case "master-name": new_defaults.MasterName = value case "master-abbr": @@ -348,6 +357,8 @@ func UpdateDefaults(label, value string, store_defaults bool) { new_defaults.FanInPrefix = value case "all-masters-prefix": new_defaults.AllMastersPrefix = value + case "reserved-ports": + new_defaults.ReservedPorts = common.StringToIntSlice(value) // case "galera-prefix": // new_defaults.GaleraPrefix = value // case "pxc-prefix": diff --git a/docs/API-1.0.md b/docs/API/API-1.0.md similarity index 100% rename from docs/API-1.0.md rename to docs/API/API-1.0.md diff --git a/docs/API-1.1.md b/docs/API/API-1.1.md similarity index 100% rename from docs/API-1.1.md rename to docs/API/API-1.1.md diff --git a/docs/API-1.2.md b/docs/API/API-1.2.md similarity index 100% rename from docs/API-1.2.md rename to docs/API/API-1.2.md diff --git a/docs/API-1.3.md b/docs/API/API-1.3.md similarity index 100% rename from docs/API-1.3.md rename to docs/API/API-1.3.md diff --git a/docs/API/API-1.4.md b/docs/API/API-1.4.md new file mode 100644 index 00000000..12b33a63 --- /dev/null +++ b/docs/API/API-1.4.md @@ -0,0 +1,652 @@ +This is the list of commands and modifiers available for +dbdeployer 1.4.0 as of 28-Apr-2018 12:56 UTC + +# main + $ dbdeployer -h + dbdeployer makes MySQL server installation an easy task. + Runs single, multiple, and replicated sandboxes. + + Usage: + dbdeployer [command] + + Available Commands: + admin sandbox management tasks + defaults tasks related to dbdeployer defaults + delete delete an installed sandbox + deploy deploy sandboxes + global Runs a given command in every sandbox + help Help about any command + sandboxes List installed sandboxes + unpack unpack a tarball into the binary directory + usage Shows usage of installed sandboxes + versions List available versions + + Flags: + --config string configuration file (default "$HOME/.dbdeployer/config.json") + -h, --help help for dbdeployer + --sandbox-binary string Binary repository (default "$HOME/opt/mysql") + --sandbox-home string Sandbox deployment directory (default "$HOME/sandboxes") + --version version for dbdeployer + + Use "dbdeployer [command] --help" for more information about a command. + + + $ dbdeployer-docs tree + - admin + - lock + - unlock + - defaults + - export + - load + - reset + - show + - store + - templates + - describe + - export + - import + - list + - reset + - show + - update + - delete + - deploy + - multiple + - replication + - single + - global + - restart + - start + - status + - stop + - test + - test-replication + - use + - sandboxes + - unpack + - usage + - versions + + +## admin + $ dbdeployer admin -h + Runs commands related to the administration of sandboxes. + + Usage: + dbdeployer admin [command] + + Aliases: + admin, manage + + Available Commands: + lock Locks a sandbox, preventing deletion + unlock Unlocks a sandbox + + Flags: + -h, --help help for admin + + + $ dbdeployer admin lock -h + Prevents deletion for a given sandbox. + Note that the deletion being prevented is only the one occurring through dbdeployer. + Users can still delete locked sandboxes manually. + + Usage: + dbdeployer admin lock sandbox_name [flags] + + Aliases: + lock, preserve + + Flags: + -h, --help help for lock + + + $ dbdeployer admin unlock -h + Removes lock, allowing deletion of a given sandbox + + Usage: + dbdeployer admin unlock sandbox_name [flags] + + Aliases: + unlock, unpreserve + + Flags: + -h, --help help for unlock + + + +## defaults + $ dbdeployer defaults -h + Runs commands related to the administration of dbdeployer, + such as showing the defaults and saving new ones. + + Usage: + dbdeployer defaults [command] + + Aliases: + defaults, config + + Available Commands: + export Export current defaults to a given file + load Load defaults from file + reset Remove current defaults file + show shows defaults + store Store current defaults + templates Templates management + update Load defaults from file + + Flags: + -h, --help help for defaults + + + $ dbdeployer defaults export -h + Saves current defaults to a user-defined file + + Usage: + dbdeployer defaults export filename [flags] + + Flags: + -h, --help help for export + + + $ dbdeployer defaults load -h + Reads defaults from file and saves them to dbdeployer configuration file ($HOME/.dbdeployer/config.json) + + Usage: + dbdeployer defaults load file_name [flags] + + Aliases: + load, import + + Flags: + -h, --help help for load + + + $ dbdeployer defaults reset -h + Removes current dbdeployer configuration file ($HOME/.dbdeployer/config.json) + Afterwards, dbdeployer will use the internally stored defaults. + + Usage: + dbdeployer defaults reset [flags] + + Aliases: + reset, remove + + Flags: + -h, --help help for reset + + + $ dbdeployer defaults show -h + Shows currently defined defaults + + Usage: + dbdeployer defaults show [flags] + + Aliases: + show, list + + Flags: + -h, --help help for show + + + $ dbdeployer defaults store -h + Saves current defaults to dbdeployer configuration file ($HOME/.dbdeployer/config.json) + + Usage: + dbdeployer defaults store [flags] + + Flags: + -h, --help help for store + + + +## defaults templates + $ dbdeployer defaults templates -h + The commands in this section show the templates used + to create and manipulate sandboxes. + + Usage: + dbdeployer defaults templates [command] + + Aliases: + templates, template, tmpl, templ + + Available Commands: + describe Describe a given template + export Exports templates to a directory + import imports templates from a directory + list list available templates + reset Removes all template files + show Show a given template + + Flags: + -h, --help help for templates + + + $ dbdeployer defaults templates describe -h + Describe a given template + + Usage: + dbdeployer defaults templates describe template_name [flags] + + Aliases: + describe, descr, structure, struct + + Flags: + -h, --help help for describe + --with-contents Shows complete structure and contents + + + $ dbdeployer defaults templates export -h + Exports a group of templates (or "ALL") to a given directory + + Usage: + dbdeployer defaults templates export group_name directory_name [template_name] [flags] + + Flags: + -h, --help help for export + + + $ dbdeployer defaults templates import -h + Imports a group of templates (or "ALL") from a given directory + + Usage: + dbdeployer defaults templates import group_name directory_name [template_name] [flags] + + Flags: + -h, --help help for import + + + $ dbdeployer defaults templates list -h + list available templates + + Usage: + dbdeployer defaults templates list [group] [flags] + + Flags: + -h, --help help for list + -s, --simple Shows only the template names, without description + + + $ dbdeployer defaults templates reset -h + Removes all template files that were imported and starts using internal values. + + Usage: + dbdeployer defaults templates reset [flags] + + Aliases: + reset, remove + + Flags: + -h, --help help for reset + + + $ dbdeployer defaults templates show -h + Show a given template + + Usage: + dbdeployer defaults templates show template_name [flags] + + Flags: + -h, --help help for show + + + $ dbdeployer defaults update -h + Updates one field of the defaults. Stores the result in the dbdeployer configuration file. + Use "dbdeployer defaults show" to see which values are available + + Usage: + dbdeployer defaults update label value [flags] + + Examples: + + $ dbdeployer defaults update master-slave-base-port 17500 + + + Flags: + -h, --help help for update + + + +## delete + $ dbdeployer delete -h + Stops the sandbox (and its depending sandboxes, if any), and removes it. + Warning: this command is irreversible! + + Usage: + dbdeployer delete sandbox_name (or "ALL") [flags] + + Aliases: + delete, remove, destroy + + Examples: + + $ dbdeployer delete msb_8_0_4 + $ dbdeployer delete rsandbox_5_7_21 + + Flags: + --concurrent Runs multiple deletion tasks concurrently. + --confirm Requires confirmation. + -h, --help help for delete + --skip-confirm Skips confirmation with multiple deletions. + + + +## deploy + $ dbdeployer deploy -h + Deploys single, multiple, or replicated sandboxes + + Usage: + dbdeployer deploy [command] + + Available Commands: + multiple create multiple sandbox + replication create replication sandbox + single deploys a single sandbox + + Flags: + --base-port int Overrides default base-port (for multiple sandboxes) + --bind-address string defines the database bind-address (default "127.0.0.1") + --concurrent Runs multiple sandbox deployments concurrently + --custom-mysqld string Uses an alternative mysqld (must be in the same directory as regular mysqld) + -p, --db-password string database password (default "msandbox") + -u, --db-user string database user (default "msandbox") + --defaults strings Change defaults on-the-fly (--defaults=label:value) + --disable-mysqlx Disable MySQLX plugin (8.0.11+) + --enable-general-log Enables general log for the sandbox (MySQL 5.1+) + --enable-mysqlx Enables MySQLX plugin (5.7.12+) + --expose-dd-tables In MySQL 8.0+ shows data dictionary tables + --force If a destination sandbox already exists, it will be overwritten + --gtid enables GTID + -h, --help help for deploy + --init-general-log uses general log during initialization (MySQL 5.1+) + -i, --init-options strings mysqld options to run during initialization + --keep-server-uuid Does not change the server UUID + --my-cnf-file string Alternative source file for my.sandbox.cnf + -c, --my-cnf-options strings mysqld options to add to my.sandbox.cnf + --native-auth-plugin in 8.0.4+, uses the native password auth plugin + --port int Overrides default port + --post-grants-sql strings SQL queries to run after loading grants + --post-grants-sql-file string SQL file to run after loading grants + --pre-grants-sql strings SQL queries to run before loading grants + --pre-grants-sql-file string SQL file to run before loading grants + --remote-access string defines the database access (default "127.%") + --rpl-password string replication password (default "rsandbox") + --rpl-user string replication user (default "rsandbox") + --sandbox-directory string Changes the default sandbox directory + --skip-load-grants Does not load the grants + --skip-report-host Does not include report host in my.sandbox.cnf + --skip-report-port Does not include report port in my.sandbox.cnf + --skip-start Does not start the database server + --use-template strings [template_name:file_name] Replace existing template with one from file + + + $ dbdeployer deploy multiple -h + Creates several sandboxes of the same version, + without any replication relationship. + For this command to work, there must be a directory $HOME/opt/mysql/5.7.21, containing + the binary files from mysql-5.7.21-$YOUR_OS-x86_64.tar.gz + Use the "unpack" command to get the tarball into the right directory. + + Usage: + dbdeployer deploy multiple MySQL-Version [flags] + + Examples: + + $ dbdeployer deploy multiple 5.7.21 + + + Flags: + -h, --help help for multiple + -n, --nodes int How many nodes will be installed (default 3) + + + $ dbdeployer deploy replication -h + The replication command allows you to deploy several nodes in replication. + Allowed topologies are "master-slave" for all versions, and "group", "all-masters", "fan-in" + for 5.7.17+. + For this command to work, there must be a directory $HOME/opt/mysql/5.7.21, containing + the binary files from mysql-5.7.21-$YOUR_OS-x86_64.tar.gz + Use the "unpack" command to get the tarball into the right directory. + + Usage: + dbdeployer deploy replication MySQL-Version [flags] + + Examples: + + $ dbdeployer deploy replication 5.7.21 + # (implies topology = master-slave) + + $ dbdeployer deploy --topology=master-slave replication 5.7.21 + # (explicitly setting topology) + + $ dbdeployer deploy --topology=group replication 5.7.21 + $ dbdeployer deploy --topology=group replication 8.0.4 --single-primary + $ dbdeployer deploy --topology=all-masters replication 5.7.21 + $ dbdeployer deploy --topology=fan-in replication 5.7.21 + + + Flags: + -h, --help help for replication + --master-ip string Which IP the slaves will connect to (default "127.0.0.1") + --master-list string Which nodes are masters in a multi-source deployment (default "1,2") + -n, --nodes int How many nodes will be installed (default 3) + --semi-sync Use semi-synchronous plugin + --single-primary Using single primary for group replication + --slave-list string Which nodes are slaves in a multi-source deployment (default "3") + -t, --topology string Which topology will be installed (default "master-slave") + + + $ dbdeployer deploy single -h + single installs a sandbox and creates useful scripts for its use. + MySQL-Version is in the format x.x.xx, and it refers to a directory named after the version + containing an unpacked tarball. The place where these directories are found is defined by + --sandbox-binary (default: $HOME/opt/mysql.) + For example: + dbdeployer deploy single 5.7.21 + + For this command to work, there must be a directory $HOME/opt/mysql/5.7.21, containing + the binary files from mysql-5.7.21-$YOUR_OS-x86_64.tar.gz + Use the "unpack" command to get the tarball into the right directory. + + Usage: + dbdeployer deploy single MySQL-Version [flags] + + Flags: + -h, --help help for single + --master Make the server replication ready + + + +## global + $ dbdeployer global -h + This command can propagate the given action through all sandboxes. + + Usage: + dbdeployer global [command] + + Examples: + + $ dbdeployer global use "select version()" + $ dbdeployer global status + $ dbdeployer global stop + + + Available Commands: + restart Restarts all sandboxes + start Starts all sandboxes + status Shows the status in all sandboxes + stop Stops all sandboxes + test Tests all sandboxes + test-replication Tests replication in all sandboxes + use Runs a query in all sandboxes + + Flags: + -h, --help help for global + + + $ dbdeployer global restart -h + Restarts all sandboxes + + Usage: + dbdeployer global restart [options] [flags] + + Flags: + -h, --help help for restart + + + $ dbdeployer global start -h + Starts all sandboxes + + Usage: + dbdeployer global start [options] [flags] + + Flags: + -h, --help help for start + + + $ dbdeployer global status -h + Shows the status in all sandboxes + + Usage: + dbdeployer global status [flags] + + Flags: + -h, --help help for status + + + $ dbdeployer global stop -h + Stops all sandboxes + + Usage: + dbdeployer global stop [flags] + + Flags: + -h, --help help for stop + + + $ dbdeployer global test -h + Tests all sandboxes + + Usage: + dbdeployer global test [flags] + + Aliases: + test, test_sb, test-sb + + Flags: + -h, --help help for test + + + $ dbdeployer global test-replication -h + Tests replication in all sandboxes + + Usage: + dbdeployer global test-replication [flags] + + Aliases: + test-replication, test_replication + + Flags: + -h, --help help for test-replication + + + $ dbdeployer global use -h + Runs a query in all sandboxes. + It does not check if the query is compatible with every version deployed. + For example, a query using @@port won't run in MySQL 5.0.x + + Usage: + dbdeployer global use {query} [flags] + + Examples: + + $ dbdeployer global use "select @@server_id, @@port" + + Flags: + -h, --help help for use + + + +## sandboxes + $ dbdeployer sandboxes -h + Lists all sandboxes installed in $SANDBOX_HOME. + If sandboxes are installed in a different location, use --sandbox-home to + indicate where to look. + Alternatively, using --catalog will list all sandboxes, regardless of where + they were deployed. + + Usage: + dbdeployer sandboxes [flags] + + Aliases: + sandboxes, installed, deployed + + Flags: + --catalog Use sandboxes catalog instead of scanning directory + --header Shows header with catalog output + -h, --help help for sandboxes + + + +## unpack + $ dbdeployer unpack -h + If you want to create a sandbox from a tarball, you first need to unpack it + into the sandbox-binary directory. This command carries out that task, so that afterwards + you can call 'single', 'multiple', and 'replication' commands with only the MySQL version + for that tarball. + If the version is not contained in the tarball name, it should be supplied using --unpack-version. + If there is already an expanded tarball with the same version, a new one can be differentiated with --prefix. + + Usage: + dbdeployer unpack MySQL-tarball [flags] + + Aliases: + unpack, extract, untar, unzip, inflate, expand + + Examples: + + $ dbdeployer unpack mysql-8.0.4-rc-linux-glibc2.12-x86_64.tar.gz + Unpacking tarball mysql-8.0.4-rc-linux-glibc2.12-x86_64.tar.gz to $HOME/opt/mysql/8.0.4 + + $ dbdeployer unpack --prefix=ps Percona-Server-5.7.21-linux.tar.gz + Unpacking tarball Percona-Server-5.7.21-linux.tar.gz to $HOME/opt/mysql/ps5.7.21 + + $ dbdeployer unpack --unpack-version=8.0.18 --prefix=bld mysql-mybuild.tar.gz + Unpacking tarball mysql-mybuild.tar.gz to $HOME/opt/mysql/bld8.0.18 + + + Flags: + -h, --help help for unpack + --prefix string Prefix for the final expanded directory + --unpack-version string which version is contained in the tarball + --verbosity int Level of verbosity during unpack (0=none, 2=maximum) (default 1) + + + +## usage + $ dbdeployer usage -h + Shows syntax and examples of tools installed in database sandboxes. + + Usage: + dbdeployer usage [single|multiple] [flags] + + Flags: + -h, --help help for usage + + + +## versions + $ dbdeployer versions -h + List available versions + + Usage: + dbdeployer versions [flags] + + Aliases: + versions, available + + Flags: + -h, --help help for versions + + diff --git a/docs/dbdeployer_completion.sh b/docs/dbdeployer_completion.sh index 1ff9f71c..b72a62eb 100644 --- a/docs/dbdeployer_completion.sh +++ b/docs/dbdeployer_completion.sh @@ -636,9 +636,12 @@ _dbdeployer_deploy_multiple() two_word_flags+=("-u") flags+=("--defaults=") flags+=("--disable-mysqlx") + flags+=("--enable-general-log") + flags+=("--enable-mysqlx") flags+=("--expose-dd-tables") flags+=("--force") flags+=("--gtid") + flags+=("--init-general-log") flags+=("--init-options=") two_word_flags+=("-i") flags+=("--keep-server-uuid") @@ -699,9 +702,12 @@ _dbdeployer_deploy_replication() two_word_flags+=("-u") flags+=("--defaults=") flags+=("--disable-mysqlx") + flags+=("--enable-general-log") + flags+=("--enable-mysqlx") flags+=("--expose-dd-tables") flags+=("--force") flags+=("--gtid") + flags+=("--init-general-log") flags+=("--init-options=") two_word_flags+=("-i") flags+=("--keep-server-uuid") @@ -754,9 +760,12 @@ _dbdeployer_deploy_single() two_word_flags+=("-u") flags+=("--defaults=") flags+=("--disable-mysqlx") + flags+=("--enable-general-log") + flags+=("--enable-mysqlx") flags+=("--expose-dd-tables") flags+=("--force") flags+=("--gtid") + flags+=("--init-general-log") flags+=("--init-options=") two_word_flags+=("-i") flags+=("--keep-server-uuid") @@ -810,9 +819,12 @@ _dbdeployer_deploy() two_word_flags+=("-u") flags+=("--defaults=") flags+=("--disable-mysqlx") + flags+=("--enable-general-log") + flags+=("--enable-mysqlx") flags+=("--expose-dd-tables") flags+=("--force") flags+=("--gtid") + flags+=("--init-general-log") flags+=("--init-options=") two_word_flags+=("-i") flags+=("--keep-server-uuid") @@ -1047,6 +1059,7 @@ _dbdeployer_unpack() flags+=("--prefix=") flags+=("--unpack-version=") + flags+=("--verbosity=") flags+=("--config=") flags+=("--sandbox-binary=") flags+=("--sandbox-home=") diff --git a/mkreadme/readme_template.md b/mkreadme/readme_template.md index bcc2f27a..2deaf763 100644 --- a/mkreadme/readme_template.md +++ b/mkreadme/readme_template.md @@ -78,12 +78,49 @@ If you want to deploy several instances of the same version and the same type (f $ dbdeployer deploy single 8.0.4 # will deploy in msb_8_0_4 using port 8004 - $ dbdeployer deploy single 8.0.4 --sandbox-directory=msb2_8_0_4 --port=8005 - # will deploy in msb2_8_0_4 using port 8005 + $ dbdeployer deploy single 8.0.4 --sandbox-directory=msb2_8_0_4 + # will deploy in msb2_8_0_4 using port 8005 (which dbdeployer detects and uses) $ dbdeployer deploy replication 8.0.4 --sandbox-directory=rsandbox2_8_0_4 --base-port=18600 # will deploy replication in rsandbox2_8_0_4 using ports 18601, 18602, 18603 +## Ports management + +dbdeployer will try using the default port for each sandbox whenever possible. For single sandboxes, the port will be the version number without dots: 5.7.22 will deploy on port 5722. For multiple sandboxes, the port number is defined by using a prefix number (visible in the defaults: ``dbdeployer defaults list``) + the port number + the revision number (for some topologies multiplied by 100.) +For example, single-primary group replication with MySQL 8.0.11 will compute the ports like this: + + base port = 8011 (version number) + 13000 (prefix) + 11 (revision) * 100 = 22111 + node1 port = base port + 1 = 22112 + node2 port = base port + 2 = 22113 + node3 port = base port + 2 = 22114 + +For group replication we need to calculate the group port, and we use the ``group-port-delta`` (= 125) to obtain it from the regular port: + + node1 group port = 22112 + 125 = 22237 + node2 group port = 22113 + 125 = 22238 + node3 group port = 22114 + 125 = 22239 + +For MySQL 8.0.11+, we also need to assign a port for the XPlugin, and we compute that using the regular port + the ``mysqlx-port-delta`` (=10000). + +Thus, for MySQL 8.0.11 group replication deployments, you would see this listing: + + $ dbdeployer sandboxes --header + name type version ports + ---------------- ------- ------- ----- + group_msb_8_0_11 : group-multi-primary 8.0.11 [20023 20148 30023 20024 20149 30024 20025 20150 30025] + group_sp_msb_8_0_11 : group-single-primary 8.0.11 [22112 22237 32112 22113 22238 32113 22114 22239 32114] + +This method makes port clashes unlikely when using the same version in different deployments, but there is a risk of port clashes when deploying many multiple sandboxes of close-by versions. +However, dbdeployer doesn't let the clash happen. Thanks to its central catalog of sandboxes, it knows which ports were already used, and will search for free ones whenever a potential clash is detected. +Bear in mind that the concept of "used" is only related to sandboxes. dbdeployer does not know if ports may be used by other applications. +You can minimize risks, however, by telling dbdeployer which ports may be occupied. The defaults have a field ``reserved-ports``, containing the ports that should not be used. You can add to that list by modifying the defaults. For example, if you want to exclude port 7001, 10000, and 15000 from being used, you can run + + dbdeployer defaults update reserved-ports '7001,10000,15000' + +or, if you want to preserve the ones that are reserved by default: + + dbdeployer defaults update reserved-ports '1186,3306,33060,7001,10000,15000' + ## Concurrent deployment and deletion Starting with version 0.3.0, dbdeployer can deploy groups of sandboxes (``deploy replication``, ``deploy multiple``) with the flag ``--concurrent``. When this flag is used, dbdeployed will run operations concurrently. @@ -150,6 +187,25 @@ Similarly, for group replication WARNING: running sandboxes with ``--skip-start`` is provided for advanced users and is not recommended. If the purpose of skipping the start is to inspect the server before the sandbox granting operations, you may consider using ``--pre-grants-sql`` and ``--pre-grants-sql-file`` to run the necessary SQL commands (see _Sandbox customization_ below.) +## MySQL Document store, mysqlsh, and defaults. + +MySQL 5.7.12+ introduces the XPlugin (a.k.a. _mysqlx_) which enables operations using a separate port (33060 by default) on special tables that can be treated as NoSQL collections. +In MySQL 8.0.11+ the XPlugin is enabled by default, giving dbdeployer the task of defining an additional port and socket for this service. When you deploy MySQL 8.0.11 or later, dbdeployer sets the ``mysqlx-port`` to the value of the regular port + ``mysqlx-delta-port`` (= 10000). + +If you want to avoid having the XPlugin enabled, you can deploy the sandbox with the option ``--disable-mysqlx``. + +For MySQL between 5.7.12 and 8.0.4, the approach is the opposite. By default, the XPlugin is disabled, and if you want to use it you will run the deployment using ``--enable-mysqlx``. In both cases the port and socket will be computed by dbdeployer. + +When the XPlugin is enabled, it makes sense to use [the MySQL shell](https://dev.mysql.com/doc/refman/8.0/en/mysql-shell.html) and dbdeployer will create a ``mysqlsh`` script for the sandboxes that use the plugin. Unfortunately, as of today (late April 2018) the MySQL shell is not released with the server tarball, and therefore we have to fix things manually. dbdeployer will look for ``mysqlsh`` in the same directory where the other clients are, so if you manually merge the mysql shell and the server tarballs, you will get the appropriate version of MySQL shell. If not, you will use the version of the shell that is available in ``$PATH``. If there is no MySQL shell available, you will get an error. + +## Logs management. + +Sometimes, when using sandboxes for testing, it makes sense to enable the general log, either during initialization or for regular operation. While you can do that with ``--my-cnf-options=general-log=1`` or ``--my-init-options=--general-log=1``, as of version 1.4.0 you have two easy boolean shortcuts: ``--init-general-log`` and ``--enable-general-log`` that will start the general log when requested. + +Additionally, each sandbox has a convenience script named ``show_log`` that can easily display either the error log or the general log. Run `./show_log -h` for usage info. + +For replication, you also have ``show_binlog`` and ``show_relaylog`` in every sandbox as a shortcut to display replication logs easily. + ## Sandbox customization There are several ways of changing the default behavior of a sandbox. @@ -250,7 +306,7 @@ Should you need to compile your own binaries for dbdeployer, follow these steps: ## Generating additional documentation -Between this file and [the API API list](https://github.com/datacharmer/dbdeployer/blob/master/docs/API-1.1.md), you have all the existing documentation for dbdeployer. +Between this file and [the API API list](https://github.com/datacharmer/dbdeployer/blob/master/docs/API/API-1.1.md), you have all the existing documentation for dbdeployer. Should you need additional formats, though, dbdeployer is able to generate them on-the-fly. Tou will need the docs-enabled binaries: in the distribution list, you will find: * dbdeployer-{{.Version}}-docs.linux.tar.gz @@ -300,7 +356,7 @@ As of version 1.0.0, dbdeployer adheres to the principles of [semantic versionin * Backward-compatible new features increment the **Minor** number. * Backward incompatible changes (either features or bug fixes that break compatibility with the API) increment the **Major** number. -The starting API is defined in [API-1.0.md](https://github.com/datacharmer/dbdeployer/blob/master/docs/API-1.0.md) (generated manually.) -The file [API-1.1.md](https://github.com/datacharmer/dbdeployer/blob/master/docs/API-1.1.md) contains the same API definition, but was generated automatically and can be used to better compare the initial API with further version. +The starting API is defined in [API-1.0.md](https://github.com/datacharmer/dbdeployer/blob/master/docs/API/API-1.0.md) (generated manually.) +The file [API-1.1.md](https://github.com/datacharmer/dbdeployer/blob/master/docs/API/API-1.1.md) contains the same API definition, but was generated automatically and can be used to better compare the initial API with further version. diff --git a/sandbox/group_replication.go b/sandbox/group_replication.go index b94164ea..415015c1 100644 --- a/sandbox/group_replication.go +++ b/sandbox/group_replication.go @@ -46,7 +46,7 @@ loose-group-replication-single-primary-mode=off ) func get_base_mysqlx_port(base_port int, sdef SandboxDef, nodes int) int { - base_mysqlx_port := base_port + 10000 + base_mysqlx_port := base_port + defaults.Defaults().MysqlXPortDelta if common.GreaterOrEqualVersion(sdef.Version, []int{8,0,11}) { base_mysqlx_port = FindFreePort(base_mysqlx_port, sdef.InstalledPorts, nodes) for check_port := base_mysqlx_port + 1; check_port < base_mysqlx_port+nodes+1; check_port++ { diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index 6dcf4320..be6fa3ad 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -61,8 +61,11 @@ type SandboxDef struct { PostGrantsSql []string PostGrantsSqlFile string MyCnfFile string + InitGeneralLog bool + EnableGeneralLog bool NativeAuthPlugin bool DisableMysqlX bool + EnableMysqlX bool KeepUuid bool SinglePrimary bool Force bool @@ -177,13 +180,6 @@ func CheckPort(sandbox_type string, installed_ports []int, port int) { if p == port { conflict = p } - /* - if sandbox_type == "group-node" { - if p == (port + defaults.Defaults().GroupPortDelta) { - conflict = p - } - } - */ } if conflict > 0 { fmt.Printf("Port conflict detected. Port %d is already used\n", conflict) @@ -236,6 +232,18 @@ func slice_to_text(s_array []string) string { return text } +func set_mysqlx_properties(sdef SandboxDef, global_tmp_dir string) SandboxDef { + mysqlx_port := sdef.MysqlXPort + if mysqlx_port == 0 { + mysqlx_port = FindFreePort(sdef.Port + defaults.Defaults().MysqlXPortDelta, sdef.InstalledPorts, 1) + } + sdef.MyCnfOptions = append(sdef.MyCnfOptions, fmt.Sprintf("mysqlx-port=%d", mysqlx_port)) + sdef.MyCnfOptions = append(sdef.MyCnfOptions, fmt.Sprintf("mysqlx-socket=%s/mysqlx-%d.sock", global_tmp_dir, mysqlx_port)) + sdef.MorePorts = append(sdef.MorePorts, mysqlx_port) + sdef.MysqlXPort = mysqlx_port + return sdef +} + func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent.ExecutionList) { var sandbox_dir string @@ -268,6 +276,20 @@ func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent fmt.Printf("TMP directory %s does not exist\n", global_tmp_dir) os.Exit(1) } + if sdef.NodeNum == 0 && !sdef.Force { + sdef.Port = FindFreePort(sdef.Port, sdef.InstalledPorts, 1) + } + if sdef.EnableMysqlX { + if !common.GreaterOrEqualVersion(sdef.Version, []int{5, 7, 12}) { + fmt.Printf("option --enable-mysqlx requires version 5.7.12+\n") + os.Exit(1) + } + // If the version is 8.0.11 or later, MySQL X is enabled already + if !common.GreaterOrEqualVersion(sdef.Version, []int{8, 0, 11}) { + sdef.MyCnfOptions = append(sdef.MyCnfOptions, "plugin_load=mysqlx=mysqlx.so") + sdef = set_mysqlx_properties(sdef, global_tmp_dir) + } + } if sdef.ExposeDdTables { if !common.GreaterOrEqualVersion(sdef.Version, []int{8, 0, 0}) { fmt.Printf("--expose-dd-tables requires MySQL 8.0.0+\n") @@ -289,6 +311,14 @@ func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent os.Exit(1) } } + if common.GreaterOrEqualVersion(sdef.Version, []int{5, 1, 0}) { + if sdef.EnableGeneralLog { + sdef.MyCnfOptions = append(sdef.MyCnfOptions, "general_log=1") + } + if sdef.InitGeneralLog { + sdef.InitOptions = append(sdef.InitOptions, "--general_log=1") + } + } if common.GreaterOrEqualVersion(sdef.Version, []int{8, 0, 4}) { if sdef.NativeAuthPlugin == true { sdef.InitOptions = append(sdef.InitOptions, "--default_authentication_plugin=mysql_native_password") @@ -299,15 +329,13 @@ func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent if sdef.DisableMysqlX { sdef.MyCnfOptions = append(sdef.MyCnfOptions, "mysqlx=OFF") } else { - mysqlx_port := sdef.MysqlXPort - if mysqlx_port == 0 { - mysqlx_port = FindFreePort(sdef.Port + 10000, sdef.InstalledPorts, 1) - } - sdef.MyCnfOptions = append(sdef.MyCnfOptions, fmt.Sprintf("mysqlx-port=%d", mysqlx_port)) - sdef.MyCnfOptions = append(sdef.MyCnfOptions, fmt.Sprintf("mysqlx-socket=%s/mysqlx-%d.sock", global_tmp_dir, mysqlx_port)) - sdef.MorePorts = append(sdef.MorePorts, mysqlx_port) + sdef = set_mysqlx_properties(sdef, global_tmp_dir) } } + mysqlsh_executable := fmt.Sprintf("%s/bin/mysqlsh", sdef.Basedir) + if !common.ExecExists(mysqlsh_executable) { + mysqlsh_executable = "mysqlsh" + } if sdef.MyCnfFile != "" { options := GetOptionsFromFile(sdef.MyCnfFile) if len(options) > 0 { @@ -318,10 +346,6 @@ func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent sdef.MyCnfOptions = append(sdef.MyCnfOptions, option) } } - //fmt.Printf("%#v\n", sdef) - if sdef.NodeNum == 0 && !sdef.Force { - sdef.Port = FindFreePort(sdef.Port, sdef.InstalledPorts, 1) - } timestamp := time.Now() var data common.Smap = common.Smap{"Basedir": sdef.Basedir, "Copyright": SingleTemplates["Copyright"].Contents, @@ -330,6 +354,8 @@ func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent "SandboxDir": sandbox_dir, "CustomMysqld": sdef.CustomMysqld, "Port": sdef.Port, + "MysqlXPort": sdef.MysqlXPort, + "MysqlShell": mysqlsh_executable, "BasePort": sdef.BasePort, "Prompt": sdef.Prompt, "Version": sdef.Version, @@ -461,6 +487,10 @@ func CreateSingleSandbox(sdef SandboxDef, origin string) (exec_list []concurrent write_script(SingleTemplates, "stop", "stop_template", sandbox_dir, data, true) write_script(SingleTemplates, "clear", "clear_template", sandbox_dir, data, true) write_script(SingleTemplates, "use", "use_template", sandbox_dir, data, true) + if sdef.MysqlXPort != 0 { + write_script(SingleTemplates, "mysqlsh", "mysqlsh_template", sandbox_dir, data, true) + } + write_script(SingleTemplates, "show_log", "show_log_template", sandbox_dir, data, true) write_script(SingleTemplates, "send_kill", "send_kill_template", sandbox_dir, data, true) write_script(SingleTemplates, "restart", "restart_template", sandbox_dir, data, true) write_script(SingleTemplates, "load_grants", "load_grants_template", sandbox_dir, data, true) diff --git a/sandbox/templates.go b/sandbox/templates.go index 66620e52..ef1041ea 100644 --- a/sandbox/templates.go +++ b/sandbox/templates.go @@ -201,6 +201,26 @@ var ( exit 1 fi ` + mysqlsh_template string = `#!/bin/bash +{{.Copyright}} +# Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} +BASEDIR={{.Basedir}} +export LD_LIBRARY_PATH=$BASEDIR/lib:$BASEDIR/lib/mysql:$LD_LIBRARY_PATH +export DYLD_LIBRARY_PATH=$BASEDIR/lib:$BASEDIR/lib/mysql:$DYLD_LIBRARY_PATH +SBDIR={{.SandboxDir}} +PIDFILE=$SBDIR/data/mysql_sandbox{{.Port}}.pid +[ -z "$MYSQL_SHELL" ] && MYSQL_SHELL="{{.MysqlShell}}" + +URI="{{.DbUser}}:{{.DbPassword}}@127.0.0.1:{{.MysqlXPort}}" + +if [ -f $PIDFILE ] +then + $MYSQL_SHELL --uri="$URI" "$@" +else + exit 1 +fi +` + stop_template string = `#!/bin/bash {{.Copyright}} # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} @@ -601,6 +621,92 @@ if [ -n "$CHANGED" ] then ./restart fi +` + show_log_template string = `#!/bin/bash +{{.Copyright}} +# Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} + +SBDIR="{{.SandboxDir}}" +cd $SBDIR + +if [ ! -d ./data ] +then + echo "$SBDIR/data not found" + exit 1 +fi + +log=$1 +[ -z "$log" ] && log='err' +function get_help { + exit_code=$1 + [ -z "$exit_code" ] && exit_code=0 + echo "# Usage: $0 [log] " + echo "# Where 'log' is one of 'err' (error log), 'gen' (general log)" + echo "# Or it can be a variable name that identifies a log" + echo "# (The default is 'err')" + exit $exit_code +} + +if [ "$log" == "-h" -o "$log" == "--help" -o "$log" == "-help" -o "$log" == "help" ] +then + get_help 0 +fi + +# Checks if the output is a terminal or a pipe +if [ -t 1 ] +then + echo "###################### WARNING ####################################" + echo "# You are not using a pager." + echo "# The output of this script can be quite large." + echo "# Please pipe this script with a pager, such as 'less' or 'vim -'" + echo "# Choose one of the following:" + echo "# * simply RETURN to continue without a pager" + echo "# * 'q' to exit " + echo "# * enter the name of the pager to use" + read answer + case $answer in + q) + exit + ;; + *) + unset pager + [ -n "$answer" ] && pager=$answer + ;; + esac +fi + +case $log in + err) + target=$SBDIR/data/msandbox.err + ;; + gen) + target=$($SBDIR/use -BN -e "show variables like 'general_log_file'" | awk '{print $2}') + ;; + slow) + target=$SBDIR/data/slow_log.data + ;; + *) + target=$($SBDIR/use -BN -e "show variables like '$log'" | awk '{print $2}') + ;; +esac + +if [ -z "$target" ] +then + echo "target not set" + exit 1 +fi +if [ ! -f $target ] +then + echo "Log file '$target' not found" + exit 1 +fi + +if [ -n "$pager" ] +then + (printf "#\n# Showing $target\n#\n" ; cat $target ) | $pager +else + (printf "#\n# Showing $target\n#\n" ; cat $target ) +fi ` show_binlog_template string = `#!/bin/bash {{.Copyright}} @@ -617,8 +723,9 @@ fi pattern=$1 [ -z "$pattern" ] && pattern='[0-9]*' -if [ "$pattern" == "-h" -o "$pattern" == "--help" -o "$pattern" == "-help" -o "$pattern" == "help" ] -then +function get_help { + exit_code=$1 + [ -z "$exit_code" ] && exit_code=0 echo "# Usage: $0 [BINLOG_PATTERN] " echo "# Where BINLOG_PATTERN is a number, or part of a number used after 'mysql-bin'" echo "# (The default is '[0-9]*]')" @@ -626,7 +733,12 @@ then echo "# ./show_binlog 000001 | less " echo "# ./show_binlog 000012 | vim - " echo "# ./show_binlog | grep -i 'CREATE TABLE'" - exit 0 + exit $exit_code +} + +if [ "$pattern" == "-h" -o "$pattern" == "--help" -o "$pattern" == "-help" -o "$pattern" == "help" ] +then + get_help 0 fi # set -x last_binlog=$(ls -lotr data/mysql-bin.$pattern | tail -n 1 | awk '{print $NF}') @@ -634,7 +746,7 @@ last_binlog=$(ls -lotr data/mysql-bin.$pattern | tail -n 1 | awk '{print $NF}') if [ -z "$last_binlog" ] then echo "No binlog found in $curdir/data" - exit 1 + get_help 1 fi # Checks if the output is a terminal or a pipe @@ -644,15 +756,28 @@ then echo "# You are not using a pager." echo "# The output of this script can be quite large." echo "# Please pipe this script with a pager, such as 'less' or 'vim -'" - echo "# ENTER 'q' to exit or simply RETURN to continue without a pager" + echo "# Choose one of the following:" + echo "# * simply RETURN to continue without a pager" + echo "# * 'q' to exit " + echo "# * enter the name of the pager to use" read answer - if [ "$answer" == "q" ] - then + case $answer in + q) exit - fi + ;; + *) + unset pager + [ -n "$answer" ] && pager=$answer + ;; + esac fi -(printf "#\n# Showing $last_binlog\n#\n" ; ./my sqlbinlog --verbose $last_binlog ) +if [ -n "$pager" ] +then + (printf "#\n# Showing $last_binlog\n#\n" ; ./my sqlbinlog --verbose $last_binlog ) | $pager +else + (printf "#\n# Showing $last_binlog\n#\n" ; ./my sqlbinlog --verbose $last_binlog ) +fi ` my_template string = `#!/bin/bash {{.Copyright}} @@ -731,8 +856,9 @@ relay_basename=$1 [ -z "$relay_basename" ] && relay_basename='mysql-relay' pattern=$2 [ -z "$pattern" ] && pattern='[0-9]*' -if [ "$pattern" == "-h" -o "$pattern" == "--help" -o "$pattern" == "-help" -o "$pattern" == "help" ] -then +function get_help { + exit_code=$1 + [ -z "$exit_code" ] && exit_code=0 echo "# Usage: $0 [ relay-base-name [BINLOG_PATTERN]] " echo "# Where relay-basename is the initial part of the relay ('$relay_basename')" echo "# and BINLOG_PATTERN is a number, or part of a number used after '$relay_basename'" @@ -741,7 +867,12 @@ then echo "# ./show_relaylog relay-log-alpha 000001 | less " echo "# ./show_relaylog relay-log 000012 | vim - " echo "# ./show_relaylog | grep -i 'CREATE TABLE'" - exit 0 + exit $exit_code +} + +if [ "$pattern" == "-h" -o "$pattern" == "--help" -o "$pattern" == "-help" -o "$pattern" == "help" ] +then + get_help 0 fi # set -x last_relaylog=$(ls -lotr data/$relay_basename.$pattern | tail -n 1 | awk '{print $NF}') @@ -749,7 +880,7 @@ last_relaylog=$(ls -lotr data/$relay_basename.$pattern | tail -n 1 | awk '{print if [ -z "$last_relaylog" ] then echo "No relay log found in $curdir/data" - exit 1 + get_help 1 fi # Checks if the output is a terminal or a pipe @@ -760,14 +891,28 @@ then echo "# The output of this script can be quite large." echo "# Please pipe this script with a pager, such as 'less' or 'vim -'" echo "# ENTER 'q' to exit or simply RETURN to continue without a pager" + echo "# Choose one of the following:" + echo "# * simply RETURN to continue without a pager" + echo "# * 'q' to exit " + echo "# * enter the name of the pager to use" read answer - if [ "$answer" == "q" ] - then + case $answer in + q) exit - fi + ;; + *) + unset pager + [ -n "$answer" ] && pager=$answer + ;; + esac fi -(printf "#\n# Showing $last_relaylog\n#\n" ; ./my sqlbinlog --verbose $last_relaylog ) +if [ -n "$pager" ] +then + (printf "#\n# Showing $last_relaylog\n#\n" ; ./my sqlbinlog --verbose $last_relaylog ) | $pager +else + (printf "#\n# Showing $last_relaylog\n#\n" ; ./my sqlbinlog --verbose $last_relaylog ) +fi ` test_sb_template string = `#!/bin/bash {{.Copyright}} @@ -819,6 +964,19 @@ function test_query { fi } +if [ -n "$CHECK_LOGS" ] +then + log_has_errors=$(grep ERROR $SBDIR/data/msandbox.err) + if [ -z "$log_has_errors" ] + then + echo "ok - no errors in log" + pass=$((pass+1)) + else + echo "not ok - errors found in log" + fail=$((fail+1)) + fi +fi + if [ -z "$(is_running)" ] then echo "not ok - server stopped" @@ -1030,6 +1188,11 @@ exit 0 Notes: "", Contents: use_template, }, + "mysqlsh_template": TemplateDesc{ + Description: "Invokes the MySQL shell with an appropriate URI", + Notes: "", + Contents: mysqlsh_template, + }, "stop_template": TemplateDesc{ Description: "Stops a database in a single sandbox", Notes: "", @@ -1090,6 +1253,11 @@ exit 0 Notes: "", Contents: add_option_template, }, + "show_log_template": TemplateDesc{ + Description: "Shows error log or custom log", + Notes: "", + Contents: show_log_template, + }, "show_binlog_template": TemplateDesc{ Description: "Shows a binlog for a single sandbox", Notes: "", diff --git a/test/all_tests.sh b/test/all_tests.sh index 576b8361..2c00c1ab 100755 --- a/test/all_tests.sh +++ b/test/all_tests.sh @@ -1,5 +1,8 @@ #!/bin/bash +export RUN_CONCURRENTLY=1 +[ -n "$SEQUENTIAL" ] && unset RUN_CONCURRENTLY + if [ ! -d "./test" ] then echo "directory ./test not found" diff --git a/test/common.sh b/test/common.sh index 1520e7c1..4b926977 100644 --- a/test/common.sh +++ b/test/common.sh @@ -429,6 +429,27 @@ function ok_generic_exists { tests=$((tests+1)) } +function ok_generic_does_not_exist { + wanted=$1 + label=$2 + op=$3 + if [ ! $op "$wanted" ] + then + echo "ok - $label $wanted does not exist" + pass=$((pass+1)) + else + echo "NOT OK - $label $wanted exists" + fail=$((fail+1)) + if [ -n "$EXIT_ON_FAILURE" ] + then + echo "pass: $pass - fail: $fail" + exit + fi + fi + tests=$((tests+1)) +} + + function ok_dir_exists { dir=$1 ok_generic_exists $dir directory -d @@ -444,6 +465,10 @@ function ok_executable_exists { ok_generic_exists $filename "file" -x } +function ok_executable_does_not_exist { + filename=$1 + ok_generic_does_not_exist $filename "file" "-x" +} function run { diff --git a/test/functional-test.sh b/test/functional-test.sh index b152a905..17126371 100755 --- a/test/functional-test.sh +++ b/test/functional-test.sh @@ -702,13 +702,13 @@ function main_deployment_methods { test_ports $V msb_ 1 1 test_ports $V rsandbox_ 3 3 test_ports $V multi_msb_ 3 3 - test_start_restart $V msb_ single - test_start_restart $V rsandbox_ multiple - test_start_restart $V multi_msb_ multiple test_slave_hosts $V rsandbox_ 2 test_use_masters_slaves $V rsandbox_ 1 2 capture_test run dbdeployer global test capture_test run dbdeployer global test-replication + test_start_restart $V msb_ single + test_start_restart $V rsandbox_ multiple + test_start_restart $V multi_msb_ multiple echo "# processes $processes_before" test_deletion $V 3 $processes_before done diff --git a/test/mock/defaults-change.sh b/test/mock/defaults-change.sh index 0cf68f40..68d738d6 100755 --- a/test/mock/defaults-change.sh +++ b/test/mock/defaults-change.sh @@ -84,6 +84,8 @@ function check_deployment_message { rm $output_file } +create_mock_version 5.0.66 +create_mock_version 5.1.66 create_mock_version 5.5.66 create_mock_version 5.6.66 create_mock_version 5.7.66 @@ -178,9 +180,56 @@ check_deployment_message 8.0.67 replication --topology=group --single-primary check_deployment_message 8.0.67 replication --topology=fan-in check_deployment_message 8.0.67 replication --topology=all-masters - run dbdeployer delete ALL --skip-confirm +dbdeployer defaults update reserved-ports '4500,9000,15000,30000' +reserved_ports=$(dbdeployer defaults show| sed -n '/reserved-ports/,/]/p'| grep '^\s*[0-9]'| tr -d ', \t') +how_many=$(echo $reserved_ports | wc -w) +ok_equal "4 reserved ports found" $how_many 4 + +for port in $reserved_ports +do + dbdeployer deploy single 5.7.66 --port=$port + port_plus_1=$((port+1)) + sandbox_list=$(dbdeployer sandboxes) + ok_contains "Deployment for port $port contains $port_plus_1" "$sandbox_list" $port_plus_1 + dbdeployer delete msb_5_7_66 +done + +function mysqlsh_exists { + version=$1 + wanted_ex=$2 + option=$3 + path_version=$(echo $version | tr '.' '_') + client=$SANDBOX_HOME/msb_${path_version}/mysqlsh + run dbdeployer deploy single $version $option + if [ "$wanted_ex" == "exists" ] + then + ls -l $client + ok_executable_exists $client + else + echo "START TESTING FOR ABSENCE" + ok_executable_does_not_exist $client + echo "END TESTING FOR ABSENCE" + fi + + for log in log binlog relaylog + do + viewer=$SANDBOX_HOME/msb_${path_version}/show_$log + ok_executable_exists $viewer + done + run dbdeployer delete msb_${path_version} + unset $option +} + +mysqlsh_exists 5.0.66 does_not "" +mysqlsh_exists 5.1.66 does_not "" +mysqlsh_exists 5.5.66 does_not "" +mysqlsh_exists 5.6.66 does_not "" +mysqlsh_exists 5.7.66 does_not "" +mysqlsh_exists 5.7.66 exists "--enable-mysqlx" +mysqlsh_exists 8.0.66 exists "" + cd $test_dir run du -sh $mock_dir