Skip to content

Commit

Permalink
#4501: Minimum viable utf8mb3 => utf8mb4 and MyISAM => InnoDB conversion
Browse files Browse the repository at this point in the history 
Convert and run e107 using the MySQL/MariaDB utf8mb4 character set and
InnoDB storage engine

Components affected:
* `db_verify` now checks and corrects the table storage engine
* `db_verify` now checks and corrects the table default character set
  * Note: Field character sets can still be overridden
  * Note: When correcting, the entire table is converted to the target
    charset.
* The alt_auth plugin now connects via PDO using the e107 default
  charset, utf8mb4
* `e_db_pdo` now sets the charset to utf8mb4. This is currently not
  customizable because it was previously not customizable.
* `install.php` now generates an `e107_config.php` file with
  `$mySQLcharset = 'utf8mb4';`, though this option is not actually used.
* `install.php` now removes plugin tables before installing plugins.
* `e_db_mysql` now only accepts the `utf8mb4` charset. Previously, it
  only accepted the `utf8` charset.
* `e_db_mysql` now configures `mysqli_real_escape_string` to match the
  new default charset, `utf8mb4`.
* Plugin installations now use the preferred MySQL table storage engines
  and charsets.

The preferred MySQL table storage engines are now mapped like so:
* If `ENGINE=MyISAM` is specified, the actual storage engine set will be
  the first available of: InnoDB, Aria, Maria, MyISAM
* If `ENGINE=Aria` is specified, the actual storage engine set will be
  the first available of: Aria, Maria, MyISAM
* If `ENGINE=InnoDB` is specified, the actual storage engine set will be
  the first available of: InnoDB, XtraDB
* If `ENGINE=XtraDB` is specified, the actual storage engine set will be
  the first available of: XtraDB, InnoDB

The preferred MySQL character set is now aliased like so:
* `utf8`    => `utf8mb4`
* `utf8mb3` => `utf8mb3`
* `utf8mb4` => `utf8mb4`

Fixes: #4501
  • Loading branch information
@Deltik
Deltik committed Sep 11, 2021
1 parent 036b301 commit 4f99536
Show file tree
Hide file tree
Showing 13 changed files with 469 additions and 134 deletions.
25 changes: 12 additions & 13 deletions e107_admin/db.php
View file
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ private function multiSiteProcess()
if(vartrue($_POST['createdb']))
{

if($sql->gen("CREATE DATABASE ".$database." CHARACTER SET `utf8`"))
if($sql->gen("CREATE DATABASE ".$database." CHARACTER SET `utf8mb4`"))
{
$mes->addSuccess(DBLAN_75);

Expand Down Expand Up @@ -533,7 +533,7 @@ private function multiSiteCreateTables($sql, $prefix)
preg_match_all("/create(.*?)(?:myisam|innodb);/si", $sql_data, $result );


$sql->gen('SET NAMES `utf8`');
$sql->gen('SET NAMES `utf8mb4`');

foreach ($result[0] as $sql_table)
{
Expand Down Expand Up @@ -746,11 +746,11 @@ private function convertUTF8Form()
<td>".$row['Name']."</td>
<td>".$row['Engine']."</td>
<td>".$row['Collation']."</td>
<td>".(($row['Collation'] == 'utf8_general_ci') ? ADMIN_TRUE_ICON : ADMIN_FALSE_ICON)."</td>
<td>".(($row['Collation'] == 'utf8mb4_general_ci') ? ADMIN_TRUE_ICON : ADMIN_FALSE_ICON)."</td>
</tr>";
// print_a($row);

if($row['Collation'] != 'utf8_general_ci')
if($row['Collation'] != 'utf8mb4_general_ci')
{
$invalidCollations = true;
}
Expand Down Expand Up @@ -819,12 +819,12 @@ private function perform_utf8_convert()


$queries = array();
$queries[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', REPLACE(column_type, 'char', 'binary'), ';') FROM information_schema.columns WHERE TABLE_SCHEMA = '".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8_general_ci' and data_type LIKE '%char%';");
$queries[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', REPLACE(column_type, 'text', 'blob'), ';') FROM information_schema.columns WHERE TABLE_SCHEMA = '".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8_general_ci' and data_type LIKE '%text%';");
$queries[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', REPLACE(column_type, 'char', 'binary'), ';') FROM information_schema.columns WHERE TABLE_SCHEMA = '".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8mb4_general_ci' and data_type LIKE '%char%';");
$queries[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', REPLACE(column_type, 'text', 'blob'), ';') FROM information_schema.columns WHERE TABLE_SCHEMA = '".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8mb4_general_ci' and data_type LIKE '%text%';");

$queries2 = array();
$queries2[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', column_type, ' CHARACTER SET utf8;') FROM information_schema.columns WHERE TABLE_SCHEMA ='".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8_general_ci' and data_type LIKE '%char%';");
$queries2[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', column_type, ' CHARACTER SET utf8;') FROM information_schema.columns WHERE TABLE_SCHEMA = '".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8_general_ci' and data_type LIKE '%text%';");
$queries2[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', column_type, ' CHARACTER SET utf8mb4;') FROM information_schema.columns WHERE TABLE_SCHEMA ='".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8mb4_general_ci' and data_type LIKE '%char%';");
$queries2[] = $this->getQueries("SELECT CONCAT('ALTER TABLE `', table_name, '` MODIFY ', column_name, ' ', column_type, ' CHARACTER SET utf8mb4;') FROM information_schema.columns WHERE TABLE_SCHEMA = '".$dbtable."' AND TABLE_NAME LIKE '".$config['mySQLprefix']."%' AND COLLATION_NAME != 'utf8mb4_general_ci' and data_type LIKE '%text%';");


// $sql->gen("USE ".$dbtable);
Expand Down Expand Up @@ -858,7 +858,7 @@ private function perform_utf8_convert()
// Convert Table Fields to utf8
$sql2 = e107::getDb('sql2');

$sql->gen('SHOW TABLE STATUS WHERE Collation != "utf8_general_ci" ');
$sql->gen('SHOW TABLE STATUS WHERE Collation != "utf8mb4_general_ci" ');
while ($row = $sql->fetch())
{
$table = $row['Name'];
Expand All @@ -869,7 +869,7 @@ private function perform_utf8_convert()
}


$tab_query = "ALTER TABLE ".$table." DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci; ";
$tab_query = "ALTER TABLE ".$table." DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci; ";

//echo "TABQRT= ".$tab_query;

Expand Down Expand Up @@ -904,7 +904,7 @@ private function perform_utf8_convert()

//------------

$lastQry = "ALTER DATABASE `".$dbtable."` DEFAULT CHARACTER SET utf8 COLLATE utf8_general_ci;";
$lastQry = "ALTER DATABASE `".$dbtable."` DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;";

if(!$sql->db_Query($lastQry))
{
Expand All @@ -913,11 +913,10 @@ private function perform_utf8_convert()
elseif($ERROR != TRUE)
{
$message = DBLAN_93;
//$message .= "<br />Please now add the following line to your e107_config.php file:<br /><b>\$mySQLcharset = 'utf8';</b>";

$mes->add($message, E_MESSAGE_SUCCESS);
$mes->addSuccess(DBLAN_94);
$mes->addSuccess('$mySQLcharset = "utf8";');
$mes->addSuccess('$mySQLcharset = "utf8mb4";');

}

Expand Down
63 changes: 31 additions & 32 deletions e107_core/sql/core_sql.php
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,13 @@
<?php
/*
/**
* e107 website system
*
* Copyright (C) e107 Inc (e107.org)
* Copyright (C) 2008-2021 e107 Inc (e107.org)
* Released under the terms and conditions of the
* GNU General Public License (http://www.gnu.org/licenses/gpl.txt)
*
* Core SQL
*
*/
*/

header("location:../index.php");
exit;
Expand All @@ -17,7 +16,7 @@
# +---------------------------------------------------------------+
# | e107 website system
# |
# | Copyright (C) 2008-2015 e107 Inc (e107.org)
# | Copyright (C) 2008-2021 e107 Inc (e107.org)
# | http://e107.org
# |
# | Released under the terms and conditions of the
Expand All @@ -41,7 +40,7 @@
dblog_remarks text NOT NULL,
PRIMARY KEY (dblog_id),
KEY dblog_datestamp (dblog_datestamp)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -59,7 +58,7 @@
dblog_remarks text NOT NULL,
PRIMARY KEY (dblog_id),
KEY dblog_datestamp (dblog_datestamp)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------


Expand All @@ -80,7 +79,7 @@
KEY banlist_ip (banlist_ip),
KEY banlist_datestamp (banlist_datestamp),
KEY banlist_banexpires (banlist_banexpires)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -105,7 +104,7 @@
PRIMARY KEY (comment_id),
KEY comment_blocked (comment_blocked),
KEY comment_author_id (comment_author_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -115,7 +114,7 @@
e107_name varchar(100) NOT NULL default '',
e107_value text NOT NULL,
PRIMARY KEY (e107_name)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -139,7 +138,7 @@
media_tags text NOT NULL,
PRIMARY KEY (media_id),
UNIQUE KEY media_url (media_url)
) ENGINE=MyISAM;
) ENGINE=InnoDB;

CREATE TABLE core_media_cat (
media_cat_id int(10) unsigned NOT NULL auto_increment,
Expand All @@ -153,7 +152,7 @@
media_cat_order int(3) unsigned NOT NULL default '0',
PRIMARY KEY (media_cat_id),
UNIQUE KEY media_cat_category (media_cat_category)
) ENGINE=MyISAM;
) ENGINE=InnoDB;


CREATE TABLE cron (
Expand Down Expand Up @@ -187,7 +186,7 @@
dblog_remarks text NOT NULL,
PRIMARY KEY (dblog_id),
KEY dblog_datestamp (dblog_datestamp)
) ENGINE=MyISAM;
) ENGINE=InnoDB;

# --------------------------------------------------------

Expand All @@ -204,7 +203,7 @@
gen_chardata text NOT NULL,
PRIMARY KEY (gen_id),
KEY gen_type (gen_type)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -226,7 +225,7 @@
link_sefurl varchar(255) NOT NULL,
link_owner varchar(50) NOT NULL default '',
PRIMARY KEY (link_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;

# --------------------------------------------------------

Expand All @@ -247,7 +246,7 @@
PRIMARY KEY (mail_target_id),
KEY mail_status (mail_status),
KEY mail_detail_id (mail_detail_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;

CREATE TABLE mail_content (
mail_source_id int(10) unsigned NOT NULL auto_increment,
Expand All @@ -273,7 +272,7 @@
mail_media text,
PRIMARY KEY (mail_source_id),
KEY mail_content_status (mail_content_status)
) ENGINE=MyISAM;
) ENGINE=InnoDB;


#
Expand All @@ -291,7 +290,7 @@
menu_layout varchar(100) NOT NULL default '',
menu_parms text NOT NULL,
PRIMARY KEY (menu_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand Down Expand Up @@ -327,7 +326,7 @@
KEY news_sticky (news_sticky),
KEY news_render_type (news_render_type),
KEY news_class (news_class)
) ENGINE=MyISAM;
) ENGINE=InnoDB;


# --------------------------------------------------------
Expand All @@ -348,7 +347,7 @@
category_template varchar(50) default NULL,
PRIMARY KEY (category_id),
KEY category_order (category_order)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand Down Expand Up @@ -406,7 +405,7 @@
menu_button_text varchar(250) NOT NULL default '',

PRIMARY KEY (page_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------


Expand All @@ -430,7 +429,7 @@
chapter_fields mediumtext,
PRIMARY KEY (chapter_id),
KEY chapter_order (chapter_order)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------


Expand All @@ -449,7 +448,7 @@
plugin_category varchar(100) NOT NULL default '',
PRIMARY KEY (plugin_id),
UNIQUE KEY plugin_path (plugin_path)
) ENGINE=MyISAM;
) ENGINE=InnoDB;

# --------------------------------------------------------
#
Expand All @@ -466,7 +465,7 @@
rate_up int(10) unsigned NOT NULL default '0',
rate_down int(10) unsigned NOT NULL default '0',
PRIMARY KEY (rate_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -479,7 +478,7 @@
session_user int(10) unsigned default NULL,
session_data mediumtext NOT NULL,
PRIMARY KEY (session_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------


Expand All @@ -504,7 +503,7 @@
submitnews_summary text,
submitnews_media text,
PRIMARY KEY (submitnews_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -517,7 +516,7 @@
tmp_info text NOT NULL,
KEY tmp_ip (tmp_ip),
KEY tmp_time (tmp_time)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -542,7 +541,7 @@
upload_owner varchar(50) NOT NULL default '',
PRIMARY KEY (upload_id),
KEY upload_active (upload_active)
) ENGINE=MyISAM;
) ENGINE=InnoDB;

# --------------------------------------------------------

Expand Down Expand Up @@ -582,7 +581,7 @@
UNIQUE KEY user_name (user_name),
UNIQUE KEY user_loginname (user_loginname),
KEY join_ban_index (user_join,user_ban)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -600,7 +599,7 @@
userclass_icon varchar(250) NOT NULL default '',
userclass_perms text NOT NULL,
PRIMARY KEY (userclass_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

#
Expand All @@ -611,7 +610,7 @@
user_extended_id int(10) unsigned NOT NULL default '0',
user_hidden_fields text,
PRIMARY KEY (user_extended_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------


Expand All @@ -635,6 +634,6 @@
user_extended_struct_order int(10) unsigned NOT NULL default '0',
user_extended_struct_parent int(10) unsigned NOT NULL default '0',
PRIMARY KEY (user_extended_struct_id)
) ENGINE=MyISAM;
) ENGINE=InnoDB;
# --------------------------------------------------------

Loading

0 comments on commit 4f99536

Please sign in to comment.