Commit e4a7ed23 authored by Cédric Anne's avatar Cédric Anne
Browse files

Use utf8mb4 MySQL character set

 - Use utf8mb4 as default charset for fresh installation
 - utf8mb4 migration command
 - warn for collation mix on migration
 - Force ROW_FORMAT = Dynamic on fresh install
 - Add warning on central page if some tables are not migrated
parent f5318ec7
......@@ -224,12 +224,13 @@ jobs:
docker exec app bin/console glpi:migration:myisam_to_innodb --config-dir=./tests --no-interaction | grep -q "Aucune migration requise." || (echo "glpi:migration:myisam_to_innodb command FAILED" && exit 1)
docker exec app bin/console glpi:migration:timestamps --config-dir=./tests --no-interaction
docker exec app bin/console glpi:migration:timestamps --config-dir=./tests --no-interaction | grep -q "Aucune migration requise." || (echo "glpi:migration:timestamps command FAILED" && exit 1)
docker exec app bin/console glpi:migration:utf8mb4 --config-dir=./tests --no-interaction
docker exec app bin/console glpi:migration:utf8mb4 --config-dir=./tests --no-interaction | grep -q "Aucune migration requise." || (echo "glpi:migration:utf8mb4 command FAILED" && exit 1)
- name: "Database tests"
if: env.skip != 'true'
run: |
docker exec app bin/console glpi:database:configure --config-dir=./tests --no-interaction --reconfigure --db-name=glpi --db-host=db --db-user=root
docker exec app bin/console glpi:database:configure --config-dir=./tests --no-interaction --reconfigure --db-name=glpi --db-host=db --db-user=root --use-utf8mb4
docker exec app vendor/bin/atoum -p 'php -d memory_limit=512M' --debug --force-terminal --use-dot-report --bootstrap-file tests/bootstrap.php --no-code-coverage --fail-if-skipped-methods --max-children-number 1 -d tests/database
docker exec app bin/console glpi:database:configure --config-dir=./tests --no-interaction --reconfigure --db-name=glpi --db-host=db --db-user=root
- name: "Unit tests"
if: env.skip != 'true'
run: |
......
......@@ -13,6 +13,9 @@ The present file will list all changes made to the project; according to the
### Deprecated
- Usage of XML-RPC API is deprecated.
### Removed
- Usage of alternative DB connection encoding (`DB::$dbenc` property).
### API changes
#### Added
......
This diff is collapsed.
......@@ -329,12 +329,16 @@ class Central extends CommonGLPI {
if ($DB->areTimezonesAvailable()) {
$not_tstamp = $DB->notTzMigrated();
if ($not_tstamp > 0) {
$warnings[] = sprintf(
__('%1$s columns are not compatible with timezones usage.'),
$not_tstamp
);
$warnings[] = sprintf(__('%1$s columns are not compatible with timezones usage.'), $not_tstamp)
. ' '
. sprintf(__('Run the "php bin/console %1$s" command to migrate them.'), 'glpi:migration:timestamps');
}
}
if (($non_utf8mb4_tables = $DB->getNonUtf8mb4Tables()->count()) > 0) {
$warnings[] = sprintf(__('%1$s tables not migrated to utf8mb4 collation.'), $non_utf8mb4_tables)
. ' '
. sprintf(__('Run the "php bin/console %1$s" command to migrate them.'), 'glpi:migration:utf8mb4');
}
}
if ($DB->isSlave()
......
......@@ -44,6 +44,7 @@ use Symfony\Component\Console\Exception\RuntimeException;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\Console\Question\ConfirmationQuestion;
abstract class AbstractCommand extends Command implements GlpiCommandInterface {
......@@ -210,4 +211,26 @@ abstract class AbstractCommand extends Command implements GlpiCommandInterface {
return $this->requires_db && $this->requires_db_up_to_date;
}
/**
* Ask for user confirmation before continuing command execution.
*
* @return void
*/
protected function askForConfirmation(): void {
if (!$this->input->getOption('no-interaction')) {
$question_helper = $this->getHelper('question');
$run = $question_helper->ask(
$this->input,
$this->output,
new ConfirmationQuestion(__('Do you want to continue ?') . ' [Yes/no]', true)
);
if (!$run) {
throw new \Glpi\Console\Exception\EarlyExitException(
'<comment>' . __('Aborted.') . '</comment>',
0 // Success code
);
}
}
}
}
......@@ -244,7 +244,12 @@ class Application extends BaseApplication {
return self::ERROR_MISSING_REQUIREMENTS;
}
$result = parent::doRunCommand($command, $input, $output);
try {
$result = parent::doRunCommand($command, $input, $output);
} catch (\Glpi\Console\Exception\EarlyExitException $e) {
$result = $e->getCode();
$output->writeln($e->getMessage(), OutputInterface::VERBOSITY_QUIET);
}
if ($output->getVerbosity() >= OutputInterface::VERBOSITY_VERY_VERBOSE) {
$output->writeln(
......
......@@ -176,10 +176,13 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
*
* @param InputInterface $input
* @param OutputInterface $output
* @param bool $use_utf8mb4
*
* @throws InvalidArgumentException
*
* @return string
*/
protected function configureDatabase(InputInterface $input, OutputInterface $output) {
protected function configureDatabase(InputInterface $input, OutputInterface $output, bool $use_utf8mb4) {
$db_pass = $input->getOption('db-password');
$db_host = $input->getOption('db-host');
......@@ -234,6 +237,8 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
return self::ERROR_DB_CONNECTION_FAILED;
}
DBConnection::setConnectionCharset($mysqli, $use_utf8mb4);
ob_start();
$db_version_data = $mysqli->query('SELECT version()')->fetch_array();
$checkdb = Config::displayCheckDbEngine(false, $db_version_data[0]);
......@@ -249,7 +254,7 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
'<comment>' . __('Saving configuration file...') . '</comment>',
OutputInterface::VERBOSITY_VERBOSE
);
if (!DBConnection::createMainConfig($db_hostport, $db_user, $db_pass, $db_name)) {
if (!DBConnection::createMainConfig($db_hostport, $db_user, $db_pass, $db_name, $use_utf8mb4)) {
$message = sprintf(
__('Cannot write configuration file "%s".'),
GLPI_CONFIG_DIR . DIRECTORY_SEPARATOR . 'config_db.php'
......
......@@ -37,6 +37,7 @@ if (!defined('GLPI_ROOT')) {
}
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ConfigureCommand extends AbstractConfigureCommand {
......@@ -48,11 +49,18 @@ class ConfigureCommand extends AbstractConfigureCommand {
$this->setName('glpi:database:configure');
$this->setAliases(['db:configure']);
$this->setDescription('Define database configuration');
$this->addOption(
'use-utf8mb4',
null,
InputOption::VALUE_NONE,
__('Use utf8mb4 character set.')
);
}
protected function execute(InputInterface $input, OutputInterface $output) {
$result = $this->configureDatabase($input, $output);
$result = $this->configureDatabase($input, $output, $input->getOption('use-utf8mb4'));
if (self::ABORTED_BY_USER === $result) {
return 0; // Considered as success
......
......@@ -37,6 +37,9 @@ if (!defined('GLPI_ROOT')) {
}
use DB;
use DBConnection;
use DBmysql;
use Glpi\System\Requirement\DbConfiguration;
use GLPIKey;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
......@@ -74,6 +77,13 @@ class InstallCommand extends AbstractConfigureCommand {
*/
const ERROR_CANNOT_CREATE_ENCRYPTION_KEY_FILE = 8;
/**
* Error code returned if DB configuration is not compatible with large indexes.
*
* @var integer
*/
const ERROR_INCOMPATIBLE_DB_CONFIG = 9;
protected function configure() {
parent::configure();
......@@ -149,7 +159,7 @@ class InstallCommand extends AbstractConfigureCommand {
}
if (!$this->isDbAlreadyConfigured() || $input->getOption('reconfigure')) {
$result = $this->configureDatabase($input, $output);
$result = $this->configureDatabase($input, $output, true);
if (self::ABORTED_BY_USER === $result) {
return 0; // Considered as success
......@@ -226,6 +236,23 @@ class InstallCommand extends AbstractConfigureCommand {
return self::ERROR_DB_CONNECTION_FAILED;
}
// Check for compatibility with utf8mb4 usage.
$db = new class($mysqli) extends DBmysql {
public function __construct($dbh) {
$this->dbh = $dbh;
}
};
$config_requirement = new DbConfiguration($db);
if (!$config_requirement->isValidated()) {
$msg = '<error>' . __('Database configuration is not compatible with "utf8mb4" usage.') . '</error>';
foreach ($config_requirement->getValidationMessages() as $validation_message) {
$msg .= "\n" . '<error> - ' . $validation_message . '</error>';
}
throw new \Glpi\Console\Exception\EarlyExitException($msg, self::ERROR_INCOMPATIBLE_DB_CONFIG);
}
DBConnection::setConnectionCharset($mysqli, true);
// Create database or select existing one
$output->writeln(
'<comment>' . __('Creating the database...') . '</comment>',
......@@ -260,15 +287,16 @@ class InstallCommand extends AbstractConfigureCommand {
return self::ERROR_DB_ALREADY_CONTAINS_TABLES;
}
if ($DB instanceof DB) {
if ($DB instanceof DBmysql) {
// If global $DB is set at this point, it means that configuration file has been loaded
// prior to reconfiguration.
// As configuration is part of a class, it cannot be reloaded and class properties
// have to be updated manually in order to make `Toolbox::createSchema()` work correctly.
$DB->dbhost = $db_hostport;
$DB->dbuser = $db_user;
$DB->dbpassword = rawurlencode($db_pass);
$DB->dbdefault = $db_name;
$DB->dbhost = $db_hostport;
$DB->dbuser = $db_user;
$DB->dbpassword = rawurlencode($db_pass);
$DB->dbdefault = $db_name;
$DB->use_utf8mb4 = true;
$DB->clearSchemaCache();
$DB->connect();
......
<?php
/**
* ---------------------------------------------------------------------
* GLPI - Gestionnaire Libre de Parc Informatique
* Copyright (C) 2015-2021 Teclib' and contributors.
*
* http://glpi-project.org
*
* based on GLPI - Gestionnaire Libre de Parc Informatique
* Copyright (C) 2003-2014 by the INDEPNET Development Team.
*
* ---------------------------------------------------------------------
*
* LICENSE
*
* This file is part of GLPI.
*
* GLPI is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* GLPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with GLPI. If not, see <http://www.gnu.org/licenses/>.
* ---------------------------------------------------------------------
*/
namespace Glpi\Console\Exception;
if (!defined('GLPI_ROOT')) {
die("Sorry. You can't access this file directly");
}
use Symfony\Component\Console\Exception\ExceptionInterface;
/**
* This exception is used to easilly trigger an exit of current command from a sub method.
*
* @since x.x.x
*/
class EarlyExitException extends \Exception implements ExceptionInterface {
}
<?php
/**
* ---------------------------------------------------------------------
* GLPI - Gestionnaire Libre de Parc Informatique
* Copyright (C) 2015-2021 Teclib' and contributors.
*
* http://glpi-project.org
*
* based on GLPI - Gestionnaire Libre de Parc Informatique
* Copyright (C) 2003-2014 by the INDEPNET Development Team.
*
* ---------------------------------------------------------------------
*
* LICENSE
*
* This file is part of GLPI.
*
* GLPI is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation; either version 2 of the License, or
* (at your option) any later version.
*
* GLPI is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with GLPI. If not, see <http://www.gnu.org/licenses/>.
* ---------------------------------------------------------------------
*/
namespace Glpi\Console\Migration;
if (!defined('GLPI_ROOT')) {
die("Sorry. You can't access this file directly");
}
use DBConnection;
use Glpi\Console\AbstractCommand;
use Glpi\System\Requirement\DbConfiguration;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Output\OutputInterface;
class Utf8mb4Command extends AbstractCommand {
/**
* Error code returned if migration failed on, at least, one table.
*
* @var integer
*/
const ERROR_MIGRATION_FAILED_FOR_SOME_TABLES = 1;
/**
* Error code returned if DB configuration file cannot be updated.
*
* @var integer
*/
const ERROR_UNABLE_TO_UPDATE_CONFIG = 2;
/**
* Error code returned if some tables are still using MyISAM engine.
*
* @var integer
*/
const ERROR_INNODB_REQUIRED = 3;
/**
* Error code returned if DB configuration is not compatible with large indexes.
*
* @var integer
*/
const ERROR_INCOMPATIBLE_DB_CONFIG = 3;
protected function configure() {
parent::configure();
$this->setName('glpi:migration:utf8mb4');
$this->setDescription(__('Convert database character set from "utf8" to "utf8mb4".'));
}
protected function execute(InputInterface $input, OutputInterface $output) {
$this->checkForPrerequisites();
$this->upgradeRowFormat();
$this->migrateToUtf8mb4();
return 0; // Success
}
/**
* Check for migration prerequisites.
*
* @return void
*/
private function checkForPrerequisites(): void {
// Check that DB configuration is compatible
$config_requirement = new DbConfiguration($this->db);
if (!$config_requirement->isValidated()) {
$msg = '<error>' . __('Database configuration is not compatible with "utf8mb4" usage.') . '</error>';
foreach ($config_requirement->getValidationMessages() as $validation_message) {
$msg .= "\n" . '<error> - ' . $validation_message . '</error>';
}
throw new \Glpi\Console\Exception\EarlyExitException($msg, self::ERROR_INCOMPATIBLE_DB_CONFIG);
}
// Check that all tables are using InnoDB engine
if (($myisam_count = $this->db->getMyIsamTables()->count()) > 0) {
$msg = sprintf(
__('%d tables are still using MyISAM storage engine. Run "php bin/console glpi:migration:myisam_to_innodb" to fix this.'),
$myisam_count
);
throw new \Glpi\Console\Exception\EarlyExitException('<error>' . $msg . '</error>', self::ERROR_INNODB_REQUIRED);
}
}
/**
* Upgrade row format from 'Compact'/'Redundant' to 'Dynamic'.
* This is mandatory to support large indexes.
*
* @return void
*/
private function upgradeRowFormat(): void {
$table_iterator = $this->db->listTables(
'glpi_%',
[
'row_format' => ['Compact', 'Redundant'],
]
);
if ($table_iterator->count() === 0) {
return;
}
$this->output->writeln(
sprintf(
'<info>' . __('Found %s table(s) requiring a migration to "ROW_FORMAT=Dynamic".') . '</info>',
$table_iterator->count()
)
);
$this->askForConfirmation();
$tables = [];
foreach ($table_iterator as $table_data) {
$tables[] = $table_data['TABLE_NAME'];
}
sort($tables);
$progress_bar = new ProgressBar($this->output);
$errors = 0;
foreach ($progress_bar->iterate($tables) as $table) {
$this->writelnOutputWithProgressBar(
sprintf(__('Migrating table "%s"...'), $table),
$progress_bar,
OutputInterface::VERBOSITY_VERY_VERBOSE
);
$result = $this->db->query(
sprintf('ALTER TABLE `%s` ROW_FORMAT = DYNAMIC', $table)
);
if (!$result) {
$this->writelnOutputWithProgressBar(
sprintf(__('<error>Error migrating table "%s".</error>'), $table),
$progress_bar,
OutputInterface::VERBOSITY_QUIET
);
$errors++;
}
}
$this->output->write(PHP_EOL);
if ($errors) {
throw new \Glpi\Console\Exception\EarlyExitException(
'<error>' . __('Errors occured during migration.') . '</error>',
self::ERROR_MIGRATION_FAILED_FOR_SOME_TABLES
);
}
}
/**
* Migrate tables to utf8mb4.
*
* @return void
*/
private function migrateToUtf8mb4(): void {
$tables = [];
// Find collations to update at table level
$table_iterator = $this->db->getNonUtf8mb4Tables();
foreach ($table_iterator as $table_data) {
$tables[] = $table_data['TABLE_NAME'];
}
$errors = 0;
if (count($tables) === 0) {
$this->output->writeln('<info>' . __('No migration needed.') . '</info>');
} else {
sort($tables);
$this->output->writeln(
sprintf(
'<info>' . __('Found %s table(s) requiring migration to "utf8mb4".') . '</info>',
count($tables)
)
);
$this->askForConfirmation();
// Early update property to prevent warnings related to bad collation detection.
$this->db->use_utf8mb4 = true;
$progress_bar = new ProgressBar($this->output);
$errors = 0;
foreach ($progress_bar->iterate($tables) as $table) {
$this->writelnOutputWithProgressBar(
sprintf(__('Migrating table "%s"...'), $table),
$progress_bar,
OutputInterface::VERBOSITY_VERY_VERBOSE
);
$result = $this->db->query(
sprintf('ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci', $table)
);
if (!$result) {
$this->writelnOutputWithProgressBar(
sprintf(__('<error>Error migrating table "%s".</error>'), $table),
$progress_bar,
OutputInterface::VERBOSITY_QUIET
);
$errors++;
}
}
$this->output->write(PHP_EOL);
}
if (!DBConnection::updateConfigProperty('use_utf8mb4', true)) {
throw new \Glpi\Console\Exception\EarlyExitException(
'<error>' . __('Unable to update DB configuration file.') . '</error>',
self::ERROR_UNABLE_TO_UPDATE_CONFIG
);
}
if ($errors) {
throw new \Glpi\Console\Exception\EarlyExitException(
'<error>' . __('Errors occured during migration.') . '</error>',
self::ERROR_MIGRATION_FAILED_FOR_SOME_TABLES
);
}
if (count($tables) > 0) {
$this->output->writeln('<info>' . __('Migration done.') . '</info>');
}
}
}
......@@ -61,19 +61,78 @@ class DBConnection extends CommonDBTM {
* @return boolean
*
**/
static function createMainConfig($host, $user, $password, $DBname) {
static function createMainConfig($host, $user, $password, $DBname, $use_utf8mb4 = false) {
$DB_str = "<?php\nclass DB extends DBmysql {\n" .
" public \$dbhost = '$host';\n" .
" public \$dbuser = '$user';\n" .
" public \$dbpassword = '". rawurlencode($password) . "';\n" .
" public \$dbdefault = '$DBname';\n" .
" public \$dbhost = '$host';\n" .
" public \$dbuser = '$user';\n" .
" public \$dbpassword = '". rawurlencode($password) . "';\n" .
" public \$dbdefault = '$DBname';\n" .
" public \$use_utf8mb4 = " . ($use_utf8mb4 ? 'true' : 'false') . ";\n" .
"}\n";
return Toolbox::writeConfig('config_db.php', $DB_str);
}
/**
* Change a variable value in config(s) file.
*
* @param string $name
* @param string $value
* @param bool $update_slave
* @param string $config_dir
*
* @return boolean
*
* @since x.x.x
*/
static function updateConfigProperty($name, $value, $update_slave = true, string $config_dir = GLPI_CONFIG_DIR): bool {
$main_config_file = 'config_db.php';
$slave_config_file = 'config_db_slave.php';
if (!file_exists($config_dir . '/' . $main_config_file)) {
return false;
}
if ($name === 'password') {
$value = rawurlencode($value);
}
$pattern = '/(?<line>' . preg_quote('$' . $name, '/') . '\s*=\s*(?<value>[^;]+)\s*;)' . '/';
$files = [$main_config_file];
if ($update_slave && file_exists($config_dir . '/' . $slave_config_file)) {
$files[] = $slave_config_file;
}