Commit 2a2da6c6 authored by Cédric Anne's avatar Cédric Anne Committed by Johan Cwiklinski
Browse files

Automatically detect 'use_utf8mb4' configuration

parent 5a6b0707
......@@ -5,7 +5,7 @@ mkdir -p $(dirname "$LOG_FILE")
bin/console glpi:database:configure \
--config-dir=./tests/config --no-interaction --ansi \
--reconfigure --db-name=glpitest-9.5.3 --db-host=db --db-user=root --use-utf8mb4 \
--reconfigure --db-name=glpitest-9.5.3 --db-host=db --db-user=root \
--log-deprecation-warnings
# Force ROW_FORMAT=DYNAMIC to prevent tests MySQL 5.6 and MariaDB 10.1 databases
......@@ -59,7 +59,7 @@ bin/console glpi:database:check_schema_integrity --config-dir=./tests/config --a
# Check updated data
bin/console glpi:database:configure \
--config-dir=./tests/config --no-interaction --ansi \
--reconfigure --db-name=glpi --db-host=db --db-user=root --use-utf8mb4 \
--reconfigure --db-name=glpi --db-host=db --db-user=root \
--log-deprecation-warnings
mkdir -p ./tests/files/_cache
tests/bin/test-updated-data --host=db --user=root --fresh-db=glpi --updated-db=glpitest-9.5.3 --ansi --no-interaction
......@@ -79,6 +79,6 @@ bin/console glpi:database:check_schema_integrity --config-dir=./tests/config --a
# Check updated data
bin/console glpi:database:configure \
--config-dir=./tests/config --no-interaction --ansi \
--reconfigure --db-name=glpi --db-host=db --db-user=root --use-utf8mb4 \
--reconfigure --db-name=glpi --db-host=db --db-user=root \
--log-deprecation-warnings
tests/bin/test-updated-data --host=db --user=root --fresh-db=glpi --updated-db=glpitest080 --ansi --no-interaction
......@@ -56,6 +56,7 @@ The present file will list all changes made to the project; according to the
- `Transfer::transferDropdownNetpoint()` has been renamed to `Transfer::transferDropdownSocket()`.
#### Deprecated
- Usage of `utf8mb3` charset/collation in database in favor of `utf8mb4` charset/collation.
- Handling of encoded/escaped value in `autoName()`
- `Netpoint` has been deprecated and replaced by `Socket`
- `Html::clean()`
......
......@@ -38,8 +38,10 @@ if (!defined('GLPI_ROOT')) {
use Config;
use DBConnection;
use DBmysql;
use Glpi\Console\AbstractCommand;
use Glpi\Console\Command\ForceNoPluginsOptionCommandInterface;
use mysqli;
use Symfony\Component\Console\Exception\InvalidArgumentException;
use Symfony\Component\Console\Helper\Table;
use Symfony\Component\Console\Input\InputInterface;
......@@ -183,13 +185,19 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
*
* @param InputInterface $input
* @param OutputInterface $output
* @param bool $auto_config_flags
* @param bool $use_utf8mb4
*
* @throws InvalidArgumentException
*
* @return string
*/
protected function configureDatabase(InputInterface $input, OutputInterface $output, bool $use_utf8mb4) {
protected function configureDatabase(
InputInterface $input,
OutputInterface $output,
bool $auto_config_flags = true,
bool $use_utf8mb4 = false
) {
$db_pass = $input->getOption('db-password');
$db_host = $input->getOption('db-host');
......@@ -226,7 +234,7 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
return self::ABORTED_BY_USER;
}
$mysqli = new \mysqli();
$mysqli = new mysqli();
if (intval($db_port) > 0) {
// Network port
@$mysqli->connect($db_host, $db_user, $db_pass, null, $db_port);
......@@ -245,8 +253,6 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
return self::ERROR_DB_CONNECTION_FAILED;
}
DBConnection::setConnectionCharset($mysqli, $use_utf8mb4);
ob_start();
$db_version_data = $mysqli->query('SELECT version()')->fetch_array();
$checkdb = Config::displayCheckDbEngine(false, $db_version_data[0]);
......@@ -256,13 +262,30 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
return self::ERROR_DB_ENGINE_UNSUPPORTED;
}
if ($auto_config_flags) {
// Instanciate DB to be able to compute boolean properties flags.
$db = new class($db_hostport, $db_user, $db_pass, $db_name) extends DBmysql {
public function __construct($dbhost, $dbuser, $dbpassword, $dbdefault) {
$this->dbhost = $dbhost;
$this->dbuser = $dbuser;
$this->dbpassword = $dbpassword;
$this->dbdefault = $dbdefault;
parent::__construct();
}
};
$config_flags = $db->getComputedConfigBooleanFlags();
$use_utf8mb4 = $config_flags[DBConnection::PROPERTY_USE_UTF8MB4] ?? $use_utf8mb4;
}
DBConnection::setConnectionCharset($mysqli, $use_utf8mb4);
$db_name = $mysqli->real_escape_string($db_name);
$output->writeln(
'<comment>' . __('Saving configuration file...') . '</comment>',
OutputInterface::VERBOSITY_VERBOSE
);
if (!DBConnection::createMainConfig($db_hostport, $db_user, $db_pass, $db_name, $use_utf8mb4, $log_deprecation_warnings)) {
if (!DBConnection::createMainConfig($db_hostport, $db_user, $db_pass, $db_name, $log_deprecation_warnings, $use_utf8mb4)) {
$message = sprintf(
__('Cannot write configuration file "%s".'),
GLPI_CONFIG_DIR . DIRECTORY_SEPARATOR . 'config_db.php'
......@@ -274,6 +297,23 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
return self::ERROR_DB_CONFIG_FILE_NOT_SAVED;
}
// Set $db instance to use new connection properties
$this->db = new class($db_hostport, $db_user, $db_pass, $db_name, $log_deprecation_warnings, $use_utf8mb4) extends DBmysql {
public function __construct($dbhost, $dbuser, $dbpassword, $dbdefault, $log_deprecation_warnings, $use_utf8mb4) {
$this->dbhost = $dbhost;
$this->dbuser = $dbuser;
$this->dbpassword = $dbpassword;
$this->dbdefault = $dbdefault;
$this->use_utf8mb4 = $use_utf8mb4;
$this->log_deprecation_warnings = $log_deprecation_warnings;
$this->clearSchemaCache();
parent::__construct();
}
};
return self::SUCCESS;
}
......
......@@ -37,7 +37,6 @@ if (!defined('GLPI_ROOT')) {
}
use Symfony\Component\Console\Input\InputInterface;
use Symfony\Component\Console\Input\InputOption;
use Symfony\Component\Console\Output\OutputInterface;
class ConfigureCommand extends AbstractConfigureCommand {
......@@ -49,18 +48,11 @@ class ConfigureCommand extends AbstractConfigureCommand {
$this->setName('glpi:database:configure');
$this->setAliases(['db:configure']);
$this->setDescription('Define database configuration');
$this->addOption(
'use-utf8mb4',
null,
InputOption::VALUE_NONE,
__('Use utf8mb4 character set')
);
}
protected function execute(InputInterface $input, OutputInterface $output) {
$result = $this->configureDatabase($input, $output, $input->getOption('use-utf8mb4'));
$result = $this->configureDatabase($input, $output);
if (self::ABORTED_BY_USER === $result) {
return 0; // Considered as success
......
......@@ -36,7 +36,6 @@ if (!defined('GLPI_ROOT')) {
die("Sorry. You can't access this file directly");
}
use DB;
use DBConnection;
use DBmysql;
use Glpi\Cache\CacheManager;
......@@ -151,8 +150,6 @@ class InstallCommand extends AbstractConfigureCommand {
protected function execute(InputInterface $input, OutputInterface $output) {
global $DB;
$default_language = $input->getOption('default-language');
$force = $input->getOption('force');
......@@ -168,7 +165,7 @@ class InstallCommand extends AbstractConfigureCommand {
}
if (!$this->isDbAlreadyConfigured() || $input->getOption('reconfigure')) {
$result = $this->configureDatabase($input, $output, true);
$result = $this->configureDatabase($input, $output, false, true);
if (self::ABORTED_BY_USER === $result) {
return 0; // Considered as success
......@@ -184,6 +181,7 @@ class InstallCommand extends AbstractConfigureCommand {
$db_pass = $input->getOption('db-password');
} else {
// Ask to confirm installation based on existing configuration.
global $DB;
// $DB->dbhost can be array when using round robin feature
$db_hostport = is_array($DB->dbhost) ? $DB->dbhost[0] : $DB->dbhost;
......@@ -216,6 +214,8 @@ class InstallCommand extends AbstractConfigureCommand {
);
return 0;
}
$this->db = $DB;
}
// Create security key
......@@ -296,33 +296,13 @@ class InstallCommand extends AbstractConfigureCommand {
return self::ERROR_DB_ALREADY_CONTAINS_TABLES;
}
if ($DB instanceof DBmysql) {
// If global $DB is set at this point, it means that configuration file has been loaded
// prior to reconfiguration.
// As configuration is part of a class, it cannot be reloaded and class properties
// have to be updated manually in order to make `Toolbox::createSchema()` work correctly.
$DB->dbhost = $db_hostport;
$DB->dbuser = $db_user;
$DB->dbpassword = rawurlencode($db_pass);
$DB->dbdefault = $db_name;
$DB->use_utf8mb4 = true;
$DB->log_deprecation_warnings = $input->getOption('log-deprecation-warnings');
$DB->clearSchemaCache();
$DB->connect();
$db_instance = $DB;
} else {
include_once (GLPI_CONFIG_DIR . "/config_db.php");
$db_instance = new DB();
}
$output->writeln(
'<comment>' . __('Loading default schema...') . '</comment>',
OutputInterface::VERBOSITY_VERBOSE
);
// TODO Get rid of output buffering
ob_start();
Toolbox::createSchema($default_language, $db_instance);
Toolbox::createSchema($default_language, $this->db);
$message = ob_get_clean();
if (!empty($message)) {
$output->writeln('<error>' . $message . '</error>', OutputInterface::VERBOSITY_QUIET);
......
......@@ -190,7 +190,7 @@ class Utf8mb4Command extends AbstractCommand {
$this->output->write(PHP_EOL);
}
if (!DBConnection::updateConfigProperty('use_utf8mb4', true)) {
if (!DBConnection::updateConfigProperty(DBConnection::PROPERTY_USE_UTF8MB4, true)) {
throw new \Glpi\Console\Exception\EarlyExitException(
'<error>' . __('Unable to update DB configuration file.') . '</error>',
self::ERROR_UNABLE_TO_UPDATE_CONFIG
......
......@@ -40,6 +40,18 @@ if (!defined('GLPI_ROOT')) {
**/
class DBConnection extends CommonDBTM {
/**
* "Log deprecation warnings" property name.
* @var string
*/
public const PROPERTY_LOG_DEPRECATION_WARNINGS = 'log_deprecation_warnings';
/**
* "Use UTF8MB4" property name.
* @var string
*/
public const PROPERTY_USE_UTF8MB4 = 'use_utf8mb4';
static protected $notable = true;
......@@ -57,8 +69,8 @@ class DBConnection extends CommonDBTM {
* @param string $user The DB user
* @param string $password The DB password
* @param string $dbname The name of the DB
* @param boolean $use_utf8mb4 Flag that indicates if utf8mb4 charset/collation should be used
* @param boolean $log_deprecation_warnings Flag that indicates if DB deprecation warnings should be logged
* @param boolean $use_utf8mb4 Flag that indicates if utf8mb4 charset/collation should be used
* @param string $config_dir
*
* @return boolean
......@@ -68,8 +80,8 @@ class DBConnection extends CommonDBTM {
string $user,
string $password,
string $dbname,
bool $use_utf8mb4 = false,
bool $log_deprecation_warnings = false,
bool $use_utf8mb4 = false,
string $config_dir = GLPI_CONFIG_DIR
): bool {
......@@ -79,11 +91,11 @@ class DBConnection extends CommonDBTM {
'dbpassword' => rawurlencode($password),
'dbdefault' => $dbname,
];
if ($use_utf8mb4) {
$properties['use_utf8mb4'] = true;
}
if ($log_deprecation_warnings) {
$properties['log_deprecation_warnings'] = true;
$properties[self::PROPERTY_LOG_DEPRECATION_WARNINGS] = true;
}
if ($use_utf8mb4) {
$properties[self::PROPERTY_USE_UTF8MB4] = true;
}
$config_str = '<?php' . "\n" . 'class DB extends DBmysql {' . "\n";
......@@ -109,6 +121,22 @@ class DBConnection extends CommonDBTM {
* @since 10.0.0
*/
static function updateConfigProperty($name, $value, $update_slave = true, string $config_dir = GLPI_CONFIG_DIR): bool {
return self::updateConfigProperties([$name => $value], $update_slave, $config_dir);
}
/**
* Change variables value in config(s) file.
*
* @param array $properties
* @param bool $update_slave
* @param string $config_dir
*
* @return boolean
*
* @since 10.0.0
*/
static function updateConfigProperties(array $properties, $update_slave = true, string $config_dir = GLPI_CONFIG_DIR): bool {
$main_config_file = 'config_db.php';
$slave_config_file = 'config_db_slave.php';
......@@ -116,12 +144,6 @@ class DBConnection extends CommonDBTM {
return false;
}
if ($name === 'password') {
$value = rawurlencode($value);
}
$pattern = '/(?<line>' . preg_quote('$' . $name, '/') . '\s*=\s*(?<value>[^;]+)\s*;)' . '/';
$files = [$main_config_file];
if ($update_slave && file_exists($config_dir . '/' . $slave_config_file)) {
$files[] = $slave_config_file;
......@@ -132,17 +154,25 @@ class DBConnection extends CommonDBTM {
return false;
}
$matches = [];
if (preg_match($pattern, $config_str, $matches)) {
// Property declaration is located in config file, we have to update it.
$updated_line = str_replace($matches['value'], var_export($value, true), $matches['line']);
$config_str = str_replace($matches['line'], $updated_line, $config_str);
} else {
// Property declaration is not located in config file, we have to add it.
$ending_bracket_pos = mb_strrpos($config_str, '}');
$config_str = mb_substr($config_str, 0, $ending_bracket_pos)
. sprintf(' public $%s = %s;', $name, var_export($value, true)) . "\n"
. mb_substr($config_str, $ending_bracket_pos);
foreach ($properties as $name => $value) {
if ($name === 'password') {
$value = rawurlencode($value);
}
$pattern = '/(?<line>' . preg_quote('$' . $name, '/') . '\s*=\s*(?<value>[^;]+)\s*;)' . '/';
$matches = [];
if (preg_match($pattern, $config_str, $matches)) {
// Property declaration is located in config file, we have to update it.
$updated_line = str_replace($matches['value'], var_export($value, true), $matches['line']);
$config_str = str_replace($matches['line'], $updated_line, $config_str);
} else {
// Property declaration is not located in config file, we have to add it.
$ending_bracket_pos = mb_strrpos($config_str, '}');
$config_str = mb_substr($config_str, 0, $ending_bracket_pos)
. sprintf(' public $%s = %s;', $name, var_export($value, true)) . "\n"
. mb_substr($config_str, $ending_bracket_pos);
}
}
if (!Toolbox::writeConfig($file, $config_str, $config_dir)) {
......@@ -161,8 +191,8 @@ class DBConnection extends CommonDBTM {
* @param string $user The DB user
* @param string $password The DB password
* @param string $dbname The name of the DB
* @param boolean $use_utf8mb4 Flag that indicates if utf8mb4 charset/collation should be used
* @param boolean $log_deprecation_warnings Flag that indicates if DB deprecation warnings should be logged
* @param boolean $use_utf8mb4 Flag that indicates if utf8mb4 charset/collation should be used
* @param string $config_dir
*
* @return boolean for success
......@@ -172,8 +202,8 @@ class DBConnection extends CommonDBTM {
string $user,
string $password,
string $dbname,
bool $use_utf8mb4 = false,
bool $log_deprecation_warnings = false,
bool $use_utf8mb4 = false,
string $config_dir = GLPI_CONFIG_DIR
): bool {
......@@ -190,11 +220,11 @@ class DBConnection extends CommonDBTM {
'dbpassword' => rawurlencode($password),
'dbdefault' => $dbname,
];
if ($use_utf8mb4) {
$properties['use_utf8mb4'] = true;
}
if ($log_deprecation_warnings) {
$properties['log_deprecation_warnings'] = true;
$properties[self::PROPERTY_LOG_DEPRECATION_WARNINGS] = true;
}
if ($use_utf8mb4) {
$properties[self::PROPERTY_USE_UTF8MB4] = true;
}
$config_str = '<?php' . "\n" . 'class DB extends DBmysql {' . "\n";
......@@ -238,7 +268,7 @@ class DBConnection extends CommonDBTM {
**/
static function createDBSlaveConfig() {
global $DB;
self::createSlaveConnectionFile("localhost", "glpi", "glpi", "glpi", $DB->use_utf8mb4, $DB->log_deprecation_warnings);
self::createSlaveConnectionFile("localhost", "glpi", "glpi", "glpi", $DB->log_deprecation_warnings, $DB->use_utf8mb4);
}
......@@ -252,7 +282,7 @@ class DBConnection extends CommonDBTM {
**/
static function saveDBSlaveConf($host, $user, $password, $DBname) {
global $DB;
self::createSlaveConnectionFile($host, $user, $password, $DBname, $DB->use_utf8mb4, $DB->log_deprecation_warnings);
self::createSlaveConnectionFile($host, $user, $password, $DBname, $DB->log_deprecation_warnings, $DB->use_utf8mb4);
}
......
......@@ -112,20 +112,20 @@ class DBmysql {
public $dbsslcacipher = null;
/**
* Determine if utf8mb4 should be used for DB connection and tables altering operations.
* Defaults to false to keep backward compatibility with old DB.
* Determine if warnings related to MySQL deprecations should be logged too.
* Defaults to false as this option should only on development/test environment.
*
* @var bool
*/
public $use_utf8mb4 = false;
public $log_deprecation_warnings = false;
/**
* Determine if warnings related to MySQL deprecations should be logged too.
* Defaults to false as this option should only on development/test environment.
* Determine if utf8mb4 should be used for DB connection and tables altering operations.
* Defaults to false to keep backward compatibility with old DB.
*
* @var bool
*/
public $log_deprecation_warnings = false;
public $use_utf8mb4 = false;
/** Is it a first connection ?
......@@ -599,9 +599,11 @@ class DBmysql {
/**
* Returns tables not using "utf8mb4_unicode_ci" collation.
*
* @param bool $exclude_plugins
*
* @return DBmysqlIterator
*/
public function getNonUtf8mb4Tables(): DBmysqlIterator {
public function getNonUtf8mb4Tables(bool $exclude_plugins = false): DBmysqlIterator {
// Find tables that does not use utf8mb4 collation
$tables_query = [
......@@ -645,10 +647,16 @@ class DBmysql {
],
];
if ($exclude_plugins) {
$tables_query['WHERE'][] = ['NOT' => ['information_schema.tables.table_name' => ['LIKE', 'glpi\_plugin\_%']]];
$columns_query['WHERE'][] = ['NOT' => ['information_schema.tables.table_name' => ['LIKE', 'glpi\_plugin\_%']]];
}
$iterator = $this->request([
'SELECT' => ['TABLE_NAME'],
'DISTINCT' => true,
'FROM' => new QueryUnion([$tables_query, $columns_query], true),
'ORDER' => ['TABLE_NAME']
]);
return $iterator;
......@@ -1746,4 +1754,20 @@ class DBmysql {
trigger_error($stmt->error, E_USER_ERROR);
}
}
/**
* Return configuration boolean properties computed using current state of tables.
*
* @return array
*/
public function getComputedConfigBooleanFlags(): array {
$config_flags = [];
if ($this->getNonUtf8mb4Tables(true)->count() === 0) {
// Use utf8mb4 charset for update process if there all core table are using this charset.
$config_flags[DBConnection::PROPERTY_USE_UTF8MB4] = true;
}
return $config_flags;
}
}
......@@ -90,8 +90,6 @@ class QueryUnion extends AbstractQuery {
* @return string
*/
public function getQuery() {
global $DB;
$union_queries = $this->getQueries();
if (empty($union_queries
)) {
......@@ -112,7 +110,7 @@ class QueryUnion extends AbstractQuery {
$alias = $this->alias !== null
? $this->alias
: 'union_' . md5($query);
$query .= ' AS ' . $DB->quoteName($alias);
$query .= ' AS ' . DBmysql::quoteName($alias);
return $query;
}
......
......@@ -264,7 +264,7 @@ function step4 ($databasename, $newdatabasename) {
prev_form($host, $user, $password);
} else {
if (DBConnection::createMainConfig($host, $user, $password, $databasename, true)) {
if (DBConnection::createMainConfig($host, $user, $password, $databasename, false, true)) {
Toolbox::createSchema($_SESSION["glpilanguage"]);
echo "<p>".__('OK - database was initialized')."</p>";
......@@ -281,7 +281,7 @@ function step4 ($databasename, $newdatabasename) {
if ($link->select_db($newdatabasename)) {
echo "<p>".__('Database created')."</p>";
if (DBConnection::createMainConfig($host, $user, $password, $newdatabasename, true)) {
if (DBConnection::createMainConfig($host, $user, $password, $newdatabasename, false, true)) {
Toolbox::createSchema($_SESSION["glpilanguage"]);
echo "<p>".__('OK - database was initialized')."</p>";
next_form();
......@@ -296,7 +296,7 @@ function step4 ($databasename, $newdatabasename) {
echo "<p>".__('Database created')."</p>";
if ($link->select_db($newdatabasename)
&& DBConnection::createMainConfig($host, $user, $password, $newdatabasename, true)) {
&& DBConnection::createMainConfig($host, $user, $password, $newdatabasename, false, true)) {
Toolbox::createSchema($_SESSION["glpilanguage"]);
echo "<p>".__('OK - database was initialized')."</p>";
......@@ -396,13 +396,8 @@ function update1($DBname) {
include_once (GLPI_CONFIG_DIR . "/config_db.php");
global $DB;
$DB = new DB();
if ($DB->listTables('glpi\_%', ['table_collation' => 'utf8mb4_unicode_ci'])->count() > 0) {
// Use utf8mb4 charset for update process if at least one table already uses this charset.
if ($success = DBConnection::updateConfigProperty('use_utf8mb4', true)) {
$DB->use_utf8mb4 = true;
$DB->setConnectionCharset();
}
}
$success = DBConnection::updateConfigProperties($DB->getComputedConfigBooleanFlags());
}
if ($success) {
$from_install = true;
......
......@@ -78,8 +78,8 @@ class DBConnection extends \GLPITestCase {
'user' => 'glpi',
'password' => 'secret',
'name' => 'glpi_db',
'use_utf8mb4' => false,
'log_deprecation_warnings' => false,
'use_utf8mb4' => false,
'expected' => <<<'PHP'
<?php
class DB extends DBmysql {
......@@ -96,8 +96,8 @@ PHP
'user' => 'root',
'password' => '',
'name' => 'db',
'use_utf8mb4' => true,
'log_deprecation_warnings' => false,
'use_utf8mb4' => true,
'expected' => <<<'PHP'
<?php
class DB extends DBmysql {
......@@ -115,8 +115,8 @@ PHP
'user' => 'root',
'password' => 'iT4%dU9*rI9#jT8>',
'name' => 'db',
'use_utf8mb4' => false,
'log_deprecation_warnings' => true,
'use_utf8mb4' => false,
'expected' => <<<'PHP'
<?php
class DB extends DBmysql {
......@@ -140,13 +140,13 @@ PHP
string $user,
string $password,
string $name,
bool $use_utf8mb4,
bool $log_deprecation_warnings,
bool $use_utf8mb4,
string $expected
): void {
vfsStream::setup('config-dir', null, []);
$result = \DBConnection::createMainConfig($host, $user, $password, $name, $use_utf8mb4, $log_deprecation_warnings, vfsStream::url('config-dir'));
$result = \DBConnection::createMainConfig($host, $user, $password, $name, $log_deprecation_warnings, $use_utf8mb4, vfsStream::url('config-dir'));
$this->boolean($result)->isTrue();
$path = vfsStream::url('config-dir/config_db.php');
......@@ -161,8 +161,8 @@ PHP
'user' => 'glpi',
'password' => 'secret',