Use utf8mb4 MySQL character set

 - Use utf8mb4 as default charset for fresh installation
 - utf8mb4 migration command
 - warn for collation mix on migration
 - Force ROW_FORMAT = Dynamic on fresh install
 - Add warning on central page if some tables are not migrated

.github/workflows/ci.yml

@@ -224,12 +224,13 @@ jobs:
           docker exec app bin/console glpi:migration:myisam_to_innodb --config-dir=./tests --no-interaction | grep -q "Aucune migration requise." || (echo "glpi:migration:myisam_to_innodb command FAILED" && exit 1)
           docker exec app bin/console glpi:migration:timestamps --config-dir=./tests --no-interaction
           docker exec app bin/console glpi:migration:timestamps --config-dir=./tests --no-interaction | grep -q "Aucune migration requise." || (echo "glpi:migration:timestamps command FAILED" && exit 1)
+          docker exec app bin/console glpi:migration:utf8mb4 --config-dir=./tests --no-interaction
+          docker exec app bin/console glpi:migration:utf8mb4 --config-dir=./tests --no-interaction | grep -q "Aucune migration requise." || (echo "glpi:migration:utf8mb4 command FAILED" && exit 1)
       - name: "Database tests"
         if: env.skip != 'true'
         run: |
-          docker exec app bin/console glpi:database:configure --config-dir=./tests --no-interaction --reconfigure --db-name=glpi --db-host=db --db-user=root
+          docker exec app bin/console glpi:database:configure --config-dir=./tests --no-interaction --reconfigure --db-name=glpi --db-host=db --db-user=root --use-utf8mb4
           docker exec app vendor/bin/atoum -p 'php -d memory_limit=512M' --debug --force-terminal --use-dot-report --bootstrap-file tests/bootstrap.php --no-code-coverage --fail-if-skipped-methods --max-children-number 1 -d tests/database
-          docker exec app bin/console glpi:database:configure --config-dir=./tests --no-interaction --reconfigure --db-name=glpi --db-host=db --db-user=root
       - name: "Unit tests"
         if: env.skip != 'true'
         run: |

CHANGELOG.md

@@ -13,6 +13,9 @@ The present file will list all changes made to the project; according to the
 ### Deprecated
 - Usage of XML-RPC API is deprecated.
 
+### Removed
+- Usage of alternative DB connection encoding (`DB::$dbenc` property).
+
 ### API changes
 
 #### Added

inc/central.class.php

@@ -329,12 +329,16 @@ class Central extends CommonGLPI {
          if ($DB->areTimezonesAvailable()) {
             $not_tstamp = $DB->notTzMigrated();
             if ($not_tstamp > 0) {
-                $warnings[] = sprintf(
-                    __('%1$s columns are not compatible with timezones usage.'),
-                    $not_tstamp
-                );
+                $warnings[] = sprintf(__('%1$s columns are not compatible with timezones usage.'), $not_tstamp)
+               . ' '
+               . sprintf(__('Run the "php bin/console %1$s" command to migrate them.'), 'glpi:migration:timestamps');
             }
          }
+         if (($non_utf8mb4_tables = $DB->getNonUtf8mb4Tables()->count()) > 0) {
+            $warnings[] = sprintf(__('%1$s tables not migrated to utf8mb4 collation.'), $non_utf8mb4_tables)
+               . ' '
+               . sprintf(__('Run the "php bin/console %1$s" command to migrate them.'), 'glpi:migration:utf8mb4');
+         }
       }
 
       if ($DB->isSlave()

inc/console/abstractcommand.class.php

@@ -44,6 +44,7 @@ use Symfony\Component\Console\Exception\RuntimeException;
 use Symfony\Component\Console\Helper\ProgressBar;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Output\OutputInterface;
+use Symfony\Component\Console\Question\ConfirmationQuestion;
 
 abstract class AbstractCommand extends Command implements GlpiCommandInterface {
 
@@ -210,4 +211,26 @@ abstract class AbstractCommand extends Command implements GlpiCommandInterface {
 
       return $this->requires_db && $this->requires_db_up_to_date;
    }
+
+   /**
+    * Ask for user confirmation before continuing command execution.
+    *
+    * @return void
+    */
+   protected function askForConfirmation(): void {
+      if (!$this->input->getOption('no-interaction')) {
+         $question_helper = $this->getHelper('question');
+         $run = $question_helper->ask(
+            $this->input,
+            $this->output,
+            new ConfirmationQuestion(__('Do you want to continue ?') . ' [Yes/no]', true)
+         );
+         if (!$run) {
+            throw new \Glpi\Console\Exception\EarlyExitException(
+               '<comment>' . __('Aborted.') . '</comment>',
+               0 // Success code
+            );
+         }
+      }
+   }
 }

inc/console/application.class.php

@@ -244,7 +244,12 @@ class Application extends BaseApplication {
          return self::ERROR_MISSING_REQUIREMENTS;
       }
 
-      $result = parent::doRunCommand($command, $input, $output);
+      try {
+         $result = parent::doRunCommand($command, $input, $output);
+      } catch (\Glpi\Console\Exception\EarlyExitException $e) {
+         $result = $e->getCode();
+         $output->writeln($e->getMessage(), OutputInterface::VERBOSITY_QUIET);
+      }
 
       if ($output->getVerbosity() >= OutputInterface::VERBOSITY_VERY_VERBOSE) {
          $output->writeln(

inc/console/database/abstractconfigurecommand.class.php

@@ -176,10 +176,13 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
     *
     * @param InputInterface $input
     * @param OutputInterface $output
+    * @param bool $use_utf8mb4
+    *
     * @throws InvalidArgumentException
+    *
     * @return string
     */
-   protected function configureDatabase(InputInterface $input, OutputInterface $output) {
+   protected function configureDatabase(InputInterface $input, OutputInterface $output, bool $use_utf8mb4) {
 
       $db_pass     = $input->getOption('db-password');
       $db_host     = $input->getOption('db-host');
@@ -234,6 +237,8 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
          return self::ERROR_DB_CONNECTION_FAILED;
       }
 
+      DBConnection::setConnectionCharset($mysqli, $use_utf8mb4);
+
       ob_start();
       $db_version_data = $mysqli->query('SELECT version()')->fetch_array();
       $checkdb = Config::displayCheckDbEngine(false, $db_version_data[0]);
@@ -249,7 +254,7 @@ abstract class AbstractConfigureCommand extends AbstractCommand implements Force
          '<comment>' . __('Saving configuration file...') . '</comment>',
          OutputInterface::VERBOSITY_VERBOSE
       );
-      if (!DBConnection::createMainConfig($db_hostport, $db_user, $db_pass, $db_name)) {
+      if (!DBConnection::createMainConfig($db_hostport, $db_user, $db_pass, $db_name, $use_utf8mb4)) {
          $message = sprintf(
             __('Cannot write configuration file "%s".'),
             GLPI_CONFIG_DIR . DIRECTORY_SEPARATOR . 'config_db.php'

inc/console/database/configurecommand.class.php

@@ -37,6 +37,7 @@ if (!defined('GLPI_ROOT')) {
 }
 
 use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Input\InputOption;
 use Symfony\Component\Console\Output\OutputInterface;
 
 class ConfigureCommand extends AbstractConfigureCommand {
@@ -48,11 +49,18 @@ class ConfigureCommand extends AbstractConfigureCommand {
       $this->setName('glpi:database:configure');
       $this->setAliases(['db:configure']);
       $this->setDescription('Define database configuration');
+
+      $this->addOption(
+         'use-utf8mb4',
+         null,
+         InputOption::VALUE_NONE,
+         __('Use utf8mb4 character set.')
+      );
    }
 
    protected function execute(InputInterface $input, OutputInterface $output) {
 
-      $result = $this->configureDatabase($input, $output);
+      $result = $this->configureDatabase($input, $output, $input->getOption('use-utf8mb4'));
 
       if (self::ABORTED_BY_USER === $result) {
          return 0; // Considered as success

inc/console/database/installcommand.class.php

@@ -37,6 +37,9 @@ if (!defined('GLPI_ROOT')) {
 }
 
 use DB;
+use DBConnection;
+use DBmysql;
+use Glpi\System\Requirement\DbConfiguration;
 use GLPIKey;
 use Symfony\Component\Console\Input\InputInterface;
 use Symfony\Component\Console\Input\InputOption;
@@ -74,6 +77,13 @@ class InstallCommand extends AbstractConfigureCommand {
     */
    const ERROR_CANNOT_CREATE_ENCRYPTION_KEY_FILE = 8;
 
+   /**
+    * Error code returned if DB configuration is not compatible with large indexes.
+    *
+    * @var integer
+    */
+   const ERROR_INCOMPATIBLE_DB_CONFIG = 9;
+
    protected function configure() {
 
       parent::configure();
@@ -149,7 +159,7 @@ class InstallCommand extends AbstractConfigureCommand {
       }
 
       if (!$this->isDbAlreadyConfigured() || $input->getOption('reconfigure')) {
-         $result = $this->configureDatabase($input, $output);
+         $result = $this->configureDatabase($input, $output, true);
 
          if (self::ABORTED_BY_USER === $result) {
             return 0; // Considered as success
@@ -226,6 +236,23 @@ class InstallCommand extends AbstractConfigureCommand {
          return self::ERROR_DB_CONNECTION_FAILED;
       }
 
+      // Check for compatibility with utf8mb4 usage.
+      $db = new class($mysqli) extends DBmysql {
+         public function __construct($dbh) {
+            $this->dbh = $dbh;
+         }
+      };
+      $config_requirement = new DbConfiguration($db);
+      if (!$config_requirement->isValidated()) {
+         $msg = '<error>' . __('Database configuration is not compatible with "utf8mb4" usage.') . '</error>';
+         foreach ($config_requirement->getValidationMessages() as $validation_message) {
+            $msg .= "\n" . '<error> - ' . $validation_message . '</error>';
+         }
+         throw new \Glpi\Console\Exception\EarlyExitException($msg, self::ERROR_INCOMPATIBLE_DB_CONFIG);
+      }
+
+      DBConnection::setConnectionCharset($mysqli, true);
+
       // Create database or select existing one
       $output->writeln(
          '<comment>' . __('Creating the database...') . '</comment>',
@@ -260,15 +287,16 @@ class InstallCommand extends AbstractConfigureCommand {
          return self::ERROR_DB_ALREADY_CONTAINS_TABLES;
       }
 
-      if ($DB instanceof DB) {
+      if ($DB instanceof DBmysql) {
          // If global $DB is set at this point, it means that configuration file has been loaded
          // prior to reconfiguration.
          // As configuration is part of a class, it cannot be reloaded and class properties
          // have to be updated manually in order to make `Toolbox::createSchema()` work correctly.
-         $DB->dbhost     = $db_hostport;
-         $DB->dbuser     = $db_user;
-         $DB->dbpassword = rawurlencode($db_pass);
-         $DB->dbdefault  = $db_name;
+         $DB->dbhost      = $db_hostport;
+         $DB->dbuser      = $db_user;
+         $DB->dbpassword  = rawurlencode($db_pass);
+         $DB->dbdefault   = $db_name;
+         $DB->use_utf8mb4 = true;
          $DB->clearSchemaCache();
          $DB->connect();
 

inc/console/exception/earlyexitexception.class.php

+<?php
+/**
+ * ---------------------------------------------------------------------
+ * GLPI - Gestionnaire Libre de Parc Informatique
+ * Copyright (C) 2015-2021 Teclib' and contributors.
+ *
+ * http://glpi-project.org
+ *
+ * based on GLPI - Gestionnaire Libre de Parc Informatique
+ * Copyright (C) 2003-2014 by the INDEPNET Development Team.
+ *
+ * ---------------------------------------------------------------------
+ *
+ * LICENSE
+ *
+ * This file is part of GLPI.
+ *
+ * GLPI is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * GLPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with GLPI. If not, see <http://www.gnu.org/licenses/>.
+ * ---------------------------------------------------------------------
+ */
+
+namespace Glpi\Console\Exception;
+
+if (!defined('GLPI_ROOT')) {
+   die("Sorry. You can't access this file directly");
+}
+
+use Symfony\Component\Console\Exception\ExceptionInterface;
+
+/**
+ * This exception is used to easilly trigger an exit of current command from a sub method.
+ *
+ * @since x.x.x
+ */
+class EarlyExitException extends \Exception implements ExceptionInterface {
+
+}

inc/console/migration/utf8mb4command.class.php

+<?php
+/**
+ * ---------------------------------------------------------------------
+ * GLPI - Gestionnaire Libre de Parc Informatique
+ * Copyright (C) 2015-2021 Teclib' and contributors.
+ *
+ * http://glpi-project.org
+ *
+ * based on GLPI - Gestionnaire Libre de Parc Informatique
+ * Copyright (C) 2003-2014 by the INDEPNET Development Team.
+ *
+ * ---------------------------------------------------------------------
+ *
+ * LICENSE
+ *
+ * This file is part of GLPI.
+ *
+ * GLPI is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * GLPI is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with GLPI. If not, see <http://www.gnu.org/licenses/>.
+ * ---------------------------------------------------------------------
+ */
+
+namespace Glpi\Console\Migration;
+
+if (!defined('GLPI_ROOT')) {
+   die("Sorry. You can't access this file directly");
+}
+
+use DBConnection;
+use Glpi\Console\AbstractCommand;
+use Glpi\System\Requirement\DbConfiguration;
+use Symfony\Component\Console\Helper\ProgressBar;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class Utf8mb4Command extends AbstractCommand {
+
+   /**
+    * Error code returned if migration failed on, at least, one table.
+    *
+    * @var integer
+    */
+   const ERROR_MIGRATION_FAILED_FOR_SOME_TABLES = 1;
+
+   /**
+    * Error code returned if DB configuration file cannot be updated.
+    *
+    * @var integer
+    */
+   const ERROR_UNABLE_TO_UPDATE_CONFIG = 2;
+
+   /**
+    * Error code returned if some tables are still using MyISAM engine.
+    *
+    * @var integer
+    */
+   const ERROR_INNODB_REQUIRED = 3;
+
+   /**
+    * Error code returned if DB configuration is not compatible with large indexes.
+    *
+    * @var integer
+    */
+   const ERROR_INCOMPATIBLE_DB_CONFIG = 3;
+
+   protected function configure() {
+      parent::configure();
+
+      $this->setName('glpi:migration:utf8mb4');
+      $this->setDescription(__('Convert database character set from "utf8" to "utf8mb4".'));
+   }
+
+   protected function execute(InputInterface $input, OutputInterface $output) {
+      $this->checkForPrerequisites();
+      $this->upgradeRowFormat();
+      $this->migrateToUtf8mb4();
+
+      return 0; // Success
+   }
+
+   /**
+    * Check for migration prerequisites.
+    *
+    * @return void
+    */
+   private function checkForPrerequisites(): void {
+      // Check that DB configuration is compatible
+      $config_requirement = new DbConfiguration($this->db);
+      if (!$config_requirement->isValidated()) {
+         $msg = '<error>' . __('Database configuration is not compatible with "utf8mb4" usage.') . '</error>';
+         foreach ($config_requirement->getValidationMessages() as $validation_message) {
+            $msg .= "\n" . '<error> - ' . $validation_message . '</error>';
+         }
+         throw new \Glpi\Console\Exception\EarlyExitException($msg, self::ERROR_INCOMPATIBLE_DB_CONFIG);
+      }
+
+      // Check that all tables are using InnoDB engine
+      if (($myisam_count = $this->db->getMyIsamTables()->count()) > 0) {
+         $msg = sprintf(
+            __('%d tables are still using MyISAM storage engine. Run "php bin/console glpi:migration:myisam_to_innodb" to fix this.'),
+            $myisam_count
+         );
+         throw new \Glpi\Console\Exception\EarlyExitException('<error>' . $msg . '</error>', self::ERROR_INNODB_REQUIRED);
+      }
+   }
+
+   /**
+    * Upgrade row format from 'Compact'/'Redundant' to 'Dynamic'.
+    * This is mandatory to support large indexes.
+    *
+    * @return void
+    */
+   private function upgradeRowFormat(): void {
+
+      $table_iterator = $this->db->listTables(
+         'glpi_%',
+         [
+            'row_format'   => ['Compact', 'Redundant'],
+         ]
+      );
+
+      if ($table_iterator->count() === 0) {
+         return;
+      }
+
+      $this->output->writeln(
+         sprintf(
+            '<info>' . __('Found %s table(s) requiring a migration to "ROW_FORMAT=Dynamic".') . '</info>',
+            $table_iterator->count()
+         )
+      );
+
+      $this->askForConfirmation();
+
+      $tables = [];
+      foreach ($table_iterator as $table_data) {
+         $tables[] = $table_data['TABLE_NAME'];
+      }
+      sort($tables);
+
+      $progress_bar = new ProgressBar($this->output);
+      $errors = 0;
+
+      foreach ($progress_bar->iterate($tables) as $table) {
+         $this->writelnOutputWithProgressBar(
+            sprintf(__('Migrating table "%s"...'), $table),
+            $progress_bar,
+            OutputInterface::VERBOSITY_VERY_VERBOSE
+         );
+
+         $result = $this->db->query(
+            sprintf('ALTER TABLE `%s` ROW_FORMAT = DYNAMIC', $table)
+         );
+
+         if (!$result) {
+            $this->writelnOutputWithProgressBar(
+               sprintf(__('<error>Error migrating table "%s".</error>'), $table),
+               $progress_bar,
+               OutputInterface::VERBOSITY_QUIET
+            );
+            $errors++;
+         }
+      }
+
+      $this->output->write(PHP_EOL);
+
+      if ($errors) {
+         throw new \Glpi\Console\Exception\EarlyExitException(
+            '<error>' . __('Errors occured during migration.') . '</error>',
+            self::ERROR_MIGRATION_FAILED_FOR_SOME_TABLES
+         );
+      }
+   }
+
+   /**
+    * Migrate tables to utf8mb4.
+    *
+    * @return void
+    */
+   private function migrateToUtf8mb4(): void {
+
+      $tables = [];
+
+      // Find collations to update at table level
+      $table_iterator = $this->db->getNonUtf8mb4Tables();
+      foreach ($table_iterator as $table_data) {
+         $tables[] = $table_data['TABLE_NAME'];
+      }
+
+      $errors = 0;
+
+      if (count($tables) === 0) {
+         $this->output->writeln('<info>' . __('No migration needed.') . '</info>');
+      } else {
+         sort($tables);
+
+         $this->output->writeln(
+            sprintf(
+               '<info>' . __('Found %s table(s) requiring migration to "utf8mb4".') . '</info>',
+               count($tables)
+            )
+         );
+
+         $this->askForConfirmation();
+
+         // Early update property to prevent warnings related to bad collation detection.
+         $this->db->use_utf8mb4 = true;
+
+         $progress_bar = new ProgressBar($this->output);
+         $errors = 0;
+
+         foreach ($progress_bar->iterate($tables) as $table) {
+            $this->writelnOutputWithProgressBar(
+               sprintf(__('Migrating table "%s"...'), $table),
+               $progress_bar,
+               OutputInterface::VERBOSITY_VERY_VERBOSE
+            );
+
+            $result = $this->db->query(
+               sprintf('ALTER TABLE `%s` CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci', $table)
+            );
+
+            if (!$result) {
+               $this->writelnOutputWithProgressBar(
+                  sprintf(__('<error>Error migrating table "%s".</error>'), $table),
+                  $progress_bar,
+                  OutputInterface::VERBOSITY_QUIET
+               );
+               $errors++;
+            }
+         }
+
+         $this->output->write(PHP_EOL);
+      }
+
+      if (!DBConnection::updateConfigProperty('use_utf8mb4', true)) {
+         throw new \Glpi\Console\Exception\EarlyExitException(
+            '<error>' . __('Unable to update DB configuration file.') . '</error>',
+            self::ERROR_UNABLE_TO_UPDATE_CONFIG
+         );
+      }
+
+      if ($errors) {
+         throw new \Glpi\Console\Exception\EarlyExitException(
+            '<error>' . __('Errors occured during migration.') . '</error>',
+            self::ERROR_MIGRATION_FAILED_FOR_SOME_TABLES
+         );
+      }
+
+      if (count($tables) > 0) {
+         $this->output->writeln('<info>' . __('Migration done.') . '</info>');
+      }
+   }
+}

inc/dbconnection.class.php

@@ -61,19 +61,78 @@ class DBConnection extends CommonDBTM {
     * @return boolean
     *
    **/
-   static function createMainConfig($host, $user, $password, $DBname) {
+   static function createMainConfig($host, $user, $password, $DBname, $use_utf8mb4 = false) {
 
       $DB_str = "<?php\nclass DB extends DBmysql {\n" .
-                "   public \$dbhost     = '$host';\n" .
-                "   public \$dbuser     = '$user';\n" .
-                "   public \$dbpassword = '". rawurlencode($password) . "';\n" .
-                "   public \$dbdefault  = '$DBname';\n" .
+                "   public \$dbhost      = '$host';\n" .
+                "   public \$dbuser      = '$user';\n" .
+                "   public \$dbpassword  = '". rawurlencode($password) . "';\n" .
+                "   public \$dbdefault   = '$DBname';\n" .
+                "   public \$use_utf8mb4 = " . ($use_utf8mb4 ? 'true' : 'false') . ";\n" .
                 "}\n";
 
       return Toolbox::writeConfig('config_db.php', $DB_str);
    }
 
 
+   /**
+    * Change a variable value in config(s) file.
+    *
+    * @param string $name
+    * @param string $value
+    * @param bool   $update_slave
+    * @param string $config_dir
+    *
+    * @return boolean
+    *
+    * @since x.x.x
+    */
+   static function updateConfigProperty($name, $value, $update_slave = true, string $config_dir = GLPI_CONFIG_DIR): bool {
+      $main_config_file = 'config_db.php';
+      $slave_config_file = 'config_db_slave.php';
+
+      if (!file_exists($config_dir . '/' . $main_config_file)) {
+         return false;
+      }
+
+      if ($name === 'password') {
+         $value = rawurlencode($value);
+      }
+
+      $pattern = '/(?<line>' . preg_quote('$' . $name, '/') . '\s*=\s*(?<value>[^;]+)\s*;)' . '/';
+
+      $files = [$main_config_file];
+      if ($update_slave && file_exists($config_dir . '/' . $slave_config_file)) {
+         $files[] = $slave_config_file;
+      }