Adding tests for 4 byte unicode characters

Jenkinsfile

@@ -16,13 +16,23 @@ timestampedNode('SLAVE') {
 
     stage 'PHPUnit 7.1/sqlite'
         executeAndReport('tests/autotest-results-sqlite.xml') {
-            sh '''
-            export NOCOVERAGE=1
-            unset USEDOCKER
-            phpenv local 7.1
-            make test-php TEST_DATABASE=sqlite
-            '''
-        }
+	        sh '''
+        	export NOCOVERAGE=1
+        	unset USEDOCKER
+        	phpenv local 7.1
+		make test-php TEST_DATABASE=sqlite
+        	'''
+	}
+
+    stage 'phpunit/7.0/mysqlmb4'
+        executeAndReport('tests/autotest-results-sqlite.xml') {
+	        sh '''
+        	export NOCOVERAGE=1
+        	unset USEDOCKER
+        	phpenv local 7.0
+		make test-php TEST_DATABASE=mysqlmb4
+        	'''
+	}
 
     stage 'PHPUnit 7.0/sqlite'
         executeAndReport('tests/autotest-results-sqlite.xml') {

build/autotest.sh

@@ -21,7 +21,7 @@ ADMINLOGIN=admin$EXECUTOR_NUMBER
 BASEDIR=$PWD
 
 PRIMARY_STORAGE_CONFIGS="local swift"
-DBCONFIGS="sqlite mysql mariadb pgsql oci"
+DBCONFIGS="sqlite mysql mariadb pgsql oci mysqlmb4"
 
 # $PHP_EXE is run through 'which' and as such e.g. 'php' or 'hhvm' is usually
 # sufficient. Due to the behaviour of 'which', $PHP_EXE may also be a path
@@ -146,6 +146,8 @@ function cleanup_config {
 	if [ -f config/autotest-storage-swift.config.php ]; then
 		rm config/autotest-storage-swift.config.php
 	fi
+	# Remove mysqlmb4.config.php
+	rm -f config/mysqlmb4.config.php
 }
 
 # restore config on exit
@@ -211,6 +213,31 @@ function execute_tests {
 			mysql -u "$DATABASEUSER" -powncloud -e "DROP DATABASE IF EXISTS $DATABASENAME" -h $DATABASEHOST || true
 		fi
 	fi
+	if [ "$DB" == "mysqlmb4" ] ; then
+		echo "Fire up the mysql docker"
+		DOCKER_CONTAINER_ID=$(docker run \
+			-v $BASEDIR/tests/docker/mysqlmb4:/etc/mysql/conf.d \
+			-e MYSQL_ROOT_PASSWORD=owncloud \
+			-e MYSQL_USER="$DATABASEUSER" \
+			-e MYSQL_PASSWORD=owncloud \
+			-e MYSQL_DATABASE="$DATABASENAME" \
+			-d mysql:5.7)
+
+		DATABASEHOST=$(docker inspect --format="{{.NetworkSettings.IPAddress}}" "$DOCKER_CONTAINER_ID")
+
+		echo "Waiting for MySQL(utf8mb4) initialisation ..."
+
+		if ! apps/files_external/tests/env/wait-for-connection $DATABASEHOST 3306 60; then
+			echo "[ERROR] Waited 60 seconds, no response" >&2
+			exit 1
+		fi
+		sleep 1
+
+		echo "MySQL(utf8mb4)  is up."
+		_DB="mysql"
+
+		cp tests/docker/mysqlmb4.config.php config
+	fi
 	if [ "$DB" == "mariadb" ] ; then
 		if [ ! -z "$USEDOCKER" ] ; then
 			echo "Fire up the mariadb docker"

config/config.sample.php

@@ -123,6 +123,7 @@
  */
 'dbtableprefix' => '',
 
+
 /**
  * Indicates whether the ownCloud instance was installed successfully; ``true``
  * indicates a successful installation, and ``false`` indicates an unsuccessful
@@ -1069,6 +1070,32 @@
  */
 'sqlite.journal_mode' => 'DELETE',
 
+/**
+ * If this setting is set to true MySQL can handle 4 byte characters instead of
+ * 3 byte characters
+ *
+ * MySQL requires a special setup for longer indexes (> 767 bytes) which are
+ * needed:
+ *
+ * [mysqld]
+ * innodb_large_prefix=ON
+ * innodb_file_format=Barracuda
+ * innodb_file_per_table=ON
+ *
+ * Tables will be created with
+ *  * character set: utf8mb4
+ *  * collation:     utf8mb4_bin
+ *  * row_format:    compressed
+ *
+ * See:
+ * https://dev.mysql.com/doc/refman/5.7/en/charset-unicode-utf8mb4.html
+ * https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_large_prefix
+ * https://mariadb.com/kb/en/mariadb/xtradbinnodb-server-system-variables/#innodb_large_prefix
+ * http://www.tocker.ca/2013/10/31/benchmarking-innodb-page-compression-performance.html
+ * http://mechanics.flite.com/blog/2014/07/29/using-innodb-large-prefix-to-avoid-error-1071/
+ */
+'mysql.utf8mb4' => false,
+
 /**
  * Database types that are supported for installation.
  *

core/Command/Db/ConvertMysqlToMB4.php

@@ -0,0 +1,85 @@
+<?php
+/**
+ * @author Thomas Müller <thomas.mueller@tmit.eu>
+ *
+ * @copyright Copyright (c) 2017, ownCloud GmbH
+ * @license AGPL-3.0
+ *
+ * This code is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3,
+ * as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License, version 3,
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>
+ *
+ */
+
+namespace OC\Core\Command\Db;
+
+use Doctrine\DBAL\Platforms\MySqlPlatform;
+use OC\DB\MySqlTools;
+use OC\Migration\ConsoleOutput;
+use OC\Repair\Collation;
+use OCP\IConfig;
+use OCP\IDBConnection;
+use OCP\IURLGenerator;
+use Symfony\Component\Console\Command\Command;
+use Symfony\Component\Console\Input\InputInterface;
+use Symfony\Component\Console\Output\OutputInterface;
+
+class ConvertMysqlToMB4 extends Command {
+	/** @var IConfig */
+	private $config;
+
+	/** @var IDBConnection */
+	private $connection;
+
+	/** @var IURLGenerator */
+	private $urlGenerator;
+
+	/**
+	 * @param IConfig $config
+	 * @param IDBConnection $connection
+	 */
+	public function __construct(IConfig $config, IDBConnection $connection, IURLGenerator $urlGenerator) {
+		$this->config = $config;
+		$this->connection = $connection;
+		$this->urlGenerator = $urlGenerator;
+		parent::__construct();
+	}
+
+	protected function configure() {
+		$this
+			->setName('db:convert-mysql-charset')
+			->setDescription('Convert charset of MySQL/MariaDB to use utf8mb4');
+	}
+
+	protected function execute(InputInterface $input, OutputInterface $output) {
+		if (!$this->connection->getDatabasePlatform() instanceof MySqlPlatform) {
+			$output->writeln("This command is only valid for MySQL/MariaDB databases.");
+			return 1;
+		}
+
+		$tools = new MySqlTools();
+		if (!$tools->supports4ByteCharset($this->connection)) {
+			$url = $this->urlGenerator->linkToDocs('admin-db-conversion');
+			$output->writeln("The database is not properly setup to use the charset utf8mb4.");
+			$output->writeln("For more information please read the documentation at $url");
+			return 1;
+		}
+
+		// enable charset
+		$this->config->setSystemValue('mysql.utf8mb4', true);
+
+		// run conversion
+		$coll = new Collation($this->config, $this->connection);
+		$coll->run(new ConsoleOutput($output));
+
+		return 0;
+	}
+}

core/register_command.php

@@ -81,7 +81,8 @@
 	$application->add(new OC\Core\Command\Config\System\SetConfig(\OC::$server->getSystemConfig()));
 
 	$application->add(new OC\Core\Command\Db\GenerateChangeScript());
-	$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory()));
+	$application->add(new OC\Core\Command\Db\ConvertType(\OC::$server->getConfig(), new \OC\DB\ConnectionFactory(\OC::$server->getSystemConfig())));
+	$application->add(new OC\Core\Command\Db\ConvertMysqlToMB4(\OC::$server->getConfig(), \OC::$server->getDatabaseConnection(), \OC::$server->getURLGenerator()));
 	$application->add(new OC\Core\Command\Db\Migrations\StatusCommand(\OC::$server->getDatabaseConnection()));
 	$application->add(new OC\Core\Command\Db\Migrations\MigrateCommand(\OC::$server->getDatabaseConnection()));
 	$application->add(new OC\Core\Command\Db\Migrations\GenerateCommand(\OC::$server->getDatabaseConnection()));

lib/private/AppFramework/Db/Db.php

@@ -246,4 +246,11 @@ public function migrateToSchema(Schema $toSchema) {
 	public function getTransactionIsolation() {
 		return $this->connection->getTransactionIsolation();
 	}
+
+	/**
+	 * @inheritdoc
+	 */
+	public function allows4ByteCharacters() {
+		return $this->connection->allows4ByteCharacters();
+	}
 }

lib/private/DB/AdapterMySQL.php

@@ -38,7 +38,8 @@ public function unlockTable() {
 	}
 
 	public function fixupStatement($statement) {
-		$statement = str_replace(' ILIKE ', ' COLLATE utf8_general_ci LIKE ', $statement);
+		$characterSet = \OC::$server->getConfig()->getSystemValue('mysql.utf8mb4', false) ? 'utf8mb4' : 'utf8';
+		$statement = str_replace(' ILIKE ', ' COLLATE ' . $characterSet . '_general_ci LIKE ', $statement);
 		return $statement;
 	}
 }

lib/private/DB/Connection.php

@@ -33,6 +33,7 @@
 use Doctrine\DBAL\Cache\QueryCacheProfile;
 use Doctrine\Common\EventManager;
 use Doctrine\DBAL\Exception\ConstraintViolationException;
+use Doctrine\DBAL\Platforms\MySqlPlatform;
 use Doctrine\DBAL\Schema\Schema;
 use OC\DB\QueryBuilder\QueryBuilder;
 use OCP\DB\QueryBuilder\IQueryBuilder;
@@ -427,4 +428,19 @@ public function migrateToSchema(Schema $toSchema) {
 		$migrator->migrate($toSchema);
 	}
 
+	/**
+	 * Are 4-byte characters allowed or only 3-byte
+	 *
+	 * @return bool
+	 * @since 10.0
+	 */
+	public function allows4ByteCharacters() {
+		if (!$this->getDatabasePlatform() instanceof MySqlPlatform) {
+			return true;
+		}
+		if ($this->getParams()['charset'] === 'utf8mb4') {
+			return true;
+		}
+		return false;
+	}
 }