SONAR-21920 fixed issues in migrations 10_4_002 and 10_4_003

server/sonar-db-dao/src/schema/schema-sq.ddl

@@ -922,7 +922,7 @@ CREATE TABLE "RULE_REPOSITORIES"(
 ALTER TABLE "RULE_REPOSITORIES" ADD CONSTRAINT "PK_RULE_REPOSITORIES" PRIMARY KEY("KEE");
 
 CREATE TABLE "RULE_TAGS"(
-    "VALUE" CHARACTER VARYING(40) NOT NULL,
+    "VALUE" CHARACTER VARYING(400) NOT NULL,
     "RULE_UUID" CHARACTER VARYING(40) NOT NULL,
     "IS_SYSTEM_TAG" BOOLEAN NOT NULL
 );

server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v104/PopulateRuleTagsTableIT.java

@@ -74,6 +74,21 @@ void execute_whenEmptyOrDuplicateTagsExist_shouldNotBeMigrated() throws SQLExcep
       );
   }
 
+  @Test
+  void execute_whenSystemAndCustomTagShareTheSameTag_removeDuplicates() throws SQLException {
+    insertRule("uuid-1", "test,other1", "test,other2");
+
+    migration.execute();
+
+    assertThat(db.select("select value, is_system_tag, rule_uuid from rule_tags"))
+      .extracting(t -> t.get("value"), t -> t.get("is_system_tag"), t -> t.get("rule_uuid"))
+      .containsExactlyInAnyOrder(
+        tuple("test", true, "uuid-1"),
+        tuple("other1", true, "uuid-1"),
+        tuple("other2", false, "uuid-1")
+      );
+  }
+
   @Test
   void execute_whenRunMoreThanOnce_shouldBeReentrant() throws SQLException {
     insertRule("uuid-3", "sys_tag", "tag");
@@ -86,7 +101,7 @@ void execute_whenRunMoreThanOnce_shouldBeReentrant() throws SQLException {
   private void verifyMapping() {
     assertThat(db.select("select value, is_system_tag, rule_uuid from rule_tags"))
       .extracting(t -> t.get("value"), t -> t.get("is_system_tag"), t -> t.get("rule_uuid"))
-      .containsExactly(
+      .containsExactlyInAnyOrder(
         tuple("sys_tag", true, "uuid-3"),
         tuple("tag", false, "uuid-3")
       );

server/sonar-db-migration/src/it/java/org/sonar/server/platform/db/migration/version/v106/ResizeValueColumnInRuleTagsTableIT.java

@@ -0,0 +1,65 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v106;
+
+import java.sql.SQLException;
+import java.sql.Types;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.sonar.db.CoreDbTester;
+import org.sonar.db.MigrationDbTester;
+import org.sonar.server.platform.db.migration.version.v104.CreateRuleTagsTable;
+
+class ResizeValueColumnInRuleTagsTableIT {
+
+  private static final String EXPECTED_TABLE_NAME = "rule_tags";
+  private static final String EXPECTED_COLUMN_NAME = "value";
+
+  /**
+   * This is database that has run the new version of the {@link CreateRuleTagsTable} migration with the 400 limit of the value column.
+   */
+  @RegisterExtension
+  public final MigrationDbTester dbWith400LimitOnValueColumn = MigrationDbTester.createForMigrationStep(ResizeValueColumnInRuleTagsTable.class);
+
+  /**
+   * This is the database that has run the old version of the {@link CreateRuleTagsTable} migration with the 40 limit of the value column.
+   */
+  @RegisterExtension
+  public final CoreDbTester dbWith40LimitOnValueColumn = CoreDbTester.createForSchema(ResizeValueColumnInRuleTagsTableIT.class, "schema.sql");
+
+  private final ResizeValueColumnInRuleTagsTable underTestNoAction = new ResizeValueColumnInRuleTagsTable(dbWith400LimitOnValueColumn.database());
+  private final ResizeValueColumnInRuleTagsTable underTestThatFixesColumnSize = new ResizeValueColumnInRuleTagsTable(dbWith40LimitOnValueColumn.database());
+
+  @Test
+  void execute_whenColumnIsNotResized_shouldResizeTheColumn() throws SQLException {
+    dbWith40LimitOnValueColumn.assertColumnDefinition(EXPECTED_TABLE_NAME, EXPECTED_COLUMN_NAME, Types.VARCHAR, 40, false);
+    underTestThatFixesColumnSize.execute();
+    dbWith40LimitOnValueColumn.assertColumnDefinition(EXPECTED_TABLE_NAME, EXPECTED_COLUMN_NAME, Types.VARCHAR, 400, false);
+  }
+
+  @Test
+  void execute_whenColumnIsAlreadyResized_shouldDoNothing() throws SQLException {
+    dbWith400LimitOnValueColumn.assertColumnDefinition(EXPECTED_TABLE_NAME, EXPECTED_COLUMN_NAME, Types.VARCHAR, 400, false);
+    underTestNoAction.execute();
+    dbWith400LimitOnValueColumn.assertColumnDefinition(EXPECTED_TABLE_NAME, EXPECTED_COLUMN_NAME, Types.VARCHAR, 400, false);
+  }
+
+
+}

server/sonar-db-migration/src/it/resources/org/sonar/server/platform/db/migration/version/v106/ResizeValueColumnInRuleTagsTableIT/schema.sql

@@ -0,0 +1,5 @@
+CREATE TABLE "RULE_TAGS"(
+    "RULE_UUID" CHARACTER VARYING(40) NOT NULL,
+    "IS_SYSTEM_TAG" BOOLEAN NOT NULL,
+    "VALUE" CHARACTER VARYING(40) NOT NULL
+);

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v104/CreateRuleTagsTable.java

@@ -32,11 +32,10 @@ public class CreateRuleTagsTable extends CreateTableChange {
 
   static final String RULE_TAGS_TABLE_NAME = "rule_tags";
 
-  static final String UUID_COLUMN_NAME = "uuid";
   static final String VALUE_COLUMN_NAME = "value";
   static final String IS_SYSTEM_TAG_COLUMN_NAME = "is_system_tag";
   static final String RULE_UUID_COLUMN_NAME = "rule_uuid";
-  static final int VALUE_COLUMN_SIZE = 40;
+  static final int VALUE_COLUMN_SIZE = 400;
 
   public CreateRuleTagsTable(Database db) {
     super(db, RULE_TAGS_TABLE_NAME);

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v104/PopulateRuleTagsTable.java

@@ -20,8 +20,11 @@
 package org.sonar.server.platform.db.migration.version.v104;
 
 import java.sql.SQLException;
+import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.stream.Collectors;
@@ -33,14 +36,14 @@
 public class PopulateRuleTagsTable extends DataChange {
 
   private static final String SELECT_QUERY = """
-   SELECT uuid, system_tags AS tag, 1 as is_system_tag
-    FROM rules
-    WHERE system_tags IS NOT NULL
-   UNION ALL
-   SELECT uuid, tags AS tag, 0 as is_system_tag
-    FROM rules
-    WHERE tags IS NOT NULL
-     """;
+    SELECT uuid, system_tags AS tag, 1 as is_system_tag
+     FROM rules
+     WHERE system_tags IS NOT NULL
+    UNION ALL
+    SELECT uuid, tags AS tag, 0 as is_system_tag
+     FROM rules
+     WHERE tags IS NOT NULL
+      """;
 
   private static final String INSERT_QUERY = """
     INSERT INTO rule_tags (rule_uuid, is_system_tag, value)
@@ -57,18 +60,55 @@ protected void execute(Context context) throws SQLException {
       return;
     }
 
-    List<PopulateRuleTagsTable.Tag> allTags = findAllTags(context);
+    List<Tags> allTags = findAllTags(context);
     if (allTags.isEmpty()) {
       return;
     }
+    allTags = removeDuplicatesForAllRule(allTags);
 
     Upsert insertTagsQuery = context.prepareUpsert(INSERT_QUERY);
-    for (PopulateRuleTagsTable.Tag tag : allTags) {
-      insertEveryTag(insertTagsQuery, tag.ruleUuid(), tag.values(), tag.isSystemTag());
+    for (Tags tags : allTags) {
+      insertEveryTag(insertTagsQuery, tags.ruleUuid(), tags.values(), tags.isSystemTag());
     }
     insertTagsQuery.execute().commit();
   }
 
+  /**
+   * System tags and custom tags can contain the same values. In this case, we keep only the system tag.
+   */
+  private static List<Tags> removeDuplicatesForAllRule(List<Tags> allTags) {
+    Map<String, List<Tags>> tagsByRuleUuid = allTags.stream().collect(Collectors.groupingBy(Tags::ruleUuid));
+    List<Tags> listWithoutDuplicates = new ArrayList<>();
+
+    for (Map.Entry<String, List<Tags>> entry : tagsByRuleUuid.entrySet()) {
+      listWithoutDuplicates.addAll(removeDuplicateForRule(entry.getValue()));
+    }
+    return listWithoutDuplicates;
+  }
+
+  private static List<Tags> removeDuplicateForRule(List<Tags> ruleTags) {
+    Optional<Tags> systemTags = ruleTags.stream().filter(Tags::isSystemTag).findFirst();
+    Optional<Tags> manualTags = ruleTags.stream().filter(t -> !t.isSystemTag()).findFirst();
+
+    if (systemTags.isEmpty()) {
+      return List.of(manualTags.orElseThrow());
+    } else if (manualTags.isEmpty()) {
+      return List.of(systemTags.orElseThrow());
+    } else {
+      Set<String> systemTagValues = systemTags.get().values();
+      Set<String> manualTagValues = manualTags.get().values();
+      Set<String> commonValues = new HashSet<>(systemTagValues);
+      commonValues.retainAll(manualTagValues);
+
+      if (commonValues.isEmpty()) {
+        return List.of(manualTags.orElseThrow(), systemTags.orElseThrow());
+      } else {
+        manualTagValues.removeAll(commonValues);
+        return List.of(systemTags.orElseThrow(), new Tags(manualTags.get().ruleUuid(), manualTagValues, false));
+      }
+    }
+  }
+
   private static void insertEveryTag(Upsert insertRuleTags, String ruleUuid, Set<String> values, boolean isSystemTag) throws SQLException {
     for (String tag : values) {
       insertRuleTags
@@ -79,9 +119,9 @@ private static void insertEveryTag(Upsert insertRuleTags, String ruleUuid, Set<S
     }
   }
 
-  private static List<PopulateRuleTagsTable.Tag> findAllTags(Context context) throws SQLException {
+  private static List<Tags> findAllTags(Context context) throws SQLException {
     return context.prepareSelect(SELECT_QUERY)
-      .list(r -> new PopulateRuleTagsTable.Tag(r.getString(1), parseTagString(r.getString(2)), r.getBoolean(3)));
+      .list(r -> new Tags(r.getString(1), parseTagString(r.getString(2)), r.getBoolean(3)));
   }
 
   private static boolean isTableAlreadyPopulated(Context context) throws SQLException {
@@ -97,7 +137,7 @@ private static Set<String> parseTagString(String tagString) {
       .collect(Collectors.toSet());
   }
 
-  private record Tag(String ruleUuid, Set<String> values, boolean isSystemTag) {
+  private record Tags(String ruleUuid, Set<String> values, boolean isSystemTag) {
   }
 
 }

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v106/DbVersion106.java

@@ -42,6 +42,7 @@ public class DbVersion106 implements DbVersion {
   public void addSteps(MigrationStepRegistry registry) {
     registry
       .add(10_6_000,"Add 'prioritized_rule' column to 'issues' table", AddPrioritizedRuleColumnToIssuesTable.class)
-      .add(10_6_001,"Add 'prioritized_rule' column to 'active_rules' table", AddPrioritizedRuleColumnToActiveRulesTable.class);
+      .add(10_6_001,"Add 'prioritized_rule' column to 'active_rules' table", AddPrioritizedRuleColumnToActiveRulesTable.class)
+      .add(10_6_002,"Ensure 'value' column is resized to 400 in 'rule_tags' table", ResizeValueColumnInRuleTagsTable.class);
   }
 }

server/sonar-db-migration/src/main/java/org/sonar/server/platform/db/migration/version/v106/ResizeValueColumnInRuleTagsTable.java

@@ -0,0 +1,46 @@
+/*
+ * SonarQube
+ * Copyright (C) 2009-2024 SonarSource SA
+ * mailto:info AT sonarsource DOT com
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 3 of the License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public License
+ * along with this program; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+ */
+package org.sonar.server.platform.db.migration.version.v106;
+
+import java.sql.SQLException;
+import org.sonar.db.Database;
+import org.sonar.server.platform.db.migration.def.VarcharColumnDef;
+import org.sonar.server.platform.db.migration.sql.AlterColumnsBuilder;
+import org.sonar.server.platform.db.migration.step.DdlChange;
+
+public class ResizeValueColumnInRuleTagsTable extends DdlChange {
+
+  private static final VarcharColumnDef columnDefinition = VarcharColumnDef.newVarcharColumnDefBuilder()
+    .setColumnName("value")
+    .setIsNullable(false)
+    .setLimit(400)
+    .build();
+
+  public ResizeValueColumnInRuleTagsTable(Database db) {
+    super(db);
+  }
+
+  @Override
+  public void execute(Context context) throws SQLException {
+    context.execute(new AlterColumnsBuilder(getDialect(), "rule_tags")
+      .updateColumn(columnDefinition)
+      .build());
+  }
+}