Merge branch 'master' into 3429-remove-webapps-sa

.ci/config/matrices.yaml

@@ -41,7 +41,6 @@ daily-stages:
   - 'sqlserver_2019'
   stages:
   - 'sql-scripts'
-  - 'upgrade-database'
   - 'instance-migration'
   - 'old-engine'
   - 'rolling-update'
@@ -54,7 +53,6 @@ sidetrack-stages:
   - 'db-unit'
   - 'db-unit-authorizations'
   - 'sql-scripts'
-  - 'upgrade-database'
   - 'instance-migration'
   - 'old-engine'
   - 'rolling-update'

.ci/config/stage-types.yaml

@@ -58,20 +58,6 @@ sql-scripts:
   - 'sqlserver'
   - 'postgresql'
   - 'cockroachdb'
-upgrade-database:
-  directory: 'qa/test-db-upgrade'
-  command: 'verify -Pupgrade-db,'
-  stash:
-    runtimeStash: true
-  labels:
-  - 'all-db'
-  - 'db2'
-  - 'mysql'
-  - 'oracle'
-  - 'mariadb'
-  - 'sqlserver'
-  - 'postgresql'
-  - 'cockroachdb'
 instance-migration:
   directory: 'qa/test-db-instance-migration'
   command: 'verify -Pinstance-migration,'

.ci/daily/Jenkinsfile

@@ -164,25 +164,6 @@ pipeline {
             ])
           }
         }
-        stage('UPGRADE-databases-h2') {
-          when {
-            expression {
-              cambpmWithLabels('h2', 'all-db')
-            }
-          }
-          steps {
-            cambpmConditionalRetry([
-              agentLabel: 'h2',
-              runSteps: {
-                cambpmRunMavenByStageType('upgrade-database', 'h2')
-              },
-              postFailure: {
-                cambpmPublishTestResult()
-                cambpmAddFailedStageType(failedStageTypes, 'upgrade-database')
-              }
-            ])
-          }
-        }
         stage('UPGRADE-instance-migration-h2') {
           when {
             expression {

distro/sql-script/src/test/java/org/camunda/bpm/sql/test/SqlScriptTest.java

@@ -20,19 +20,17 @@
 
 import java.io.IOException;
 import java.io.InputStream;
+import java.net.URI;
 import java.net.URISyntaxException;
 import java.nio.file.Paths;
 import java.util.Arrays;
 import java.util.List;
 import java.util.Properties;
 import java.util.SortedSet;
-
-import org.camunda.commons.utils.IoUtil;
-import org.junit.Before;
-import org.junit.Test;
-
+import java.util.stream.Collectors;
 import liquibase.Contexts;
 import liquibase.Liquibase;
+import liquibase.change.Change;
 import liquibase.change.core.SQLFileChange;
 import liquibase.changelog.ChangeSet;
 import liquibase.database.Database;
@@ -58,6 +56,10 @@
 import liquibase.snapshot.SnapshotGeneratorFactory;
 import liquibase.structure.DatabaseObject;
 import liquibase.structure.core.UniqueConstraint;
+import org.camunda.commons.utils.IoUtil;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
 
 public class SqlScriptTest {
 
@@ -78,6 +80,7 @@ public class SqlScriptTest {
   protected Database database;
   protected String databaseType;
   protected String projectVersion;
+  protected Liquibase liquibase;
 
   @Before
   public void setup() throws Exception {
@@ -87,49 +90,116 @@ public void setup() throws Exception {
 
     databaseType = properties.getProperty("database.type");
     projectVersion = properties.getProperty("project.version");
-  }
 
-  @Test
-  public void shouldEqualLiquibaseChangelogAndCreateScripts() throws Exception {
-    // given
     SnapshotParserFactory.getInstance().register(new DirectAccessSnapshotParser());
+
     database = getDatabase();
+    liquibase = getLiquibase();
+    liquibase.dropAll();
+  }
 
-    try (Liquibase liquibase = getLiquibase()) {
-      //   database cleared and set up with Liquibase changelog
+  @After
+  public void tearDown() throws Exception {
+    try {
       liquibase.dropAll();
+    } finally {
+      liquibase.close();
+    }
+  }
 
-      // execute the SQL scripts
-      executeSqlScript("create", "engine");
-      executeSqlScript("create", "identity");
+  @Test
+  public void shouldEqualLiquibaseChangelogAndCreateScripts() throws Exception {
+    // given
+    executeSqlScript("create", "engine");
+    executeSqlScript("create", "identity");
+    DatabaseSnapshot snapshotManualScripts = createCurrentDatabaseSnapshot();
+    liquibase.dropAll();
+
+    // when set up with Liquibase changelog
+    liquibase.update(new Contexts());
+
+    // then
+    DatabaseSnapshot snapshotLiquibaseChangelog = createCurrentDatabaseSnapshot();
+    Database currentDatabase = getDatabaseForSnapshot(snapshotManualScripts);
+    Database upgradedDatabase = getDatabaseForSnapshot(snapshotLiquibaseChangelog);
+    DiffResult diffResult = liquibase.diff(currentDatabase, upgradedDatabase, new CompareControl());
+    List<ChangeSet> changeSetsToApply = new DiffToChangeLog(diffResult, new CustomDiffOutputControl()).generateChangeSets();
+
+    assertThat(changeSetsToApply)
+      .withFailMessage("Liquibase database schema misses changes: %s", getChanges(changeSetsToApply))
+      .isEmpty();
+  }
 
-      //   snapshot created of the database for manual scripts
-      DatabaseSnapshot snapshotManualScripts = createCurrentDatabaseSnapshot();
-      //   database cleared and set up with Liquibase changelog
-      liquibase.dropAll();
+  @Test
+  public void shouldEqualOldUpgradedAndNewCreatedViaLiquibase() throws Exception {
+    try (Liquibase liquibaseOld = getLiquibase("scripts-old/")) {
+      // given
       liquibase.update(new Contexts());
-      //   snapshot created of the database for Liquibase changelog
-      DatabaseSnapshot snapshotLiquibaseChangelog = createCurrentDatabaseSnapshot();
+      DatabaseSnapshot snapshotCurrent = createCurrentDatabaseSnapshot();
+      liquibase.dropAll();
 
-      //   diff created for both snapshot
-      DiffResult diffResult =  liquibase.diff(getDatabaseForSnapshot(snapshotManualScripts),
-          getDatabaseForSnapshot(snapshotLiquibaseChangelog), new CompareControl());
+      // old changelog executed
+      liquibaseOld.update(new Contexts());
 
-      // when generating changes to apply between both databases
-      List<ChangeSet> changeSetsToApply = new DiffToChangeLog(diffResult, new CustomDiffOutputControl()).generateChangeSets();
+      // when new changelog executed afterward
+      liquibase.update(new Contexts());
 
       // then
-      assertThat(changeSetsToApply).isEmpty();
-    } finally {
-      database = getDatabase();
-      try (Liquibase liquibase = getLiquibase()){
-        liquibase.dropAll();
-      }
+      DatabaseSnapshot snapshotUpgraded = createCurrentDatabaseSnapshot();
+      Database currentDatabase = getDatabaseForSnapshot(snapshotCurrent);
+      Database upgradedDatabase = getDatabaseForSnapshot(snapshotUpgraded);
+      DiffResult diffResult = liquibase.diff(currentDatabase, upgradedDatabase, new CompareControl());
+      List<ChangeSet> changeSetsToApply = new DiffToChangeLog(diffResult, new DiffOutputControl()).generateChangeSets();
+
+      assertThat(changeSetsToApply)
+        .withFailMessage("Resulting upgraded database misses changes: %s", getChanges(changeSetsToApply))
+        .isEmpty();
     }
   }
 
-  protected Liquibase getLiquibase() throws URISyntaxException {
-    return new Liquibase("camunda-changelog.xml", getAccessorForChangelogDirectory(), database);
+  @Test
+  public void shouldEqualOldUpgradedAndNewCreatedViaScripts() throws Exception {
+    // given
+    String currentMajorMinor = properties.getProperty("current.majorminor");
+    String oldMajorMinor = properties.getProperty("old.majorminor");
+
+    executeSqlScript("create", "engine");
+    executeSqlScript("create", "identity");
+    DatabaseSnapshot snapshotCurrent = createCurrentDatabaseSnapshot();
+
+    liquibase.dropAll();
+
+    // old CREATE scripts executed
+    executeSqlScript("scripts-old/", "create", "engine_" + oldMajorMinor + ".0");
+    executeSqlScript("scripts-old/", "create", "identity_" + oldMajorMinor + ".0");
+
+    // when UPGRADE scripts executed
+    executeSqlScript("local-upgrade-test/", "upgrade", "engine_" + oldMajorMinor + "_patch");
+    executeSqlScript("local-upgrade-test/", "upgrade", "engine_" + oldMajorMinor + "_to_" + currentMajorMinor);
+    executeSqlScript("local-upgrade-test/", "upgrade", "engine_" + currentMajorMinor + "_patch");
+
+    // then
+    DatabaseSnapshot snapshotUpgraded = createCurrentDatabaseSnapshot();
+    Database currentDatabase = getDatabaseForSnapshot(snapshotCurrent);
+    Database upgradedDatabase = getDatabaseForSnapshot(snapshotUpgraded);
+    DiffResult diffResult = liquibase.diff(currentDatabase, upgradedDatabase, new CompareControl());
+    List<ChangeSet> changeSetsToApply = new DiffToChangeLog(diffResult, new DiffOutputControl()).generateChangeSets();
+
+    assertThat(changeSetsToApply)
+      .withFailMessage("Resulting upgraded database schema differs: %s", getChanges(changeSetsToApply))
+      .isEmpty();
+  }
+
+  protected void executeSqlScript(String sqlFolder, String sqlScript) throws LiquibaseException {
+    executeSqlScript("", sqlFolder, sqlScript + "_" + projectVersion);
+  }
+
+  protected void executeSqlScript(String baseDirectory, String sqlFolder, String sqlScript) throws LiquibaseException {
+    String scriptFileName = String.format("%ssql/%s/%s_%s.sql", baseDirectory, sqlFolder, databaseType, sqlScript);
+    String statements = IoUtil.inputStreamAsString(getClass().getClassLoader().getResourceAsStream(scriptFileName));
+    SQLFileChange sqlFileChange = new SQLFileChange();
+    sqlFileChange.setSql(statements);
+    database.execute(sqlFileChange.generateStatements(database), null);
   }
 
   protected Database getDatabase() throws DatabaseException {
@@ -141,28 +211,38 @@ protected Database getDatabase() throws DatabaseException {
         null, null, null, new ClassLoaderResourceAccessor());
   }
 
-  protected void executeSqlScript(String sqlFolder, String sqlScript) throws LiquibaseException {
-    String statements = IoUtil.inputStreamAsString(getClass().getClassLoader().getResourceAsStream(
-        String.format("sql/%s/%s_%s_%s.sql", sqlFolder, databaseType, sqlScript, projectVersion)));
-    SQLFileChange sqlFileChange = new SQLFileChange();
-    sqlFileChange.setSql(statements);
-    database.execute(sqlFileChange.generateStatements(database), null);
+  protected Liquibase getLiquibase() throws URISyntaxException {
+    return new Liquibase("camunda-changelog.xml", getAccessorForChangelogDirectory(""), database);
+  }
+
+  protected Liquibase getLiquibase(String baseDirectory) throws URISyntaxException {
+    return new Liquibase("camunda-changelog.xml", getAccessorForChangelogDirectory(baseDirectory), database);
   }
 
-  protected FileSystemResourceAccessor getAccessorForChangelogDirectory() throws URISyntaxException {
-    return new FileSystemResourceAccessor(Paths.get(getClass().getClassLoader().getResource("sql/liquibase").toURI()).toAbsolutePath().toFile());
+  protected FileSystemResourceAccessor getAccessorForChangelogDirectory(String baseDirectory) throws URISyntaxException {
+    URI changelogUri = getClass().getClassLoader().getResource(baseDirectory + "sql/liquibase").toURI();
+    return new FileSystemResourceAccessor(Paths.get(changelogUri).toAbsolutePath().toFile());
   }
 
   protected DatabaseSnapshot createCurrentDatabaseSnapshot() throws Exception {
-    return SnapshotGeneratorFactory.getInstance().createSnapshot(database.getDefaultSchema(), database, new SnapshotControl(database));
+    return SnapshotGeneratorFactory.getInstance()
+        .createSnapshot(database.getDefaultSchema(), database, new SnapshotControl(database));
   }
 
   protected Database getDatabaseForSnapshot(DatabaseSnapshot snapshot) throws Exception {
     String offlineDatabaseUrl = "offline:" + databaseType + "?snapshot=foo";
-    OfflineConnection offlineDatabaseConnection = new OfflineConnection(offlineDatabaseUrl, new SnapshotResourceAccessor(snapshot));
+    SnapshotResourceAccessor snapshotAccessor = new SnapshotResourceAccessor(snapshot);
+    OfflineConnection offlineDatabaseConnection = new OfflineConnection(offlineDatabaseUrl, snapshotAccessor);
     return DatabaseFactory.getInstance().findCorrectDatabaseImplementation(offlineDatabaseConnection);
   }
 
+  protected List<String> getChanges(List<ChangeSet> changeSetsToApply) {
+    return changeSetsToApply.stream()
+        .flatMap(cs -> cs.getChanges().stream())
+        .map(Change::getDescription)
+        .collect(Collectors.toList());
+  }
+
   protected static class DirectAccessSnapshotParser implements SnapshotParser {
 
     @Override
@@ -200,7 +280,10 @@ public InputStream openStream(String relativeTo, String streamPath) throws IOExc
     }
 
     @Override
-    public SortedSet<String> list(String relativeTo, String path, boolean recursive, boolean includeFiles,
+    public SortedSet<String> list(String relativeTo,
+        String path,
+        boolean recursive,
+        boolean includeFiles,
         boolean includeDirectories) throws IOException {
       return null;
     }
@@ -224,7 +307,8 @@ public CustomDiffOutputControl() {
     private static class IgnoreUniqueConstraintsChangeFilter implements ObjectChangeFilter {
 
       @Override
-      public boolean includeUnexpected(DatabaseObject object, Database referenceDatabase, Database comparisionDatabase) {
+      public boolean includeUnexpected(DatabaseObject object, Database referenceDatabase,
+          Database comparisionDatabase) {
         return include(object);
       }
 
@@ -234,7 +318,10 @@ public boolean includeMissing(DatabaseObject object, Database referenceDatabase,
       }
 
       @Override
-      public boolean includeChanged(DatabaseObject object, ObjectDifferences differences, Database referenceDatabase, Database comparisionDatabase) {
+      public boolean includeChanged(DatabaseObject object,
+          ObjectDifferences differences,
+          Database referenceDatabase,
+          Database comparisionDatabase) {
         return include(object);
       }
 

distro/sql-script/src/test/resources/properties-from-pom.properties

@@ -3,4 +3,6 @@ database.username=${database.username}
 database.password=${database.password}
 database.driver=${database.driver}
 database.type=${database.type}
-project.version=${project.version}
+project.version=${project.version}
+current.majorminor=${camunda.current.majorVersion}.${camunda.current.minorVersion}
+old.majorminor=${camunda.old.majorVersion}.${camunda.old.minorVersion}

qa/pom.xml

@@ -78,7 +78,6 @@
         <module>integration-tests-webapps</module>
         <module>integration-tests-engine</module>
         <module>integration-tests-engine-jakarta</module>
-        <module>test-db-upgrade</module>
         <module>test-db-util</module>
         <module>test-db-instance-migration</module>
         <module>test-db-rolling-update</module>
@@ -146,13 +145,6 @@
       </modules>
     </profile>
 
-    <profile>
-      <id>upgrade-db</id>
-      <modules>
-        <module>test-db-upgrade</module>
-      </modules>
-    </profile>
-
     <profile>
       <id>instance-migration</id>
       <modules>