From eb69e19ae379ce86dcfb2a84fc9c0ab135a64527 Mon Sep 17 00:00:00 2001 From: Boris Skert Date: Wed, 24 Apr 2019 09:55:05 +0200 Subject: [PATCH] [issue/#7] Omit already imported import-files (#10) * [issue/#7] Omit keycloak import run if checksum is same * [issue/#7] Provide opt-in mechanism to force imports --- README.md | 2 + .../keycloak/config/model/RealmImport.java | 12 +++ .../service/KeycloakImportProvider.java | 42 ++++++++--- .../config/service/RealmImportService.java | 44 ++++++++++- .../service/checksum/ChecksumService.java | 35 +++++++++ .../keycloak/config/ImportSimpleRealmIT.java | 27 +++++++ .../service/checksum/ChecksumServiceTest.java | 73 +++++++++++++++++++ ..._update_simple-realm_with_same_config.json | 4 + docker-compose.yml | 1 + 9 files changed, 227 insertions(+), 13 deletions(-) create mode 100644 config-cli/src/main/java/de/adorsys/keycloak/config/service/checksum/ChecksumService.java create mode 100644 config-cli/src/test/java/de/adorsys/keycloak/config/service/checksum/ChecksumServiceTest.java create mode 100644 config-cli/src/test/resources/import-files/simple-realm/0.1_update_simple-realm_with_same_config.json diff --git a/README.md b/README.md index 046736ac4..89390d6f6 100644 --- a/README.md +++ b/README.md @@ -69,6 +69,7 @@ $ docker run -e KEYCLOAK_URL=http://:8080 \ -e KEYCLOAK_ADMIN_PASSWORD= \ -e KEYCLOAK_ENCRYPT_ADMIN_PASSWORD= \ -e WAIT_TIME_IN_SECONDS=120 \ + -e IMPORT_FORCE=false \ -e JWKS_CONNECT_TIMEOUT=250 \ -e JWKS_READ_TIMEOUT=250 \ -e JWKS_SIZE_LIMIT=51200 \ @@ -109,6 +110,7 @@ services: - KEYCLOAK_ADMIN_PASSWORD= - KEYCLOAK_ENCRYPT_ADMIN_PASSWORD= - WAIT_TIME_IN_SECONDS=120 + - IMPORT_FORCE=false - JWKS_CONNECT_TIMEOUT=250 - JWKS_READ_TIMEOUT=250 - JWKS_SIZE_LIMIT=51200 diff --git a/config-cli/src/main/java/de/adorsys/keycloak/config/model/RealmImport.java b/config-cli/src/main/java/de/adorsys/keycloak/config/model/RealmImport.java index 7de26be7b..aab5de50a 100644 --- a/config-cli/src/main/java/de/adorsys/keycloak/config/model/RealmImport.java +++ b/config-cli/src/main/java/de/adorsys/keycloak/config/model/RealmImport.java @@ -21,6 +21,8 @@ public class RealmImport extends RealmRepresentation { private RolesImport rolesImport = new RolesImport(); + private String checksum; + public Optional getCustomImport() { return Optional.ofNullable(customImport); } @@ -143,4 +145,14 @@ private List getNonTopLevelFlows() { .filter(f -> !f.isTopLevel()) .collect(Collectors.toList()); } + + @JsonIgnore + public String getChecksum() { + return checksum; + } + + @JsonIgnore + public void setChecksum(String checksum) { + this.checksum = checksum; + } } diff --git a/config-cli/src/main/java/de/adorsys/keycloak/config/service/KeycloakImportProvider.java b/config-cli/src/main/java/de/adorsys/keycloak/config/service/KeycloakImportProvider.java index a087f5e2f..e050a7102 100644 --- a/config-cli/src/main/java/de/adorsys/keycloak/config/service/KeycloakImportProvider.java +++ b/config-cli/src/main/java/de/adorsys/keycloak/config/service/KeycloakImportProvider.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import de.adorsys.keycloak.config.model.KeycloakImport; import de.adorsys.keycloak.config.model.RealmImport; +import de.adorsys.keycloak.config.service.checksum.ChecksumService; import org.apache.logging.log4j.util.Strings; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.beans.factory.annotation.Value; @@ -10,6 +11,7 @@ import java.io.File; import java.io.IOException; +import java.nio.file.Files; import java.util.HashMap; import java.util.Map; @@ -24,16 +26,22 @@ public class KeycloakImportProvider { private final ObjectMapper objectMapper; - public KeycloakImportProvider(@Qualifier("json") ObjectMapper objectMapper) { + private final ChecksumService checksumService; + + public KeycloakImportProvider( + @Qualifier("json") ObjectMapper objectMapper, + ChecksumService checksumService + ) { this.objectMapper = objectMapper; + this.checksumService = checksumService; } public KeycloakImport get() { KeycloakImport keycloakImport; - if(Strings.isNotBlank(importFilePath)) { + if (Strings.isNotBlank(importFilePath)) { keycloakImport = readFromFile(importFilePath); - } else if(Strings.isNotBlank(importDirectoryPath)) { + } else if (Strings.isNotBlank(importDirectoryPath)) { keycloakImport = readFromDirectory(importDirectoryPath); } else { throw new RuntimeException("Either 'import.path' or 'import.file' has to be defined"); @@ -45,10 +53,10 @@ public KeycloakImport get() { private KeycloakImport readFromFile(String filename) { try { File configFile = new File(filename); - if(!configFile.exists()) { + if (!configFile.exists()) { throw new RuntimeException("Is not existing: " + filename); } - if(configFile.isDirectory()) { + if (configFile.isDirectory()) { throw new RuntimeException("Is a directory: " + filename); } @@ -61,10 +69,10 @@ private KeycloakImport readFromFile(String filename) { private KeycloakImport readFromDirectory(String filename) { try { File configDirectory = new File(filename); - if(!configDirectory.exists()) { + if (!configDirectory.exists()) { throw new RuntimeException("Is not existing: " + filename); } - if(!configDirectory.isDirectory()) { + if (!configDirectory.isDirectory()) { throw new RuntimeException("Is not a directory: " + filename); } @@ -78,10 +86,10 @@ public KeycloakImport readRealmImportsFromDirectory(File importFilesDirectory) t Map realmImports = new HashMap<>(); File[] files = importFilesDirectory.listFiles(); - if(files != null) { + if (files != null) { for (File importFile : files) { - if(!importFile.isDirectory()) { - RealmImport realmImport = objectMapper.readValue(importFile, RealmImport.class); + if (!importFile.isDirectory()) { + RealmImport realmImport = readRealmImport(importFile); realmImports.put(importFile.getName(), realmImport); } } @@ -93,11 +101,21 @@ public KeycloakImport readRealmImportsFromDirectory(File importFilesDirectory) t private KeycloakImport readRealmImportFromFile(File importFile) throws IOException { Map realmImports = new HashMap<>(); - if(!importFile.isDirectory()) { - RealmImport realmImport = objectMapper.readValue(importFile, RealmImport.class); + if (!importFile.isDirectory()) { + RealmImport realmImport = readRealmImport(importFile); realmImports.put(importFile.getName(), realmImport); } return new KeycloakImport(realmImports); } + + private RealmImport readRealmImport(File importFile) throws IOException { + byte[] importFileInBytes = Files.readAllBytes(importFile.toPath()); + String checksum = checksumService.checksum(importFileInBytes); + + RealmImport realmImport = objectMapper.readValue(importFile, RealmImport.class); + realmImport.setChecksum(checksum); + + return realmImport; + } } diff --git a/config-cli/src/main/java/de/adorsys/keycloak/config/service/RealmImportService.java b/config-cli/src/main/java/de/adorsys/keycloak/config/service/RealmImportService.java index 5023761f1..67cda0fd9 100644 --- a/config-cli/src/main/java/de/adorsys/keycloak/config/service/RealmImportService.java +++ b/config-cli/src/main/java/de/adorsys/keycloak/config/service/RealmImportService.java @@ -8,14 +8,18 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.stereotype.Service; import java.util.List; +import java.util.Map; @Service public class RealmImportService { private static final Logger logger = LoggerFactory.getLogger(RealmImportService.class); + private static final String REALM_CHECKSUM_ATTRIBUTE_KEY = "de.adorsys.keycloak.config.import-checksum"; + private final String[] ignoredPropertiesForCreation = new String[]{ "users", "browserFlow", @@ -63,6 +67,10 @@ public class RealmImportService { private final RequiredActionsImportService requiredActionsImportService; private final CustomImportService customImportService; + @Value("${import.force:#{false}}") + private Boolean forceImport; + + @Autowired public RealmImportService( KeycloakProvider keycloakProvider, @@ -90,7 +98,7 @@ public void doImport(RealmImport realmImport) { boolean realmExists = realmRepository.exists(realmImport.getRealm()); if(realmExists) { - updateRealm(realmImport); + updateRealmIfNecessary(realmImport); } else { createRealm(realmImport); } @@ -110,12 +118,26 @@ private void createRealm(RealmImport realmImport) { setupFlows(realmImport); importComponents(realmImport); customImportService.doImport(realmImport); + setupImportChecksum(realmImport); } private void importComponents(RealmImport realmImport) { componentImportService.doImport(realmImport); } + private void updateRealmIfNecessary(RealmImport realmImport) { + if(forceImport || hasToBeUpdated(realmImport)) { + updateRealm(realmImport); + } else { + if(logger.isDebugEnabled()) + logger.debug( + "No need to update realm '{}', import checksum same: '{}'", + realmImport.getRealm(), + realmImport.getChecksum() + ); + } + } + private void updateRealm(RealmImport realmImport) { if(logger.isDebugEnabled()) logger.debug("Updating realm '{}'...", realmImport.getRealm()); @@ -130,6 +152,7 @@ private void updateRealm(RealmImport realmImport) { setupFlows(realmImport); importComponents(realmImport); customImportService.doImport(realmImport); + setupImportChecksum(realmImport); } private void importRequiredActions(RealmImport realmImport) { @@ -152,4 +175,23 @@ private void setupFlows(RealmImport realmImport) { realmRepository.update(realmToUpdate); } + + private boolean hasToBeUpdated(RealmImport realmImport) { + RealmRepresentation existingRealm = realmRepository.get(realmImport.getRealm()); + Map customAttributes = existingRealm.getAttributes(); + String readChecksum = customAttributes.get(REALM_CHECKSUM_ATTRIBUTE_KEY); + + return !realmImport.getChecksum().equals(readChecksum); + } + + private void setupImportChecksum(RealmImport realmImport) { + RealmRepresentation existingRealm = realmRepository.get(realmImport.getRealm()); + Map customAttributes = existingRealm.getAttributes(); + + String importChecksum = realmImport.getChecksum(); + customAttributes.put(REALM_CHECKSUM_ATTRIBUTE_KEY, importChecksum); + realmRepository.update(existingRealm); + + if(logger.isDebugEnabled()) logger.debug("Updated import checksum of realm '{}' to '{}'", realmImport.getRealm(), importChecksum); + } } diff --git a/config-cli/src/main/java/de/adorsys/keycloak/config/service/checksum/ChecksumService.java b/config-cli/src/main/java/de/adorsys/keycloak/config/service/checksum/ChecksumService.java new file mode 100644 index 000000000..f1b76ca77 --- /dev/null +++ b/config-cli/src/main/java/de/adorsys/keycloak/config/service/checksum/ChecksumService.java @@ -0,0 +1,35 @@ +package de.adorsys.keycloak.config.service.checksum; + +import org.bouncycastle.jcajce.provider.digest.SHA3; +import org.bouncycastle.util.encoders.Hex; +import org.springframework.stereotype.Service; + +import java.security.MessageDigest; + +@Service +public class ChecksumService { + + private MessageDigest digest = new SHA3.Digest512(); + + public String checksum(String text) { + if (text == null) { + throw new IllegalArgumentException("Cannot calculate checksum of null"); + } + + byte[] textInBytes = text.getBytes(); + return calculateSha3Checksum(textInBytes); + } + + public String checksum(byte[] textInBytes) { + if (textInBytes == null) { + throw new IllegalArgumentException("Cannot calculate checksum of null"); + } + + return calculateSha3Checksum(textInBytes); + } + + private String calculateSha3Checksum(byte[] textInBytes) { + byte[] shaInBytes = this.digest.digest(textInBytes); + return Hex.toHexString(shaInBytes); + } +} diff --git a/config-cli/src/test/java/de/adorsys/keycloak/config/ImportSimpleRealmIT.java b/config-cli/src/test/java/de/adorsys/keycloak/config/ImportSimpleRealmIT.java index 81c24b6e2..b2cc2cbb8 100644 --- a/config-cli/src/test/java/de/adorsys/keycloak/config/ImportSimpleRealmIT.java +++ b/config-cli/src/test/java/de/adorsys/keycloak/config/ImportSimpleRealmIT.java @@ -66,6 +66,7 @@ public void shouldReadImports() { @Test public void integrationTests() throws Exception { shouldCreateSimpleRealm(); + shouldNotUpdateSimpleRealm(); shouldUpdateSimpleRealm(); shouldCreateSimpleRealmWithLoginTheme(); } @@ -78,6 +79,24 @@ private void shouldCreateSimpleRealm() throws Exception { assertThat(createdRealm.getRealm(), is(REALM_NAME)); assertThat(createdRealm.isEnabled(), is(true)); assertThat(createdRealm.getLoginTheme(), is(nullValue())); + assertThat( + createdRealm.getAttributes().get("de.adorsys.keycloak.config.import-checksum"), + is("3796660d3087308ee757d9d86e14dd6e6fe4bfd66cc1435851ff2f5c6fa432c5991b3042f95c4f11238e1dfb81676ae2a00bde0bbad17c1f66ef530841df2e66") + ); + } + + private void shouldNotUpdateSimpleRealm() throws Exception { + doImport("0.1_update_simple-realm_with_same_config.json"); + + RealmRepresentation createdRealm = keycloakProvider.get().realm(REALM_NAME).toRepresentation(); + + assertThat(createdRealm.getRealm(), is(REALM_NAME)); + assertThat(createdRealm.isEnabled(), is(true)); + assertThat(createdRealm.getLoginTheme(), is(nullValue())); + assertThat( + createdRealm.getAttributes().get("de.adorsys.keycloak.config.import-checksum"), + is("3796660d3087308ee757d9d86e14dd6e6fe4bfd66cc1435851ff2f5c6fa432c5991b3042f95c4f11238e1dfb81676ae2a00bde0bbad17c1f66ef530841df2e66") + ); } private void shouldUpdateSimpleRealm() throws Exception { @@ -88,6 +107,10 @@ private void shouldUpdateSimpleRealm() throws Exception { assertThat(updatedRealm.getRealm(), is(REALM_NAME)); assertThat(updatedRealm.isEnabled(), is(true)); assertThat(updatedRealm.getLoginTheme(), is("moped")); + assertThat( + updatedRealm.getAttributes().get("de.adorsys.keycloak.config.import-checksum"), + is("d3913c179bf6d1ed1afbc2580207f3d7d78efed3ef13f9e12dea3afd5c28e9b307dd930fecfcc100038e540d1e23dc5b5c74d0321a410c7ba330e9dbf9d4211c") + ); } private void shouldCreateSimpleRealmWithLoginTheme() throws Exception { @@ -98,6 +121,10 @@ private void shouldCreateSimpleRealmWithLoginTheme() throws Exception { assertThat(createdRealm.getRealm(), is("simpleWithLoginTheme")); assertThat(createdRealm.isEnabled(), is(true)); assertThat(createdRealm.getLoginTheme(), is("moped")); + assertThat( + createdRealm.getAttributes().get("de.adorsys.keycloak.config.import-checksum"), + is("5d75698bacb06b1779e2b303069266664d63eec9c52038e2e6ae930bfc6e33ec7e7493b067ee0253e73a6b19cdf8905fd75cc6bb394ca333d32c784063aa65c8") + ); } private void doImport(String realmImport) { diff --git a/config-cli/src/test/java/de/adorsys/keycloak/config/service/checksum/ChecksumServiceTest.java b/config-cli/src/test/java/de/adorsys/keycloak/config/service/checksum/ChecksumServiceTest.java new file mode 100644 index 000000000..be01f73e7 --- /dev/null +++ b/config-cli/src/test/java/de/adorsys/keycloak/config/service/checksum/ChecksumServiceTest.java @@ -0,0 +1,73 @@ +package de.adorsys.keycloak.config.service.checksum; + +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import static com.googlecode.catchexception.CatchException.catchException; +import static com.googlecode.catchexception.CatchException.caughtException; +import static org.hamcrest.core.AllOf.allOf; +import static org.hamcrest.core.Is.is; +import static org.hamcrest.core.IsEqual.equalTo; +import static org.hamcrest.core.IsInstanceOf.instanceOf; +import static org.junit.Assert.assertThat; + +public class ChecksumServiceTest { + + private ChecksumService checksumService; + + @Before + public void setup() throws Exception { + checksumService = new ChecksumService(); + } + + @Test + public void shouldThrowOnNullString() throws Exception { + String nullString = null; + + catchException(checksumService).checksum(nullString); + + Assert.assertThat(caughtException(), + allOf( + instanceOf(IllegalArgumentException.class) + ) + ); + } + + @Test + public void shouldThrowOnNullBytes() throws Exception { + byte[] nullBytes = null; + + catchException(checksumService).checksum(nullBytes); + + Assert.assertThat(caughtException(), + allOf( + instanceOf(IllegalArgumentException.class) + ) + ); + } + + @Test + public void shouldReturnChecksumForEmptyString() throws Exception { + String checksum = checksumService.checksum(""); + assertThat(checksum, is(equalTo("a69f73cca23a9ac5c8b567dc185a756e97c982164fe25859e0d1dcc1475c80a615b2123af1f5f94c11e3e9402c3ac558f500199d95b6d3e301758586281dcd26"))); + } + + @Test + public void shouldReturnChecksumForABC() throws Exception { + String checksum = checksumService.checksum("ABC"); + assertThat(checksum, is(equalTo("077aa33882b1aaf06da41c7ed3b6a40d7128dee23505ca2689c47637111c4701645fabc5ee1b9dcd039231d2d086bff9819ce2da8647432a73966494dd1a77ad"))); + } + + @Test + public void shouldReturnChecksumForABCasBytes() throws Exception { + String checksum = checksumService.checksum(new byte[]{65, 66, 67}); + assertThat(checksum, is(equalTo("077aa33882b1aaf06da41c7ed3b6a40d7128dee23505ca2689c47637111c4701645fabc5ee1b9dcd039231d2d086bff9819ce2da8647432a73966494dd1a77ad"))); + } + + @Test + public void shouldReturnChecksumForJson() throws Exception { + String checksum = checksumService.checksum("{\"property\":\"value\"}"); + assertThat(checksum, is(equalTo("118dd3237b94e86dc939bf28cdfbb24265101e754178c29b80f46efcaedc84aa5c2c9711a5b6438389c87f9f0ba0a2f105ec272412b69bcbeeba8eb96cfb7771"))); + } +} diff --git a/config-cli/src/test/resources/import-files/simple-realm/0.1_update_simple-realm_with_same_config.json b/config-cli/src/test/resources/import-files/simple-realm/0.1_update_simple-realm_with_same_config.json new file mode 100644 index 000000000..8733f6afd --- /dev/null +++ b/config-cli/src/test/resources/import-files/simple-realm/0.1_update_simple-realm_with_same_config.json @@ -0,0 +1,4 @@ +{ + "enabled": true, + "realm": "simple" +} diff --git a/docker-compose.yml b/docker-compose.yml index a279b9fb2..8bcec4e20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,4 +27,5 @@ services: - KEYCLOAK_ADMIN_PASSWORD=admin123 - WAIT_TIME_IN_SECONDS=120 - SPRING_PROFILES_INCLUDE=debug + - IMPORT_FORCE=false command: config-cli