CSIRO Extra test rules and code changes

api/src/main/java/org/ihtsdo/rvf/controller/AssertionGroupController.java

@@ -8,6 +8,7 @@
 import java.util.Set;
 
 import javax.servlet.http.HttpServletResponse;
+
 import org.ihtsdo.rvf.entity.Assertion;
 import org.ihtsdo.rvf.entity.AssertionGroup;
 import org.ihtsdo.rvf.execution.service.config.MysqlExecutionConfig;
@@ -41,10 +42,10 @@ public class AssertionGroupController {
 
 	@Autowired
 	private AssertionService assertionService;
-	
+
 	@Autowired
 	private AssertionGroupRepository assertionGroupRepository;
-	
+
 	@Autowired
 	private AssertionHelper assertionHelper;
 	private final ObjectMapper objectMapper = new ObjectMapper();
@@ -68,7 +69,7 @@ public List<AssertionGroup> getGroups() {
 	@ApiOperation(value = "Get all assertions for a given assertion group", notes = "Retrieves all assertions for a given assertion group identified by the group id.")
 	public List<Assertion> getAssertionsForGroup(@ApiParam(value = "Assertion group id") @PathVariable final Long id) {
 
-		AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		AssertionGroup group = assertionGroupRepository.getOne(id);
 		return new ArrayList<>(group.getAssertions());
 	}
 
@@ -79,7 +80,7 @@ public AssertionGroup addAssertionsToGroup(@PathVariable final Long id,
 			@RequestBody(required = false) final List<String> assertionsList,
 			final HttpServletResponse response) {
 
-		final AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		final AssertionGroup group = assertionGroupRepository.getOne(id);
 		// Do we have anything to add?
 		if (assertionsList == null || assertionsList.size() == 0) {
 			response.setStatus(HttpServletResponse.SC_NOT_MODIFIED);
@@ -101,7 +102,7 @@ public AssertionGroup addAssertionsToGroup(@PathVariable final Long id,
 			+ " This api may only be used when user desires to add all assertion found in the system to an assertion group"
 			+ " otherwise use {id}/assertions api as post call.")
 	public AssertionGroup addAllAssertions(@PathVariable final Long id) {
-		AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		AssertionGroup group = assertionGroupRepository.getOne(id);
 		List<Assertion> assertionList = assertionService.findAll();
 		group.setAssertions(new HashSet<>(assertionList));
 		return assertionGroupRepository.save(group);
@@ -114,7 +115,7 @@ public AssertionGroup addAllAssertions(@PathVariable final Long id) {
 	public AssertionGroup removeAssertionsFromGroup(
 			@PathVariable final Long id,
 			@ApiParam(value = "Only assertion id is required") @RequestBody final List<Assertion> assertions) {
-		AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		AssertionGroup group = assertionGroupRepository.getOne(id);
 		for (final Assertion assertion : group.getAssertions()) {
 			group = assertionService.removeAssertionFromGroup(assertion, group);
 		}
@@ -128,10 +129,10 @@ public AssertionGroup removeAssertionsFromGroup(
 	public AssertionGroup setAsAssertionsInGroup(@PathVariable final Long id,
 			@RequestBody(required = false) final Set<Assertion> assertions) {
 
-		final AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		final AssertionGroup group = assertionGroupRepository.getOne(id);
 		// replace all existing assertions with current list
 		group.setAssertions(assertions);
-		return (AssertionGroup) assertionGroupRepository.save(group);
+		return assertionGroupRepository.save(group);
 	}
 
 	@RequestMapping(value = "{id}", method = RequestMethod.GET)
@@ -142,15 +143,15 @@ public AssertionGroup getAssertionGroup(@PathVariable final Long id) {
 		if (!assertionGroupRepository.existsById(id)) {
 			throw new EntityNotFoundException(id);
 		}
-		return (AssertionGroup) assertionGroupRepository.getOne(id);
+		return assertionGroupRepository.getOne(id);
 	}
 
 	@RequestMapping(value = "{id}", method = RequestMethod.DELETE)
 	@ResponseBody
 	@ResponseStatus(HttpStatus.OK)
 	@ApiOperation(value = "Delete an assertion group", notes = "Deletes an assertion group from the system")
 	public AssertionGroup deleteAssertionGroup(@ApiParam(value="Assertion group id") @PathVariable final Long id) {
-		final AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		final AssertionGroup group = assertionGroupRepository.getOne(id);
 		group.removeAllAssertionsFromGroup();
 		assertionGroupRepository.delete(group);
 		return group;
@@ -172,7 +173,7 @@ public AssertionGroup createAssertionGroupWithName(@RequestParam final String na
 	@ApiOperation(value = "Update an assertion group", notes = "Updates the group name for the assertion group identified by the group id.")
 	public AssertionGroup updateAssertionGroup(@PathVariable final Long id,
 			@ApiParam(value = "Assertion group name") @RequestParam final String name) {
-		AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		AssertionGroup group = assertionGroupRepository.getOne(id);
 		group.setName(name);
 		return assertionGroupRepository.save(group);
 	}
@@ -184,9 +185,9 @@ public AssertionGroup updateAssertionGroup(@PathVariable final Long id,
 	public Map<String, Object> executeAssertions(@ApiParam(value="Assertion group id")@PathVariable final Long id,
 			@ApiParam(value="Unique number") @RequestParam final Long runId,
 			@ApiParam(value="Prospective version") @RequestParam final String prospectiveReleaseVersion,
-			@ApiParam(value="Previous release version") @RequestParam final String previousReleaseVersion) {
+			@ApiParam(value="Previous release version", required = false) @RequestParam final String previousReleaseVersion) {
 
-		AssertionGroup group = (AssertionGroup) assertionGroupRepository.getOne(id);
+		AssertionGroup group = assertionGroupRepository.getOne(id);
 		MysqlExecutionConfig config = new MysqlExecutionConfig(runId);
 		config.setPreviousVersion(previousReleaseVersion);
 		config.setProspectiveVersion(prospectiveReleaseVersion);

docker-compose.yml

@@ -22,7 +22,7 @@ services:
     volumes:
       - mysql:/var/lib/mysql
     command:
-        mysqld --sql_mode="NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES"
+        mysqld --sql_mode="NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES" --lower_case_table_names=1
   rvf:
     image: snomedinternational/release-validation-framework:latest
     container_name: rvf

execution-service/src/main/java/org/ihtsdo/rvf/execution/service/AssertionExecutionService.java

@@ -27,6 +27,7 @@
 import org.ihtsdo.rvf.entity.Test;
 import org.ihtsdo.rvf.entity.TestRunItem;
 import org.ihtsdo.rvf.execution.service.config.MysqlExecutionConfig;
+import org.ihtsdo.rvf.importer.AssertionGroupImporter.ProductName;
 import org.ihtsdo.rvf.service.AssertionService;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
@@ -49,7 +50,7 @@ public class AssertionExecutionService {
 	private String deltaTableSuffix = "d";
 	private String snapshotTableSuffix = "s";
 	private String fullTableSuffix = "f";
-	
+
 	private ExecutorService executorService = Executors.newCachedThreadPool();
 
 	private final Logger logger = LoggerFactory.getLogger(AssertionExecutionService.class);
@@ -88,9 +89,9 @@ public Collection<TestRunItem> executeAssertions(final Collection<Assertion> ass
 
 		return items;
 	}
-	
+
 public List<TestRunItem> executeAssertionsConcurrently(List<Assertion> assertions, final MysqlExecutionConfig executionConfig) {
-		
+
 		final List<Future<Collection<TestRunItem>>> concurrentTasks = new ArrayList<>();
 		final List<TestRunItem> results = new ArrayList<>();
 		int counter = 1;
@@ -116,7 +117,7 @@ public Collection<TestRunItem> call() throws Exception {
 			}
 			counter++;
 		}
-		
+
 		// Wait for all concurrent tasks to finish
 		for (final Future<Collection<TestRunItem>> concurrentTask : concurrentTasks) {
 			try {
@@ -165,12 +166,12 @@ public TestRunItem executeTest(final Assertion assertion, final Test test, final
 				logger.warn("Failed to excute command {},Nested exception is : " + e.fillInStackTrace(), command.toString());
 				runItem.setFailureMessage("Error executing SQL command object Nested exception : " + e.fillInStackTrace());
 				return runItem;
-			} 
+			}
 		} else {
 			runItem.setFailureMessage("Test does not have any associated execution command:" + test);
 			return runItem;
 		}
-		
+
 		try {
 			long extractTimeStart = System.currentTimeMillis();
 			extractTestResult(assertion, runItem, config);
@@ -249,13 +250,17 @@ private List<String> transformSql(String[] parts, Assertion assertion, MysqlExec
 		List<String> result = new ArrayList<>();
 		String defaultCatalog = dataSource.getDefaultCatalog();
 		String prospectiveSchema = config.getProspectiveVersion();
+		final String[] nameParts = config.getProspectiveVersion().split("_");
+		String moduleId = (nameParts.length >= 2 ? ProductName.toModuleId(nameParts[1]) : "NOT_SUPPLIED");
+		String version = (nameParts.length >= 3 ? nameParts[2] : "NOT_SUPPLIED");
+
 		String previousReleaseSchema = config.getPreviousVersion();
-		
+
 		//We need both these schemas to exist
 		if (prospectiveSchema == null) {
 			throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + prospectiveSchema);
 		}
-		
+
 		if (config.isReleaseValidation() && !config.isFirstTimeRelease() && previousReleaseSchema == null) {
 			throw new ConfigurationException (FAILED_TO_FIND_RVF_DB_SCHEMA + previousReleaseSchema);
 		}
@@ -267,6 +272,8 @@ private List<String> transformSql(String[] parts, Assertion assertion, MysqlExec
 			// replace all substitutions for exec
 			part = part.replaceAll("<RUNID>", String.valueOf(config.getExecutionId()));
 			part = part.replaceAll("<ASSERTIONUUID>", String.valueOf(assertion.getAssertionId()));
+			part = part.replaceAll("<MODULEID>", moduleId);
+			part = part.replaceAll("<VERSION>", version);
 			// watch out for any 's that users might have introduced
 			part = part.replaceAll("qa_result", defaultCatalog+ "." + qaResulTableName);
 			part = part.replaceAll("<PROSPECTIVE>", prospectiveSchema);
@@ -315,7 +322,7 @@ private void extractTestResult(final Assertion assertion, final TestRunItem runI
 					}
 				}
 			}
-			
+
 			if ( counter < config.getFailureExportMax() ) {
 				runItem.setFailureCount(counter);
 			} else {

execution-service/src/main/java/org/ihtsdo/rvf/execution/service/ReleaseDataManager.java

@@ -25,9 +25,14 @@
 import java.util.Arrays;
 import java.util.Calendar;
 import java.util.Collection;
+import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
+import java.util.Map;
+import java.util.Optional;
 import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 import java.util.stream.Collectors;
 
 import javax.annotation.PostConstruct;
@@ -621,4 +626,63 @@ private File downloadFile(InputStream input, String outputFilename) {
 			IOUtils.closeQuietly(out);
 		}
 	}
+
+	public String getEditionAndVersion(final File zipDataFile)  throws BusinessServiceException {
+		String editionAndVersion = "";
+		String snomedFile = "";
+		List<String> zipFileList = getFileList(zipDataFile);
+		Optional<String> sctOrDerFile = zipFileList.stream()
+							.filter(file -> ( file.indexOf("sct_") != -1 
+										|| file.indexOf("der2_") != -1)
+										&& file.endsWith(".txt"))
+							.findFirst();
+		if (!sctOrDerFile.isPresent()) {
+			throw new BusinessServiceException("There are no RF2 files in data file: " + zipDataFile);
+		}
+		snomedFile = sctOrDerFile.get();
+		Matcher matcher = Pattern.compile(".*_([0-9]+)\\.txt").matcher(snomedFile);
+		if (matcher.find()) {
+			editionAndVersion = 
+				mapFilenameToEdition(snomedFile).toLowerCase() + "_"
+				+ matcher.group(1);
+		} else {
+			throw new BusinessServiceException(
+				"Could not find RF2 file with standard name in data zip file " 
+				+ zipDataFile.getName());
+		}
+		logger.info ("Identified edition and version " + editionAndVersion + " from zip file " + zipDataFile.getName());
+		return editionAndVersion;
+	}
+
+	private String mapFilenameToEdition(String name) {
+		String edition = "INT";
+		Map<String,String> fileNameToEditionMap = new HashMap<String,String>();
+		fileNameToEditionMap.put("SpanishExtension.*_INT", "ES");
+		fileNameToEditionMap.put("_NL_[0-9]+\\.txt", "NL");
+		fileNameToEditionMap.put("_AU1000036_[0-9]+\\.txt", "AU");
+		fileNameToEditionMap.put("_NZ1000210_[0-9]+\\.txt", "NZ");
+		fileNameToEditionMap.put("_US1000124_[0-9]+\\.txt", "US");
+		fileNameToEditionMap.put("_BE1000172_[0-9]+\\.txt", "BE");
+		fileNameToEditionMap.put("_SE1000052_[0-9]+\\.txt", "SE");
+		fileNameToEditionMap.put("GB1000000_[0-9]+\\.txt", "UK");
+		fileNameToEditionMap.put("_INT_[0-9]+\\.txt", "INT");
+		for (String pattern : fileNameToEditionMap.keySet()) {
+			Matcher editionMatcher = Pattern.compile(pattern).matcher(name);
+			if (editionMatcher.find()) {
+				edition = fileNameToEditionMap.get(pattern);
+				break;
+			}
+
+		}
+		return edition;
+	}
+
+	private List<String> getFileList(final File dataFile) throws BusinessServiceException {
+		try {
+			List<String> fileList = ZipFileUtils.listFiles(dataFile);
+			return fileList;
+		} catch (IOException e) {
+			throw new BusinessServiceException("Could not get file list from " + dataFile, e);
+		}
+	}
 }

execution-service/src/main/java/org/ihtsdo/rvf/execution/service/ValidationVersionLoader.java

@@ -152,9 +152,10 @@ private String loadRelease(String releaseVersion) throws IOException, BusinessSe
 		return releaseVersion;
 	}
 
-	public MysqlExecutionConfig createExecutionConfig(ValidationRunConfig validationConfig) {
+	public MysqlExecutionConfig createExecutionConfig(ValidationRunConfig validationConfig) throws BusinessServiceException {
 		MysqlExecutionConfig executionConfig = new MysqlExecutionConfig(validationConfig.getRunId(), validationConfig.isFirstTimeRelease());
-		executionConfig.setProspectiveVersion(RVF_DB_PREFIX + executionConfig.getExecutionId().toString());
+		executionConfig.setProspectiveVersion(RVF_DB_PREFIX + getProspectiveVersionFromFileNames(validationConfig) 
+							+ "_" + executionConfig.getExecutionId().toString());
 		executionConfig.setGroupNames(validationConfig.getGroupsList());
 		executionConfig.setExtensionValidation( isExtension(validationConfig));
 		executionConfig.setFirstTimeRelease(validationConfig.isFirstTimeRelease());
@@ -173,6 +174,13 @@ public MysqlExecutionConfig createExecutionConfig(ValidationRunConfig validation
 		return executionConfig;
 	}
 
+	private String getProspectiveVersionFromFileNames(ValidationRunConfig validationConfig) throws BusinessServiceException {
+		if (validationConfig.getLocalProspectiveFile() == null) {
+			return null;
+		}
+		return releaseDataManager.getEditionAndVersion(validationConfig.getLocalProspectiveFile());
+	}
+
 	public List<String> loadProspectiveDeltaAndCombineWithPreviousSnapshotIntoDB(MysqlExecutionConfig executionConfig, ValidationRunConfig validationConfig,
 				List<String> excludeTableNames) throws BusinessServiceException {
 		List<String> filesLoaded = new ArrayList<>();

execution-service/src/main/java/org/ihtsdo/rvf/execution/service/util/RF2FileTableMapper.java

@@ -29,6 +29,7 @@ public class RF2FileTableMapper {
 	private static final String MRCM_MODULE_SCOPE_FILE_HEADER = "der2_cRefset_.*MRCMModuleScope";
 	private static final String MRCM_ATRRIBUTE_RANGE_FILE_HEADER = "der2_ssccRefset_.*MRCMAttributeRange";
 	private static final String MRCM_DOMAIN_FILE_HEADER = "der2_sssssssRefset_.*MRCMDomain";
+	private static final String DESCRIPTION_TYPE_FILE_HEADER = "der2_ciRefset_.*DescriptionType";
 	private static final Map<String,String> tableNameMap = new HashMap<>();
 	private static final String DELTA = ".*Delta.*_*_\\d{8}.txt";
 	private static final String SNAPSHOT = ".*Snapshot.*_*_\\d{8}.txt";
@@ -58,6 +59,7 @@ public class RF2FileTableMapper {
 		tableNameMap.put(MRCM_MODULE_SCOPE_FILE_HEADER + DELTA, "mrcmmodulescoperefset_d");
 		tableNameMap.put(MRCM_ATRRIBUTE_RANGE_FILE_HEADER + DELTA, "mrcmattributerangerefset_d");
 		tableNameMap.put(MRCM_DOMAIN_FILE_HEADER + DELTA, "mrcmdomainrefset_d");
+		tableNameMap.put(DESCRIPTION_TYPE_FILE_HEADER + DELTA, "descriptiontyperefset_d");
 
 		//Full
 		tableNameMap.put(CONCEPT_FILE_HEADER + FULL, "concept_f");
@@ -82,6 +84,7 @@ public class RF2FileTableMapper {
 		tableNameMap.put(MRCM_MODULE_SCOPE_FILE_HEADER + FULL, "mrcmmodulescoperefset_f");
 		tableNameMap.put(MRCM_ATRRIBUTE_RANGE_FILE_HEADER + FULL, "mrcmattributerangerefset_f");
 		tableNameMap.put(MRCM_DOMAIN_FILE_HEADER + FULL, "mrcmdomainrefset_f");
+		tableNameMap.put(DESCRIPTION_TYPE_FILE_HEADER + FULL, "descriptiontyperefset_f");
 
 		//Snapshot
 		tableNameMap.put(CONCEPT_FILE_HEADER + SNAPSHOT, "concept_s");
@@ -106,6 +109,7 @@ public class RF2FileTableMapper {
 		tableNameMap.put(MRCM_MODULE_SCOPE_FILE_HEADER + SNAPSHOT, "mrcmmodulescoperefset_s");
 		tableNameMap.put(MRCM_ATRRIBUTE_RANGE_FILE_HEADER + SNAPSHOT, "mrcmattributerangerefset_s");
 		tableNameMap.put(MRCM_DOMAIN_FILE_HEADER + SNAPSHOT, "mrcmdomainrefset_s");
+		tableNameMap.put(DESCRIPTION_TYPE_FILE_HEADER + SNAPSHOT, "descriptiontyperefset_s");
 	}
 
 	public static String getLegacyTableName(final String filename) {