Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix release pipeline Originally Issue 4106 process map to properties when uploading loinc terminology #4246

Merged
merged 13 commits into from
Nov 4, 2022
Merged

Fix release pipeline Originally Issue 4106 process map to properties when uploading loinc terminology #4246

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
13 commits
Select commit Hold shift + click to select a range
6a29bbf
Add loinc upload handler to process MAP_TO properties.
Oct 5, 2022
071a890
Move test to test folder
Oct 6, 2022
3d094bb
Fix mapto file location property name
Oct 6, 2022
c19ce09
Allow override as full loinc upload tests take (much) longer than def…
Oct 6, 2022
c6644f8
merge master in
Oct 31, 2022
c90fc74
Move handler later to make sure all concepts were added to map
Oct 31, 2022
b16a44e
Update test to check all project enhancements
Nov 2, 2022
b65def5
Restore project unrelated changes
Nov 2, 2022
b54ea2d
Rename base test class properly and move to right package
Nov 2, 2022
fe1059e
Condition MapTo.csv file processing to file presence for backwards as…
Nov 2, 2022
80bb839
Document and annotate test-only usage for disabling deferred tasks ti…
Nov 2, 2022
8f8ced4
Merge branch 'master' into issue-4106-process-map-to-properties-when-…
Nov 4, 2022
4564369
Fix alignments to try to make release pipeline work
Nov 4, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Allow override as full loinc upload tests take (much) longer than def… 
…ined timeout
  • Loading branch information
juan.marchionatto committed Oct 6, 2022
commit c19ce09a5cac7a045ca8b1bb03eae08399ffc048
16 changes: 11 additions & 5 deletions hapi-fhir-jpaserver-base/src/main/java/ca/uhn/fhir/jpa/term/TermDeferredStorageSvcImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,7 @@ public class TermDeferredStorageSvcImpl implements ITermDeferredStorageSvc {
private static final Logger ourLog = LoggerFactory.getLogger(TermDeferredStorageSvcImpl.class);
private static final long SAVE_ALL_DEFERRED_WARN_MINUTES = 1;
private static final long SAVE_ALL_DEFERRED_ERROR_MINUTES = 5;
public static boolean ourAllowTimeout = true;
private final List<TermCodeSystem> myDeferredCodeSystemsDeletions = Collections.synchronizedList(new ArrayList<>());
private final Queue<TermCodeSystemVersion> myDeferredCodeSystemVersionsDeletions = new ConcurrentLinkedQueue<>();
private final List<TermConcept> myDeferredConcepts = Collections.synchronizedList(new ArrayList<>());
Expand Down Expand Up @@ -274,13 +275,18 @@ private <T> T runInTransaction(Supplier<T> theRunnable) {

@Override
public void saveAllDeferred() {
TimeoutManager timeoutManager = new TimeoutManager(TermDeferredStorageSvcImpl.class.getName() + ".saveAllDeferred()",
Duration.of(SAVE_ALL_DEFERRED_WARN_MINUTES, ChronoUnit.MINUTES),
Duration.of(SAVE_ALL_DEFERRED_ERROR_MINUTES, ChronoUnit.MINUTES));
TimeoutManager timeoutManager = null;
if (ourAllowTimeout) {
timeoutManager = new TimeoutManager(TermDeferredStorageSvcImpl.class.getName() + ".saveAllDeferred()",
Duration.of(SAVE_ALL_DEFERRED_WARN_MINUTES, ChronoUnit.MINUTES),
Duration.of(SAVE_ALL_DEFERRED_ERROR_MINUTES, ChronoUnit.MINUTES));
}

while (!isStorageQueueEmpty()) {
if (timeoutManager.checkTimeout()) {
ourLog.info(toString());
if (ourAllowTimeout) {
if (timeoutManager.checkTimeout()) {
ourLog.info(toString());
}
}
saveDeferred();
}
Expand Down
40 changes: 15 additions & 25 deletions ...jpaserver-test-utilities/src/test/java/ca/uhn/fhir/jpa/term/LoincFullLoadR4SandboxIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@
import ca.uhn.fhir.util.StopWatch;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Multimap;
import com.google.common.collect.MultimapBuilder;
import com.google.common.collect.testing.google.MultimapAsMapGetTester;
import net.ttddyy.dsproxy.support.ProxyDataSourceBuilder;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
Expand Down Expand Up @@ -68,7 +66,6 @@
import java.util.Arrays;
import java.util.Collection;
import java.util.Comparator;
import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
Expand All @@ -77,7 +74,6 @@
import java.util.Optional;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import java.util.zip.ZipEntry;
import java.util.zip.ZipFile;

Expand All @@ -95,22 +91,16 @@
/**
* Sandbox test (not intended to run on CI build) so must be kept disabled
*
* Requires the loinc-full resource directory to contain the following three files:
* _ Loinc.csv.gz
* _ Loinc_1.11.zip and
* Requires the loinc-full resource directory to contain the following files:
* _ Loinc_1.11.zip
* _ v1.11_loincupload.properties
*
* but they are too large for the repo, so before running this test, copy them from:
* but last one is too large for the repo, so before running this test, copy it from:
* https://drive.google.com/drive/folders/18be2R5IurlWnugkl18nDG7wrwPsOtfR-?usp=sharing
* (SmileCDR has access)
*
* Can be executed with Lucene or Elastic configuration
*
* Requires 4Gb mem to run, so pom needs to be changed to run from IDE:
* <surefire_jvm_args>-Dfile.encoding=UTF-8 -Xmx5g</surefire_jvm_args>
* or to run from maven use:
* mvn test -pl :hapi-fhir-jpaserver-test-utilities -Dtest=LoincFullLoadR4SandboxIT#uploadLoincCodeSystem -Dsurefire_jvm_args="-Xmx5g"
*
*
*/
@Disabled("Sandbox test")
@ExtendWith(SpringExtension.class)
Expand Down Expand Up @@ -162,12 +152,12 @@ public class LoincFullLoadR4SandboxIT extends BaseJpaTest {
public static final String LOINC_PROPERTIES_CLASSPATH =
ResourceUtils.CLASSPATH_URL_PREFIX + TEST_FILES_CLASSPATH + "v1.11_loincupload.properties";

public static final String LOINC_ZIP_CLASSPATH =
ResourceUtils.CLASSPATH_URL_PREFIX + TEST_FILES_CLASSPATH + "Loinc_1.11_changed.zip";
public static final String BASE_LOINC_FILE_NAME = "Loinc_1.11";

public static final String LOINC_CSV_FILE_ZIP_PATH = "Loinc_1.11_changed/LoincTable/Loinc.csv";

public static final String LOINC_MAP_TO_FILE_ZIP_PATH = "Loinc_1.11_changed/LoincTable/MapTo.csv";
public static final String LOINC_ZIP_CLASSPATH =
ResourceUtils.CLASSPATH_URL_PREFIX + TEST_FILES_CLASSPATH + BASE_LOINC_FILE_NAME + ".zip";
public static final String LOINC_CSV_ZIP_ENTRY_PATH = BASE_LOINC_FILE_NAME + "/LoincTable/Loinc.csv";
public static final String LOINC_MAP_TO_ZIP_ENTRY_PATH = BASE_LOINC_FILE_NAME + "/LoincTable/MapTo.csv";
// -----------------------------------------------------------------------------------------

@Autowired private FhirContext myFhirCtx;
Expand All @@ -180,12 +170,9 @@ public class LoincFullLoadR4SandboxIT extends BaseJpaTest {
@Autowired private ITermCodeSystemDao myTermCodeSystemDao;
@Autowired private ITermCodeSystemVersionDao myTermCodeSystemVersionDao;


@Autowired
@Qualifier("myValueSetDaoR4")
protected IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao;


private IFhirResourceDaoValueSet<ValueSet, Coding, CodeableConcept> myValueSetDao;

private long termCodeSystemVersionWithVersionId;

Expand Down Expand Up @@ -225,6 +212,9 @@ public class LoincFullLoadR4SandboxIT extends BaseJpaTest {
public void uploadLoincCodeSystem() throws Exception {

if (USE_REAL_DB && LOAD_DB) {
// real test load requires longer time than unit tests
TermDeferredStorageSvcImpl.ourAllowTimeout = false;

List<ITermLoaderSvc.FileDescriptor> myFileDescriptors = buildFileDescriptors();

// upload terminology
Expand Down Expand Up @@ -443,7 +433,7 @@ private List<String> parsePropertyCodeValues(String theValue) {


private List<Map<String, String>> readLoincCsvRecordsAsMap() throws Exception {
CSVParser parser = getParserForZipFile(LOINC_ZIP_CLASSPATH, LOINC_CSV_FILE_ZIP_PATH);
CSVParser parser = getParserForZipFile(LOINC_ZIP_CLASSPATH, LOINC_CSV_ZIP_ENTRY_PATH);
Iterator<CSVRecord> iter = parser.iterator();

Map<String, Integer> headerMap = parser.getHeaderMap();
Expand Down Expand Up @@ -472,7 +462,7 @@ private List<Map<String, String>> readLoincCsvRecordsAsMap() throws Exception {


private Multimap<String, Pair<String, String>> readMapToCsvRecordsAsMap() throws Exception {
CSVParser parser = getParserForZipFile(LOINC_ZIP_CLASSPATH, LOINC_MAP_TO_FILE_ZIP_PATH);
CSVParser parser = getParserForZipFile(LOINC_ZIP_CLASSPATH, LOINC_MAP_TO_ZIP_ENTRY_PATH);

Map<String, Integer> headerMap = parser.getHeaderMap();
recordPropertyNames = headerMap.entrySet().stream()
Expand Down