Merge branch 'develop'

Illumina · Jun 3, 2021 · a730317 · a730317
2 parents 985c538 + a8b2bac
commit a730317
Show file tree

Hide file tree

Showing 242 changed files with 6,737 additions and 2,228 deletions.
diff --git a/AnnotationLambda/AnnotationLambda.cs b/AnnotationLambda/AnnotationLambda.cs
@@ -46,8 +46,8 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
 
                 snsTopicArn = LambdaUtilities.GetEnvironmentVariable(LambdaUtilities.SnsTopicKey);
 
-                string vcfUrl = config.vcfUrl;
-
+                string vcfUrl       = config.vcfUrl;
+                int    variantCount = 0;
                 using (var annotationResources = GetAnnotationResources(config))
                 {
                     if (annotationResources.InputStartVirtualPosition == -1) return GetSuccessOutput(result);
@@ -60,7 +60,7 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
                     }
 
                     Logger.WriteLine("Scan for positions to preload complete.");
-
+                    
                     using (var aes = new AesCryptoServiceProvider())
                     {
                         FileMetadata jsonMetadata, jasixMetadata;
@@ -84,7 +84,7 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
 
                             using (var jsonCompressStream = new BlockGZipStream(jsonMd5Stream, CompressionMode.Compress))
                             {
-                                StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true);
+                                variantCount = StreamAnnotation.Annotate(headerStream, inputVcfStream, jsonCompressStream, jasixMd5Stream, annotationResources, vcfFilter, true).variantCount;
                             }
 
                             Logger.WriteLine("Annotation done.");
@@ -107,6 +107,7 @@ public AnnotationResult Run(AnnotationConfig config, ILambdaContext context)
                 LambdaUtilities.DeleteTempOutput();
                 if (string.IsNullOrEmpty(result.filePath)) throw new FileNotFoundException();
 
+                result.variantCount = variantCount;
                 return GetSuccessOutput(result);
             }
             catch (Exception exception)
@@ -155,7 +156,9 @@ private static AnnotationResources GetAnnotationResources(AnnotationConfig annot
             var genomeAssembly      = GenomeAssemblyHelper.Convert(annotationConfig.genomeAssembly);
             string cachePathPrefix  = LambdaUrlHelper.GetCacheFolder().UrlCombine(genomeAssembly.ToString()).UrlCombine(LambdaUrlHelper.DefaultCacheSource);
             string nirvanaS3Ref     = LambdaUrlHelper.GetRefUrl(genomeAssembly);
-            string saManifestUrl    = LambdaUtilities.GetManifestUrl(annotationConfig.supplementaryAnnotations ?? "latest", genomeAssembly);
+            // SaVersion will be provided as an environment variable. Defaults to "latest"
+            string saVersion = Environment.GetEnvironmentVariable("SaVersion");
+            string saManifestUrl    = LambdaUtilities.GetManifestUrl(string.IsNullOrEmpty(saVersion)? "latest": saVersion, genomeAssembly);
             var metrics = new PerformanceMetrics();
 
             var annotationResources = new AnnotationResources(nirvanaS3Ref, cachePathPrefix,

diff --git a/AnnotationLambda/AnnotationLambda.csproj b/AnnotationLambda/AnnotationLambda.csproj
@@ -1,17 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <GenerateRuntimeConfigurationFiles>true</GenerateRuntimeConfigurationFiles>
     <AWSProjectType>Lambda</AWSProjectType>
-    <OutputPath>bin\$(Configuration)</OutputPath>
+    <OutputPath>bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
-    <PackageReference Include="AWSSDK.Lambda" Version="3.3.106.4" />
-    <PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
-    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
+    <PackageReference Include="AWSSDK.Lambda" Version="3.7.0.15" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
+    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Cloud\Cloud.csproj" />

diff --git a/AnnotationLambda/Properties/launchSettings.json b/AnnotationLambda/Properties/launchSettings.json
diff --git a/CacheUtils/CacheUtils.csproj b/CacheUtils/CacheUtils.csproj
@@ -1,8 +1,8 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
     <OutputType>Exe</OutputType>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
-    <OutputPath>..\bin\$(Configuration)</OutputPath>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
+    <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <Import Project="..\CommonAssemblyInfo.props" />
@@ -12,9 +12,9 @@
     </Content>
   </ItemGroup>
   <ItemGroup>
-    <PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="3.1.0" />
-    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="3.1.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="5.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="5.0.0" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\CommandLine\CommandLine.csproj" />

diff --git a/CacheUtils/Properties/launchSettings.json b/CacheUtils/Properties/launchSettings.json
diff --git a/CacheUtils/TranscriptCache/TranscriptCacheUtilities.cs b/CacheUtils/TranscriptCache/TranscriptCacheUtilities.cs
@@ -20,13 +20,13 @@ public static List<ITranscript> GetTranscripts(DataBundle bundle, IChromosomeInt
 
         public static IntervalArray<T>[] ToIntervalArrays<T>(this IEnumerable<T> items, int numRefSeqs) where T : IChromosomeInterval
         {
-            var intervalArrays = new IntervalArray<T>[numRefSeqs];
-            var itemsByRef     = items.GetMultiValueDict(x => x.Chromosome.Index);
+            var                         intervalArrays = new IntervalArray<T>[numRefSeqs];
+            Dictionary<ushort, List<T>> itemsByRef     = items.GetMultiValueDict(x => x.Chromosome.Index);
 
             foreach (ushort refIndex in itemsByRef.Keys.OrderBy(x => x))
             {
-                var unsortedItems = itemsByRef[refIndex];
-                var intervals     = unsortedItems.OrderBy(x => x.Start).ThenBy(x => x.End).ToIntervals(unsortedItems.Count);
+                List<T>       unsortedItems = itemsByRef[refIndex];
+                Interval<T>[] intervals     = unsortedItems.OrderBy(x => x.Start).ThenBy(x => x.End).ToIntervals(unsortedItems.Count);
                 intervalArrays[refIndex] = new IntervalArray<T>(intervals);
             }
 

diff --git a/Cloud/Cloud.csproj b/Cloud/Cloud.csproj
@@ -1,15 +1,15 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="Amazon.Lambda.Core" Version="1.1.0" />
-    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="1.7.0" />
-    <PackageReference Include="AWSSDK.S3" Version="3.3.109.4" />
-    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.3.101.100" />
-    <PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
+    <PackageReference Include="Amazon.Lambda.Core" Version="2.0.0" />
+    <PackageReference Include="Amazon.Lambda.Serialization.Json" Version="2.0.0" />
+    <PackageReference Include="AWSSDK.S3" Version="3.7.0.17" />
+    <PackageReference Include="AWSSDK.SimpleNotificationService" Version="3.7.1.1" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\Genome\Genome.csproj" />

diff --git a/Cloud/Messages/Annotation/AnnotationConfig.cs b/Cloud/Messages/Annotation/AnnotationConfig.cs
@@ -6,15 +6,14 @@ public sealed class AnnotationConfig
     {
         // ReSharper disable InconsistentNaming
         // ReSharper disable NotAccessedField.Global
-        public string id;
-        public string genomeAssembly;
-        public string vcfUrl;
-        public string tabixUrl;
-        public S3Path outputDir;
-        public string outputPrefix;
-        public string supplementaryAnnotations;
-        public List<SaUrls> customAnnotations;
-        public string customStrUrl;
+        public string          id;
+        public string          genomeAssembly;
+        public string          vcfUrl;
+        public string          tabixUrl;
+        public S3Path          outputDir;
+        public string          outputPrefix;
+        public List<SaUrls>    customAnnotations;
+        public string          customStrUrl;
         public AnnotationRange annotationRange;
         // ReSharper restore NotAccessedField.Global
         // ReSharper restore InconsistentNaming

diff --git a/Cloud/Messages/Annotation/AnnotationResult.cs b/Cloud/Messages/Annotation/AnnotationResult.cs
@@ -11,6 +11,8 @@ public sealed class AnnotationResult
         public string status;
         public string filePath;
         public ErrorCategory? errorCategory;
+
+        public int variantCount;
         // ReSharper restore UnassignedField.Global
         // ReSharper restore InconsistentNaming
     }

diff --git a/Cloud/Messages/Custom/CustomConfig.cs b/Cloud/Messages/Custom/CustomConfig.cs
@@ -4,9 +4,12 @@ namespace Cloud.Messages.Custom
 {
     public sealed class CustomConfig
     {
-        public string id;
-        public string tsvUrl;
-        public S3Path outputDir;
+        public string    id;
+        public string    tsvUrl;
+        public S3Path    outputDir;
+        public JwtFields jwtFields;
+
         public bool skipGeneIdValidation;
+        public bool skipRefBaseValidation;
     }
 }
diff --git a/Cloud/Messages/Custom/CustomResult.cs b/Cloud/Messages/Custom/CustomResult.cs
@@ -4,11 +4,13 @@
     // ReSharper disable InconsistentNaming
     public sealed class CustomResult
     {
-        public string id;
-        public string status;
-        public string genomeAssembly;
-        public FileList created;
-        public bool noValidEntries;
+        public string    id;
+        public string    status;
+        public string    genomeAssembly;
+        public FileList  created;
+        public bool      noValidEntries;
+        public JwtFields jwtFields;
+        public int       variantCount;
     }
     // ReSharper restore InconsistentNaming
     // ReSharper restore NotAccessedField.Global

diff --git a/Cloud/Messages/JwtFields.cs b/Cloud/Messages/JwtFields.cs
@@ -0,0 +1,12 @@
+namespace Cloud.Messages
+{
+    public class JwtFields
+    {
+        // ReSharper disable InconsistentNaming
+        public string authorizedParty;
+        public string subject;
+        public string tenantId;
+
+        // ReSharper restore InconsistentNaming
+    }
+}
diff --git a/Cloud/Messages/Nirvana/NirvanaConfig.cs b/Cloud/Messages/Nirvana/NirvanaConfig.cs
@@ -12,10 +12,11 @@ public sealed class NirvanaConfig
         public string vcfUrl;
         public string tabixUrl;
         public S3Path outputDir;
-        public string supplementaryAnnotations;
         // ReSharper disable once UnassignedField.Global
         public List<SaUrls> customAnnotations;
-        public string customStrUrl;
+        public string       customStrUrl;
+        public JwtFields    jwtFields;
+
         // ReSharper restore InconsistentNaming
 
         internal void CheckRequiredFieldsNotNull()

diff --git a/Cloud/Messages/Nirvana/NirvanaResult.cs b/Cloud/Messages/Nirvana/NirvanaResult.cs
@@ -4,9 +4,12 @@ public sealed class NirvanaResult
     {
         // ReSharper disable InconsistentNaming
         // ReSharper disable NotAccessedField.Global
-        public string id;
-        public string status;
-        public FileList created;
+        public string    id;
+        public string    status;
+        public FileList  created;
+        public JwtFields jwtFields;
+        public int       variantCount;
+
         // ReSharper restore NotAccessedField.Global
         // ReSharper restore InconsistentNaming
     }

diff --git a/CommandLine/CommandLine.csproj b/CommandLine/CommandLine.csproj
@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>

diff --git a/CommonAssemblyInfo.props b/CommonAssemblyInfo.props
@@ -1,10 +1,10 @@
 <Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <Company>Illumina</Company>
-    <Copyright>© 2020 Illumina, Inc.</Copyright>
-    <FileVersion>3.13.0</FileVersion>
-    <AssemblyVersion>3.13.0</AssemblyVersion>
-    <Version>3.13.0</Version>
+    <Copyright>© 2021 Illumina, Inc.</Copyright>
+    <FileVersion>3.15.0</FileVersion>
+    <AssemblyVersion>3.15.0</AssemblyVersion>
+    <Version>3.15.0</Version>
     <Authors>Stromberg, Roy, Lajugie, Jiang, Li, and Kang</Authors>
   </PropertyGroup>
 </Project>
diff --git a/Compression/Compression.csproj b/Compression/Compression.csproj
@@ -1,6 +1,6 @@
 <Project Sdk="Microsoft.NET.Sdk">
   <PropertyGroup>
-    <TargetFramework>netcoreapp2.1</TargetFramework>
+    <TargetFramework>netcoreapp3.1</TargetFramework>
     <OutputPath>..\bin\$(Configuration)</OutputPath>
     <LangVersion>latest</LangVersion>
   </PropertyGroup>
@@ -12,9 +12,6 @@
     <None Include="Packages\BlockCompression\BlockCompression.dll" Link="BlockCompression.dll">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
-    <None Include="Packages\BlockCompression\BlockCompression.pdb" Link="BlockCompression.pdb">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
     <None Include="Packages\BlockCompression\libBlockCompression.so" Link="libBlockCompression.so">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </None>
@@ -23,4 +20,4 @@
     </None>
   </ItemGroup>
   <Import Project="..\CommonAssemblyInfo.props" />
-</Project>
+</Project>
diff --git a/Compression/Packages/BlockCompression/BlockCompression.dll b/Compression/Packages/BlockCompression/BlockCompression.dll
diff --git a/Compression/Packages/BlockCompression/BlockCompression.pdb b/Compression/Packages/BlockCompression/BlockCompression.pdb
diff --git a/Compression/Packages/BlockCompression/libBlockCompression.dylib b/Compression/Packages/BlockCompression/libBlockCompression.dylib
diff --git a/Compression/Packages/BlockCompression/libBlockCompression.so b/Compression/Packages/BlockCompression/libBlockCompression.so