Skip to content
This repository has been archived by the owner on Dec 28, 2019. It is now read-only.

Commit

Permalink
ResourceTypesSaver: fetch resource types from official schema JSONs
Browse files Browse the repository at this point in the history
  • Loading branch information
@shalupov
shalupov committed Feb 26, 2019
1 parent f48ae9f commit 42c7ea5
Showing 1 changed file with 66 additions and 30 deletions.
96 changes: 66 additions & 30 deletions metadata-crawler/src/main/kotlin/ResourceTypesSaver.kt
View file
Original file line number Diff line number Diff line change
@@ -1,29 +1,29 @@
@file:Suppress("LoopToCallChain", "Destructure")

import com.google.gson.JsonParser
import com.intellij.aws.cloudformation.CloudFormationConstants
import com.intellij.aws.cloudformation.metadata.CloudFormationLimits
import com.intellij.aws.cloudformation.metadata.CloudFormationManualResourceType
import com.intellij.aws.cloudformation.metadata.CloudFormationMetadata
import com.intellij.aws.cloudformation.metadata.CloudFormationResourceTypesDescription
import com.intellij.aws.cloudformation.metadata.MetadataSerializer
import com.intellij.aws.cloudformation.metadata.awsServerless20161031ResourceTypes
import org.apache.commons.io.IOUtils
import org.jsoup.Jsoup
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.io.File
import java.io.FileOutputStream
import java.io.IOException
import java.lang.Exception
import java.net.URL
import java.util.ArrayList
import java.util.TreeMap
import java.util.regex.Pattern
import java.util.zip.GZIPInputStream

object ResourceTypesSaver {
private val RESOURCE_TYPE_PATTERN = Pattern.compile("<li><a href=\"([^\"]+)\">(AWS::[^<]+)</a></li>")
private val FETCH_TIMEOUT_MS = 10000

fun CloudFormationManualResourceType.toResourceTypeBuilder(): ResourceTypeBuilder {
private fun CloudFormationManualResourceType.toResourceTypeBuilder(): ResourceTypeBuilder {
val builder = ResourceTypeBuilder(name, url)
builder.description = description
builder.transform = CloudFormationConstants.awsServerless20161031TransformName
Expand Down Expand Up @@ -56,15 +56,29 @@ object ResourceTypesSaver {
val predefinedParameters = fetchPredefinedParameters()

val resourceTypes = resourceTypeLocations.pmap(numThreads = 10) {
val builder = ResourceTypeBuilder(it.name, it.location)
fetchResourceType(builder)
val location = when (it.name) {
"AWS::CloudWatch::Dashboard" -> "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-properties-cw-dashboard.html"
"AWS::DMS::ReplicationSubnetGroup" -> "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-dms-replicationsubnet-group.html"
"AWS::ElasticBeanstalk::ConfigurationTemplate" -> "https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-resource-beanstalk-configurationtemplate.html"
else -> it.location
}

// Check everything is set
builder.toResourceType()
builder.toResourceTypeDescription()
// No documentation on site
if (it.name == "AWS::EC2::TrunkInterfaceAssociation") return@pmap null

builder
}
try {
val builder = ResourceTypeBuilder(name = it.name, url = location)
fetchResourceType(builder)

// Check everything is set
builder.toResourceType()
builder.toResourceTypeDescription()

builder
} catch (e: Throwable) {
throw Exception("Unable to parse resource type ${it.name} from $location: ${e.message}", e)
}
}.filterNotNull()

fetchResourceAttributes(resourceTypes)

Expand Down Expand Up @@ -125,7 +139,8 @@ object ResourceTypesSaver {
}

fun addAttribute(_resourceTypeName: String, _attribute: String, _description: String) {
val builder = resourceTypes.single { it.name == _resourceTypeName }
val builder = resourceTypes.singleOrNull { it.name == _resourceTypeName }
?: error("Can't find resource type $_resourceTypeName")
builder.addAttribute(_attribute).description = _description
}

Expand Down Expand Up @@ -318,7 +333,8 @@ object ResourceTypesSaver {
// Most likely a documentation bug, this property was introduced later
false
} else if (requiredValue != null) {
if (requiredValue.equals("Yes", ignoreCase = true)) {
if (requiredValue.equals("Yes", ignoreCase = true) ||
requiredValue.equals("true", ignoreCase = true)) {
true
} else if (
requiredValue.startsWith("No", ignoreCase = true) ||
Expand All @@ -341,6 +357,7 @@ object ResourceTypesSaver {
// TODO
if (builder.name != "AWS::RDS::DBParameterGroup" &&
builder.name != "AWS::Route53::RecordSet" &&
builder.name != "AWS::EC2::EC2Fleet" &&
builder.name != "AWS::RDS::DBSecurityGroupIngress") {
throw RuntimeException("Required is not found in property $name in ${builder.url}")
}
Expand Down Expand Up @@ -431,24 +448,43 @@ object ResourceTypesSaver {

private data class ResourceTypeLocation(val name: String, val location: String)

private fun fetchResourceTypeLocations(): List<ResourceTypeLocation> {
val url = URL("https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/aws-template-resource-type-ref.html")
val content = IOUtils.toString(url)
private fun fetchResourceTypeLocations(url: String): Map<String, ResourceTypeLocation> {
val content = URL(url).openStream().use { stream ->
GZIPInputStream(stream).bufferedReader().readText()
}

val result: MutableList<ResourceTypeLocation> = ArrayList()
val root = JsonParser().parse(content)

val matcher = RESOURCE_TYPE_PATTERN.matcher(content)
while (matcher.find()) {
val href = matcher.group(1)
val name = matcher.group(2)
if ("AWS::CloudFormation::Init" == name) {
continue
}
val resourceTypes = root.asJsonObject["ResourceTypes"].asJsonObject

result.add(ResourceTypeLocation(name.replace(Regex("\\s"), "").trim(), URL(url, href.trim()).toExternalForm()))
}
return resourceTypes.entrySet()
.mapNotNull { (key, resourceTypeJson) ->
val documentationElement = resourceTypeJson.asJsonObject["Documentation"]
if (documentationElement == null) {
return@mapNotNull null
} else {
key to ResourceTypeLocation(
name = key,
location = documentationElement.asString.replace("http://", "https://")
)
}
}
.toMap()
}

private fun fetchResourceTypeLocations(): List<ResourceTypeLocation> {
// from https://docs.aws.amazon.com/AWSCloudFormation/latest/UserGuide/cfn-resource-specification.html
val urls = listOf(
"https://d1mta8qj7i28i2.cloudfront.net/latest/gzip/CloudFormationResourceSpecification.json", // EU (Frankfurt)
"https://d3teyb21fexa9r.cloudfront.net/latest/gzip/CloudFormationResourceSpecification.json", // EU (Ireland)
"https://d68hl49wbnanq.cloudfront.net/latest/gzip/CloudFormationResourceSpecification.json", // US West (N. California)
"https://d201a2mn26r7lk.cloudfront.net/latest/gzip/CloudFormationResourceSpecification.json" // US West (Oregon)
)

return result.sortedBy { it.name }
return urls.map { fetchResourceTypeLocations(it) }
.fold(mapOf<String, ResourceTypeLocation>()) { acc, map -> acc + map }
.map { it.value }
.sortedBy { it.name }
}

private fun fetchPredefinedParameters(): List<String> {
Expand All @@ -468,9 +504,9 @@ object ResourceTypesSaver {
val limits = table.filter { it.size == 4 }.map { it[0] to it[2] }.toMap()

return CloudFormationLimits(
maxMappings = Integer.parseInt(limits["Mappings"]!!.replace(" mappings", "")),
maxParameters = Integer.parseInt(limits["Parameters"]!!.replace(" parameters", "")),
maxOutputs = Integer.parseInt(limits["Outputs"]!!.replace(" outputs", ""))
maxMappings = Integer.parseInt(limits.getValue("Mappings").replace(" mappings", "")),
maxParameters = Integer.parseInt(limits.getValue("Parameters").replace(" parameters", "")),
maxOutputs = Integer.parseInt(limits.getValue("Outputs").replace(" outputs", ""))
)
}
}

0 comments on commit 42c7ea5

Please sign in to comment.