From 5e63403106607f1de8a03adeae85574701acf4c2 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 13 Mar 2023 01:57:19 +0100 Subject: [PATCH 1/3] 2i2c-aws-us: node sharing profile lists --- .../clusters/2i2c-aws-us/common.values.yaml | 196 ++++++++++++++++++ .../2i2c-aws-us/researchdelight.values.yaml | 58 ------ 2 files changed, 196 insertions(+), 58 deletions(-) diff --git a/config/clusters/2i2c-aws-us/common.values.yaml b/config/clusters/2i2c-aws-us/common.values.yaml index af4582eb2..ab87ad150 100644 --- a/config/clusters/2i2c-aws-us/common.values.yaml +++ b/config/clusters/2i2c-aws-us/common.values.yaml @@ -3,3 +3,199 @@ basehub: scheduling: userScheduler: enabled: true + singleuser: + profileList: + # NOTE: About node sharing + # + # CPU/Memory requests/limits are actively considered still. This + # profile list is setup to involve node sharing as considered in + # https://github.com/2i2c-org/infrastructure/issues/2121. + # + # - Memory requests are different from the description, based on: + # whats found to remain allocate in k8s, subtracting 1GiB + # overhead for misc system pods, and transitioning from GB in + # description to GiB in mem_guarantee. + # - CPU requests are lower than the description, with a factor of + # 10%. + # + - display_name: "Small: up to 4 CPU / 32 GB RAM" + description: &profile_list_description "Start a container with at least a chosen share of capacity on a node of this type" + slug: small + default: true + profile_options: + requests: + # NOTE: Node share choices are in active development, see comment + # next to profileList: above. + display_name: Node share + choices: + mem_1: + default: true + display_name: ~1 GB, ~0.125 CPU + kubespawner_override: + mem_guarantee: 0.904G + cpu_guarantee: 0.013 + mem_2: + display_name: ~2 GB, ~0.25 CPU + kubespawner_override: + mem_guarantee: 1.809G + cpu_guarantee: 0.025 + mem_4: + display_name: ~4 GB, ~0.5 CPU + kubespawner_override: + mem_guarantee: 3.617G + cpu_guarantee: 0.05 + mem_8: + display_name: ~8 GB, ~1.0 CPU + kubespawner_override: + mem_guarantee: 7.234G + cpu_guarantee: 0.1 + mem_16: + display_name: ~16 GB, ~2.0 CPU + kubespawner_override: + mem_guarantee: 14.469G + cpu_guarantee: 0.2 + mem_32: + display_name: ~32 GB, ~4.0 CPU + kubespawner_override: + mem_guarantee: 28.937G + cpu_guarantee: 0.4 + kubespawner_override: + cpu_limit: null + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.xlarge + - display_name: "Medium: up to 16 CPU / 128 GB RAM" + description: *profile_list_description + slug: medium + profile_options: + requests: + # NOTE: Node share choices are in active development, see comment + # next to profileList: above. + display_name: Node share + choices: + mem_1: + display_name: ~1 GB, ~0.125 CPU + kubespawner_override: + mem_guarantee: 0.942G + cpu_guarantee: 0.013 + mem_2: + display_name: ~2 GB, ~0.25 CPU + kubespawner_override: + mem_guarantee: 1.883G + cpu_guarantee: 0.025 + mem_4: + default: true + display_name: ~4 GB, ~0.5 CPU + kubespawner_override: + mem_guarantee: 3.766G + cpu_guarantee: 0.05 + mem_8: + display_name: ~8 GB, ~1.0 CPU + kubespawner_override: + mem_guarantee: 7.532G + cpu_guarantee: 0.1 + mem_16: + display_name: ~16 GB, ~2.0 CPU + kubespawner_override: + mem_guarantee: 15.064G + cpu_guarantee: 0.2 + mem_32: + display_name: ~32 GB, ~4.0 CPU + kubespawner_override: + mem_guarantee: 30.128G + cpu_guarantee: 0.4 + mem_64: + display_name: ~64 GB, ~8.0 CPU + kubespawner_override: + mem_guarantee: 60.257G + cpu_guarantee: 0.8 + mem_128: + display_name: ~128 GB, ~16.0 CPU + kubespawner_override: + mem_guarantee: 120.513G + cpu_guarantee: 1.6 + kubespawner_override: + cpu_limit: null + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.4xlarge + - display_name: "Large: up to 64 CPU / 512 GB RAM" + description: *profile_list_description + slug: large + profile_options: + requests: + # NOTE: Node share choices are in active development, see comment + # next to profileList: above. + display_name: Node share + choices: + mem_4: + display_name: ~4 GB, ~0.5 CPU + kubespawner_override: + mem_guarantee: 3.821G + cpu_guarantee: 0.05 + mem_8: + display_name: ~8 GB, ~1.0 CPU + kubespawner_override: + mem_guarantee: 7.643G + cpu_guarantee: 0.1 + mem_16: + default: true + display_name: ~16 GB, ~2.0 CPU + kubespawner_override: + mem_guarantee: 15.285G + cpu_guarantee: 0.2 + mem_32: + display_name: ~32 GB, ~4.0 CPU + kubespawner_override: + mem_guarantee: 30.571G + cpu_guarantee: 0.4 + mem_64: + display_name: ~64 GB, ~8.0 CPU + kubespawner_override: + mem_guarantee: 61.141G + cpu_guarantee: 0.8 + mem_128: + display_name: ~128 GB, ~16.0 CPU + kubespawner_override: + mem_guarantee: 122.282G + cpu_guarantee: 1.6 + mem_256: + display_name: ~256 GB, ~32.0 CPU + kubespawner_override: + mem_guarantee: 244.565G + cpu_guarantee: 3.2 + mem_512: + display_name: ~512 GB, ~64.0 CPU + kubespawner_override: + mem_guarantee: 489.13G + cpu_guarantee: 6.4 + kubespawner_override: + cpu_limit: null + mem_limit: null + node_selector: + node.kubernetes.io/instance-type: r5.16xlarge + + - display_name: "Large + GPU" + description: "14GB RAM, 4 CPUs, T4 GPU" + profile_options: + image: + display_name: Image + choices: + tensorflow: + display_name: Pangeo Tensorflow ML Notebook + slug: "tensorflow" + kubespawner_override: + node.kubernetes.io/instance-type: g4dn.xlarge + image: "pangeo/ml-notebook:b9584f6" + pytorch: + display_name: Pangeo PyTorch ML Notebook + default: true + slug: "pytorch" + kubespawner_override: + node.kubernetes.io/instance-type: g4dn.xlarge + image: "pangeo/pytorch-notebook:b9584f6" + kubespawner_override: + mem_limit: null + mem_guarantee: 14G + extra_resource_limits: + nvidia.com/gpu: "1" diff --git a/config/clusters/2i2c-aws-us/researchdelight.values.yaml b/config/clusters/2i2c-aws-us/researchdelight.values.yaml index fcb3a6989..11acfa87d 100644 --- a/config/clusters/2i2c-aws-us/researchdelight.values.yaml +++ b/config/clusters/2i2c-aws-us/researchdelight.values.yaml @@ -42,64 +42,6 @@ basehub: # Temporarily set for *all* pods, including pods without any GPUs, # to work around https://github.com/2i2c-org/infrastructure/issues/1530 NVIDIA_DRIVER_CAPABILITIES: compute,utility - profileList: - # The mem-guarantees are here so k8s doesn't schedule other pods - # on these nodes. - - display_name: "Small: m5.large" - description: "~2 CPU, ~8G RAM" - default: true - kubespawner_override: - # Explicitly unset mem_limit, so it overrides the default memory limit we set in - # basehub/values.yaml - mem_limit: 8G - mem_guarantee: 6.5G - node_selector: - node.kubernetes.io/instance-type: m5.large - - display_name: "Medium: m5.xlarge" - description: "~4 CPU, ~15G RAM" - kubespawner_override: - mem_limit: 15G - mem_guarantee: 12G - node_selector: - node.kubernetes.io/instance-type: m5.xlarge - - display_name: "Large: m5.2xlarge" - description: "~8 CPU, ~30G RAM" - kubespawner_override: - mem_limit: 30G - mem_guarantee: 25G - node_selector: - node.kubernetes.io/instance-type: m5.2xlarge - - display_name: "Huge: m5.8xlarge" - description: "~16 CPU, ~60G RAM" - kubespawner_override: - mem_limit: 60G - mem_guarantee: 50G - node_selector: - node.kubernetes.io/instance-type: m5.8xlarge - - display_name: "Large + GPU" - description: "14GB RAM, 4 CPUs, T4 GPU" - profile_options: - image: - display_name: Image - choices: - tensorflow: - display_name: Pangeo Tensorflow ML Notebook - slug: "tensorflow" - kubespawner_override: - node.kubernetes.io/instance-type: g4dn.xlarge - image: "pangeo/ml-notebook:b9584f6" - pytorch: - display_name: Pangeo PyTorch ML Notebook - default: true - slug: "pytorch" - kubespawner_override: - node.kubernetes.io/instance-type: g4dn.xlarge - image: "pangeo/pytorch-notebook:b9584f6" - kubespawner_override: - mem_limit: null - mem_guarantee: 14G - extra_resource_limits: - nvidia.com/gpu: "1" hub: config: Authenticator: From 7d3b55a1828b4d7b4e92f36ff271c96b0ad369e1 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 13 Mar 2023 03:31:27 +0100 Subject: [PATCH 2/3] 2i2c-aws-us: avoid already reported issue in kubespawner about profileList --- config/clusters/2i2c-aws-us/common.values.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/config/clusters/2i2c-aws-us/common.values.yaml b/config/clusters/2i2c-aws-us/common.values.yaml index ab87ad150..0519a6e27 100644 --- a/config/clusters/2i2c-aws-us/common.values.yaml +++ b/config/clusters/2i2c-aws-us/common.values.yaml @@ -175,8 +175,9 @@ basehub: node_selector: node.kubernetes.io/instance-type: r5.16xlarge - - display_name: "Large + GPU" - description: "14GB RAM, 4 CPUs, T4 GPU" + - display_name: NVIDIA Tesla T4, ~16 GB, ~4 CPUs + slug: gpu + description: "Start a container on a dedicated node with a GPU" profile_options: image: display_name: Image From 717203ed4e3f366278418ee91356608247dc3a74 Mon Sep 17 00:00:00 2001 From: Erik Sundell Date: Mon, 13 Mar 2023 03:32:23 +0100 Subject: [PATCH 3/3] 2i2c-aws-us: k8s 1.25, highmem nodes, ssh-keys, node sharing --- eksctl/2i2c-aws-us.jsonnet | 19 ++++++------------- eksctl/ssh-keys/2i2c-aws-us.key.pub | 1 + eksctl/ssh-keys/secret/2i2c-aws-us.key | 21 +++++++++++++++++++++ 3 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 eksctl/ssh-keys/2i2c-aws-us.key.pub create mode 100644 eksctl/ssh-keys/secret/2i2c-aws-us.key diff --git a/eksctl/2i2c-aws-us.jsonnet b/eksctl/2i2c-aws-us.jsonnet index 7afaeb864..3ea843262 100644 --- a/eksctl/2i2c-aws-us.jsonnet +++ b/eksctl/2i2c-aws-us.jsonnet @@ -25,10 +25,9 @@ local nodeAz = "us-west-2a"; // A `node.kubernetes.io/instance-type label is added, so pods // can request a particular kind of node with a nodeSelector local notebookNodes = [ - { instanceType: "m5.large" }, - { instanceType: "m5.xlarge" }, - { instanceType: "m5.2xlarge" }, - { instanceType: "m5.8xlarge" }, + { instanceType: "r5.xlarge" }, + { instanceType: "r5.4xlarge" }, + { instanceType: "r5.16xlarge" }, { instanceType: "g4dn.xlarge", tags+: { @@ -44,10 +43,7 @@ local daskNodes = [ // *first* item in instanceDistribution.instanceTypes, to match // what we do with notebook nodes. Pods can request a particular // kind of node with a nodeSelector - { instancesDistribution+: { instanceTypes: ["m5.large"] }}, - { instancesDistribution+: { instanceTypes: ["m5.xlarge"] }}, - { instancesDistribution+: { instanceTypes: ["m5.2xlarge"] }}, - { instancesDistribution+: { instanceTypes: ["m5.8xlarge"] }}, + { instancesDistribution+: { instanceTypes: ["r5.4xlarge"] }}, ]; @@ -57,10 +53,7 @@ local daskNodes = [ metadata+: { name: "2i2c-aws-us", region: clusterRegion, - // Warning: version 1.23 introduces some breaking changes - // Checkout the docs before upgrading - // ref: https://docs.aws.amazon.com/eks/latest/userguide/ebs-csi-migration-faq.html - version: '1.22' + version: '1.25' }, availabilityZones: masterAzs, iam: { @@ -92,7 +85,7 @@ local daskNodes = [ ssh: { publicKeyPath: 'ssh-keys/2i2c-aws-us.key.pub' }, - instanceType: "m5.xlarge", + instanceType: "r5.xlarge", minSize: 1, maxSize: 6, labels+: { diff --git a/eksctl/ssh-keys/2i2c-aws-us.key.pub b/eksctl/ssh-keys/2i2c-aws-us.key.pub new file mode 100644 index 000000000..1245d8a9c --- /dev/null +++ b/eksctl/ssh-keys/2i2c-aws-us.key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDQcjyAtAy1dt/geq+lKfQ/EeUgZevd1LAbVMoT8x0BrhjiLZ+vK8YSTrIPvsGOziRCTtxudtufOCSh8fNW6+yq9Ano7OCFAADzVOoUtnzJkceUf9YhqV8RYEqrabc5/oxxK8ResOx1z4ibu3aL5jBWYxFKKJsM7hhe06IaJ6c71Vio3rh4MI2WFu5y8TzdR1jkfvQpouI5+DRVy/Nlc5M7K6DhpX6HVmTSDwd85TBNRzeCZkJ9yf4rsnokvJUFDw94FJN7x1yLzLg9uwb8Fmzp0WFc4WVy7ICn0O0JgXQ8FTxYxsJvPZo7eP6ib0nDo2csHMQZ+jGErWXUFY+GOJ8/Rba1jnMt0XkX3tG++7DGNhxCEz4zMthBG7dkRO9OYEEpRHJhAvwgyA55Ij1k5y2BOla+nDdwKhlabd5iCVo+wTOZO2Pz7iq4hI3Pa6zn+rC2zXu++UTiJj/E6CE0s4EQrp8OLkkWmkweIxihhxUBVzeUgWu57tEqjpUAfyhNcGk= erik@dl diff --git a/eksctl/ssh-keys/secret/2i2c-aws-us.key b/eksctl/ssh-keys/secret/2i2c-aws-us.key new file mode 100644 index 000000000..27b827f3f --- /dev/null +++ b/eksctl/ssh-keys/secret/2i2c-aws-us.key @@ -0,0 +1,21 @@ +{ + "data": "ENC[AES256_GCM,data:6i3uRRze7MnHVZ3RHR/i9VnABJ4TR1n192P7gu+YwSuTFIg9+fG9sj/IhU+OK2jkJf1w+160WoifILDXh92it9vQEPPurmsN8R0wvTd/XiGSsvbnWSxg4cS2NHzNBv6r4LftJgJ78E/uhKnNag0/2AI9Y3k3IZBHWRWNFsd/eazqXFM8zrF3K+mbqg1hm64tlQHbb2SQAGXt7x0tIL3ekkzB6+qQXc+jFqqh8eU5zLpprOiK6HanK+OpSeh6/3aMosndKPStnEZ1fwMcxVR6h0nk7VZlfK5NKIQjuQhISS6qarBGGTJYX+51zxS9T8AqUxwDLpJ7xQ68ixJFL6+CRWxceGsqjOd2J9j5/Jayl+3d62bumrI6idvoVzV2LiSMbsp5FQICE6wEtLzKd3YGAQZROAXSMXmHaz022G4bib6EvtB9RpgJDVKHyCfU7Dp0RqkLQhDCEcJJMyQZ1nWpLJlDDGuPYT6iujqkovCqH6LxvLrPNEVp+8pxTKaogt8nm33YlXgwxj/KqzDY02TnTWgWwL3GoBHW1Udg7Kxk2uYGhLcZlelIZErX24W79gSIXVx+mMVlqu+zpqXgrf45v5r6yg/1UGjmERUidMERWcJg++c53ermoSFMGUki7WRWK1Md52A1UVgUxqnDpRSBUTRoAnUU91oXv+oSCZFbdfYC9q6M0RA23HMkC837bLr6S+cswOwf/E3Fd4stozpcxEq4ocEb3lBwix4tblimLTxgqRpCHPCQgCgWgMMWQHDzM6yNVxyrTMK8ljxsY8DAQwMV52VT5LaSSBCzK074T4BoqbcGw4N5098bR0XdfoOEfXPkki/FyUkuRRz2dStCOTzWIIv3mbLENEyoGyDFOqhXCBLV3Lqm7nCTjJX0yaIDPO6jF8L7fThUPQN6rPznVgH8Dvi3iR71l9ALSfQF1+EtD+10cT6bHKU5aGjeuAgxFa/q5V/NkhY6cUR4S9xO5BB/piOyWehpP4hR0b7mGKoH2CD0Wvot2hMBywITVwRAaOcmWhGGw6B1W+jeMAAYgdKtJ18c+Nm8N+3M+rpIdLVyaTffGZxELAhM1SYTqjBj1IUZ3HzqmwDzu444o4nhktKMWHgdmtbbdhi3b30LzuAejfhY3T3hKgt2FqjycFfZ+9Oydt4LP21HW+Q9Xr2yzptXXOThC6/iLthKpHueL6PVGON6eOypOl0depKlNzFu3Se1u76BXaNvJ+KaQlAS7AdPnDR2D3naXmnp/XLyk+KKbTUI60P+ZnEhhYJp0BNHjoI5b6goknU7GKJzCwKyW3B47dR/vm82MohI8b+ztyh8iJYDsUluIUfSBIo+/2rRulB5yHUVXMjVnCVy0ly7wTxrZ/RjOSAwDgYky8O0gUBkA5TXMLaOb8o5ivKLQLlYX5jOroe37tbI2v9RDIwYfzGK1r7q4V26GP5IxyqRaWGWLrQJZtjMvpek1M5+rIR9GppfitSh21lH3Gru9Jt+O9UyRB8dFlqm3S1wxYOanW0ltqPiExUoVBTLLHbU/PGZwlVSAhTQo8g0dkKSvYxnXa4kraJ3g9b9y275+Z/0NV8QZf1cGDSGTYEw626iXq7CwY5nw95x9IjrIIe/XaPHErL+dcda1k7/YUOPEgb0C/r5bZRmTgURXVpbAsKrOpVaKT2px08nNDkEzAqdyd3pcVsI2niy7UgRQS9J6SdD3JpCRdjh5RTCypdm/b2XMLGGqIi/NYWwf9IK1TR5XtjpoAUCwJ12tY79GPzC8mGBFsYebaPMF8HJtGkamzlJMVAIxzlJr7dbKscdU0DQrxMG9avmSq4UotEOWWQA1uhhGnEeCFeIUI1+a3k/9wCmR4JzMF2f1uc9o8LK/+VkVQp0OmMHPEXilMo0UX+m/gQ/lFzbjXgsPPGuelujcIurkfDZrXn12eQkq0dQcYBosztpcqBbeTRCScPl6GTqF7lsP445NDF/L+Wv0mz5oje7qs0zgUIiC0v9QzprvlmophFeUhNcYEVJ/BrzO13BLtAI+Q9Jkpo6VNiWSdJsR24nWwcbaRrL7WEOh9hkGvVvZ/SOI7x6MhMExvj5rXlvp30fbXw1Qnt6Rhzt1JlIvTbbCGOvpH1sdeuHa+th9tjsjQGVBrdqjSCgtwLz/uTWf5I4gN6Eu4f7SZhuFf09NS+cuHIQxpIDsJ0cRO6PKrNb5QN4fxqCSpMl243ZOo3Ffg9AQO3yMNmHfeFqUIhQV5pGGIgZhAQ/ReJ2wTEsQ3vncLFmoCZJUMs35tiTR0dGXWH3/f4G76TMc+9go/sSRO81SpGbsO7iKWeI1nt2gaTeeJZNYBMHURp8+otTNwixyXelRfc9SDNG6fKuMrO7PcHogVogdyLGpwVwsGI6hwudnKk9FwaSkGYIickkH0us3GvUoXU6D+niTUI1FXE4N3d1HvG1xre/mut1e/WCmWvszwi3Qd/jZzoy0szwAH8Xb0Cj/EC+O+VwRuo1VbpX09gCVYIOA3hRMRSCu8xCbUK9Otmg93HDPsgFcAGHxsTXK6dLWK4/BMKK4AdINaZ/V1iheP0CC7pDhiMXqeZ+5hMIfo8UnEPh3hFFBlNrLGoad3/KoTL3Zdq9d3KSsv/3PB6D68lzhTibWIbJytDi1nfg8JhmWAh7KnVR4AhgAu9OF6frn3jUkHJpiW0QDbwGS5gTsd/9qinXyICMtlmWKs98Xz/NWCl9P0eSJMwG1B9fOVCCQCIjw3jzscKm+UY3ph+D5j7Chz6TDAZSmcUy2qBEDOPSzmzr0fhoHqgV0W+Lu776EoDl8PnWXNlyhPXrfweZ2M6I5h0D4aOmfuYX5/GnTNm2YH4AXc5nIpaAeTYh89YS0+JXpfVdJZk7pq8R1EaiuH3aiLqd+A/uAvwyUrNiif2rjY4hqu1hZ30vFlAiL5u8FtkDtsP5OgSQrojbPi+XYKX+QobF2RzzHpIRhyQHaPJlOHQ4NcKGKIJj6vi560EVTGspYb8mrPFvgV7wYY0QXXRwA8KIjv4sGIs30CgeyRWSxm8VUbKgzX+2cqe8Ynons/SNx9wLXajED+vM6Ss4yjTN2zwhHDKmkwzHakIWNxDAZ9EcUwe1YJKCWXMrDvOzvCNiRdTR2VbevBe64ngd0KuIFRHCtngrilJHdXjZ1lLvhYmgTK2rr9VjCu+4LeSMfNNDOELcbHAIn4toX55X3F9w4HNvxogsIgotYrMAuJxZghlyFUYeco85XCKeUiAbo9HO01aEWWqivWYPZeVRi57XP8tegZtPMiPhjUkSaFiK+KPbwoc+iLO+D10METHsqFGVltgCeqcQRHfOf4gNeHPKtyRR85n918U4b6kZ1rdYyG2BtfkK8iWRDLM9QLveJqjD2EoJWegmZvQqSJIQGlKQzmjyF/99V9Ba1LFI/VU/wm2Yz+eNmV2j1v4X1E+An+zpVS0mkLQ656dHIPMg6Q==,iv:MdZKij6wQXdl0gVbAkOFqLuiCEHesv+ZLVeoK3zKee8=,tag:3N/UVnR2vAqkoaJMwq+EmA==,type:str]", + "sops": { + "kms": null, + "gcp_kms": [ + { + "resource_id": "projects/two-eye-two-see/locations/global/keyRings/sops-keys/cryptoKeys/similar-hubs", + "created_at": "2023-03-13T01:03:44Z", + "enc": "CiUA4OM7eDPJQVDE7NEVqjWdmXpOQTl+7QK5hGpY3AdSZ3yEw45JEkkALQgViChPhBj8tmk9q6QdVCMb4VivkTQireyREkccm5p2zKgsPCOXz/xmV8FoFKOSv1MEcOjRSR2jyAzAZ7v6+1cpSkJbqVvQ" + } + ], + "azure_kv": null, + "hc_vault": null, + "age": null, + "lastmodified": "2023-03-13T01:03:45Z", + "mac": "ENC[AES256_GCM,data:YP8wuY4D1dYvWgSPLySqXnZAtmDC3Ti8fBILmu4uUXw+rpYu2tkuhMUYDaWtfSm7K4wZXkWgHkOyn1PRuI8d6SZxscup0rgGeGM9M7dNFoGQtKec0JMLpkmUdnTBqJQ/9U4agp/4CcqAryYn1pIIqmf9cYd/PiCl/uiIzuDtTJY=,iv:wSsby3xmTALAKzZ08M2Y+gGf/2SZelb5V7sdnaTjG8Y=,tag:QJmC828jRr+i52bpuJDNUQ==,type:str]", + "pgp": null, + "unencrypted_suffix": "_unencrypted", + "version": "3.7.2" + } +} \ No newline at end of file