Skip to content

Commit 262bc74

Browse files
alessioviertialessiovierti
committed
Initial commit
0 parents  commit 262bc74

File tree

14 files changed

+1093
-0
lines changed

14 files changed

+1093
-0
lines changed

‎.github/workflows/create-release.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: Update version and create release
2+
3+
on:
4+
pull_request:
5+
types: [closed]
6+
branches:
7+
- master
8+
9+
jobs:
10+
11+
fetch-version:
12+
if: github.event.pull_request.merged
13+
runs-on: ubuntu-latest
14+
steps:
15+
- uses: actions/checkout@v2
16+
- name: Fetch latest release version
17+
id: fetch-latest-release
18+
uses: reloc8/action-latest-release-version@1.0.0
19+
- uses: actions/setup-python@v2
20+
with:
21+
python-version: 3.7
22+
- name: Choose new release version
23+
id: choose-release-version
24+
uses: reloc8/action-choose-release-version@1.0.0
25+
with:
26+
source-branch: ${{ github.event.pull_request.head.ref }}
27+
latest-version: ${{ steps.fetch-latest-release.outputs.latest-release }}
28+
outputs:
29+
new-version: ${{ steps.choose-release-version.outputs.new-version }}
30+
31+
update-version:
32+
needs: fetch-version
33+
runs-on: ubuntu-latest
34+
steps:
35+
- uses: actions/checkout@v2
36+
- run: git pull --ff-only
37+
- name: Update version file
38+
run: echo ${{ needs.fetch-version.outputs.new-version }} > version
39+
- name: Push local repository changes
40+
id: push-local-repository-changes
41+
uses: reloc8/action-push-local-changes@1.0.0
42+
env:
43+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
44+
with:
45+
commit-message: "Version ${{ needs.fetch-version.outputs.new-version }}"
46+
outputs:
47+
commit-hash: ${{ steps.push-local-repository-changes.outputs.commit-hash }}
48+
49+
create-release:
50+
needs: [fetch-version, update-version]
51+
runs-on: ubuntu-latest
52+
steps:
53+
- uses: actions/checkout@v2
54+
- run: git pull --ff-only
55+
- name: Create new release
56+
uses: actions/create-release@v1
57+
env:
58+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
59+
with:
60+
tag_name: ${{ needs.fetch-version.outputs.new-version }}
61+
release_name: ${{ needs.fetch-version.outputs.new-version }}
62+
draft: false
63+
prerelease: false
64+
commitish: ${{ needs.update-version.outputs.commit-hash }}

‎.gitignore‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
.DS_Store
2+
.idea
3+
.venv
4+
*.egg-info
5+
stack
6+
cdk.out
7+
build
8+
dist
9+
__pycache__

‎LICENSE.txt‎

Lines changed: 661 additions & 0 deletions
Large diffs are not rendered by default.

‎README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
## Development
2+
3+
1. Create a virtual environment:
4+
5+
`$ python3 -m venv .venv`
6+
7+
2. Activate the created environment:
8+
9+
`$ source .venv/bin/activate`
10+
11+
3. Upgrade `pip`:
12+
13+
`$ python3 -m pip install --upgrade pip`
14+
15+
4. Install the requirements:
16+
17+
`$ pip install --upgrade -r requirements.txt`
18+
19+
5. Mark the main package as Sources Root.
20+
21+
6. Install the stack dependencies:
22+
23+
`$ python3 install.py`
24+
25+
7. Try to synthesise the CloudFormation template:
26+
27+
`$ cdk synth`
28+
29+
## Test
30+
31+
1. Install the testing requirements:
32+
33+
`$ pip install --upgrade -r requirements-test.txt`
34+
35+
2. Run all tests in package `tests`

‎app.py‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
#!/usr/bin/env python3
2+
3+
from aws_cdk import (
4+
core,
5+
aws_dynamodb as dynamodb
6+
)
7+
8+
from data_processing_stack import DataProcessingStack, MongoDBConfiguration, AccessKeysConfiguration, ImportedAssetsConfiguration
9+
10+
11+
class MockDataMiningStack(core.Stack):
12+
13+
def __init__(self, scope: core.Construct, id_: str, **kwargs):
14+
15+
super().__init__(scope, id_, **kwargs)
16+
17+
self.property_table = dynamodb.Table(
18+
self,
19+
'Property',
20+
table_name='Property',
21+
partition_key=dynamodb.Attribute(
22+
name='id',
23+
type=dynamodb.AttributeType.STRING
24+
),
25+
stream=dynamodb.StreamViewType.NEW_IMAGE
26+
)
27+
28+
29+
app = core.App()
30+
mock_data_mining_stack = MockDataMiningStack(app, 'MockDataMiningStack')
31+
data_processing_stack = DataProcessingStack(
32+
app,
33+
'DataProcessingStack',
34+
mongodb_config=MongoDBConfiguration(
35+
uri='MockUri',
36+
max_page_size='10'
37+
),
38+
access_keys_config=AccessKeysConfiguration(
39+
geocoding='MockAccessKey'
40+
),
41+
imported_assets_config=ImportedAssetsConfiguration(
42+
table_property=mock_data_mining_stack.property_table
43+
)
44+
)
45+
app.synth()

‎cdk.json‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
{
2+
"app": "python3 app.py",
3+
"context": {
4+
"@aws-cdk/core:enableStackNameDuplicates": "true",
5+
"aws-cdk:enableDiffNoFail": "true",
6+
"@aws-cdk/core:stackRelativeExports": "true"
7+
}
8+
}

‎data_processing_stack/__init__.py‎

Lines changed: 191 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,191 @@
1+
from aws_cdk import (
2+
aws_sns as sns,
3+
aws_sns_subscriptions as sns_subscriptions,
4+
aws_lambda as lambda_,
5+
aws_logs as logs,
6+
aws_apigateway as api_gateway,
7+
aws_lambda_event_sources as event_sources,
8+
aws_dynamodb as dynamodb,
9+
core
10+
)
11+
from dataclasses import dataclass
12+
from typing import AnyStr
13+
14+
15+
DEFAULT_LAMBDA_HANDLER = 'main.lambda_handler'
16+
DEFAULT_LAMBDA_RUNTIME = lambda_.Runtime.PYTHON_3_8
17+
DEFAULT_LAMBDA_LOG_RETENTION = logs.RetentionDays.ONE_WEEK
18+
19+
20+
@dataclass
21+
class MongoDBConfiguration:
22+
23+
uri: AnyStr
24+
max_page_size: AnyStr
25+
26+
27+
@dataclass
28+
class AccessKeysConfiguration:
29+
30+
geocoding: AnyStr
31+
32+
33+
@dataclass
34+
class ImportedAssetsConfiguration:
35+
36+
table_property: dynamodb.Table
37+
38+
39+
class DataProcessingStack(core.Stack):
40+
41+
def __init__(self, scope: core.Construct, id_: str,
42+
imported_assets_config: ImportedAssetsConfiguration,
43+
mongodb_config: MongoDBConfiguration,
44+
access_keys_config: AccessKeysConfiguration,
45+
**kwargs):
46+
47+
super().__init__(scope, id_, **kwargs)
48+
49+
# LAMBDAS DEFINITIONS
50+
51+
lambda_dispatch_stream = lambda_.Function(
52+
self,
53+
'DispatchStream',
54+
code=lambda_.AssetCode('stack/lambda/dispatch_stream/1.0.0/python/dispatch_stream'),
55+
timeout=core.Duration.seconds(10),
56+
description='',
57+
function_name='DispatchStream',
58+
reserved_concurrent_executions=10,
59+
handler=DEFAULT_LAMBDA_HANDLER,
60+
runtime=DEFAULT_LAMBDA_RUNTIME,
61+
log_retention=DEFAULT_LAMBDA_LOG_RETENTION,
62+
memory_size=128,
63+
retry_attempts=0,
64+
dead_letter_queue_enabled=False
65+
)
66+
67+
lambda_geocode_property = lambda_.Function(
68+
self,
69+
'GeocodeProperty',
70+
code=lambda_.AssetCode('stack/lambda/geocode_property/1.0.0/python/geocode_property'),
71+
timeout=core.Duration.seconds(15),
72+
description='',
73+
function_name='GeocodeProperty',
74+
reserved_concurrent_executions=10,
75+
handler=DEFAULT_LAMBDA_HANDLER,
76+
runtime=DEFAULT_LAMBDA_RUNTIME,
77+
log_retention=DEFAULT_LAMBDA_LOG_RETENTION,
78+
memory_size=128,
79+
retry_attempts=0,
80+
dead_letter_queue_enabled=True
81+
)
82+
83+
lambda_fetch_properties = lambda_.Function(
84+
self,
85+
'FetchProperties',
86+
code=lambda_.AssetCode('stack/lambda/fetch_properties/1.0.0/python/fetch_properties'),
87+
timeout=core.Duration.seconds(10),
88+
description='',
89+
function_name='FetchProperties',
90+
reserved_concurrent_executions=10,
91+
handler=DEFAULT_LAMBDA_HANDLER,
92+
runtime=DEFAULT_LAMBDA_RUNTIME,
93+
log_retention=DEFAULT_LAMBDA_LOG_RETENTION,
94+
memory_size=128,
95+
retry_attempts=0,
96+
dead_letter_queue_enabled=True
97+
)
98+
99+
# LAYERS DEFINITIONS
100+
101+
layer_dispatch_stream = lambda_.LayerVersion(
102+
self,
103+
'DispatchStreamLibs',
104+
code=lambda_.Code.from_asset('stack/lambda/dispatch_stream/1.0.0/'),
105+
description='',
106+
layer_version_name='DispatchStreamLibs',
107+
compatible_runtimes=[DEFAULT_LAMBDA_RUNTIME]
108+
)
109+
110+
layer_geocode_property = lambda_.LayerVersion(
111+
self,
112+
'GeocodePropertyLibs',
113+
code=lambda_.Code.from_asset('stack/lambda/geocode_property/1.0.0/'),
114+
description='',
115+
layer_version_name='GeocodePropertyLibs',
116+
compatible_runtimes=[DEFAULT_LAMBDA_RUNTIME]
117+
)
118+
119+
layer_fetch_properties = lambda_.LayerVersion(
120+
self,
121+
'FetchPropertiesLibs',
122+
code=lambda_.Code.from_asset('stack/lambda/fetch_properties/1.0.0/'),
123+
description='',
124+
layer_version_name='FetchPropertiesLibs',
125+
compatible_runtimes=[DEFAULT_LAMBDA_RUNTIME]
126+
)
127+
128+
# CLOUDWATCH RULES DEFINITIONS
129+
# -
130+
131+
# SQS QUEUES DEFINITIONS
132+
# -
133+
134+
# SNS TOPICS DEFINITIONS
135+
136+
topic_new_properties = sns.Topic(
137+
self,
138+
'NewProperties',
139+
display_name='',
140+
topic_name='NewProperties'
141+
)
142+
143+
# API GATEWAYS
144+
api_gateway_graphql = api_gateway.LambdaRestApi(
145+
self,
146+
'GraphQLApi',
147+
handler=lambda_fetch_properties,
148+
rest_api_name='GraphQLApi',
149+
description='GraphQL API',
150+
cloud_watch_role=True
151+
)
152+
api_gateway_graphql.root.add_resource('graphql').add_method('POST')
153+
154+
# DYNAMODB PERMISSIONS
155+
lambda_dispatch_stream.add_event_source(event_sources.DynamoEventSource(
156+
table=imported_assets_config.table_property,
157+
starting_position=lambda_.StartingPosition.LATEST,
158+
batch_size=10,
159+
max_batching_window=core.Duration.seconds(30),
160+
parallelization_factor=10,
161+
retry_attempts=0
162+
))
163+
164+
# CLOUDWATCH SCHEDULING RULES
165+
# -
166+
167+
# SQS PERMISSIONS
168+
# -
169+
170+
# SNS PERMISSIONS
171+
172+
topic_new_properties.grant_publish(lambda_dispatch_stream)
173+
topic_new_properties.add_subscription(sns_subscriptions.LambdaSubscription(lambda_geocode_property))
174+
175+
# LAYERS ASSIGNMENTS
176+
177+
lambda_dispatch_stream.add_layers(layer_dispatch_stream)
178+
lambda_geocode_property.add_layers(layer_geocode_property)
179+
lambda_fetch_properties.add_layers(layer_fetch_properties)
180+
181+
# ENVIRONMENT VARIABLES
182+
183+
lambda_geocode_property.add_environment(key='MONGODB_URI', value=mongodb_config.uri)
184+
lambda_geocode_property.add_environment(
185+
key='API_ACCESS_TOKEN_GEOCODING', value=access_keys_config.geocoding
186+
)
187+
lambda_fetch_properties.add_environment(key='MONGODB_URI', value=mongodb_config.uri)
188+
lambda_fetch_properties.add_environment(key='MONGODB_MAX_PAGE_SIZE', value=mongodb_config.max_page_size)
189+
190+
# EXPOSED ENTITIES
191+
# -

‎data_processing_stack/dependencies.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
import os
2+
from dataclasses import dataclass
3+
from typing import AnyStr
4+
5+
6+
GITHUB_PERSONAL_ACCESS_TOKEN = os.getenv('GITHUB_PERSONAL_ACCESS_TOKEN')
7+
8+
9+
def private_dependency(personal_access_token: AnyStr,
10+
repo_user: AnyStr, repo_name: AnyStr,
11+
package_name: AnyStr, package_version: AnyStr):
12+
"""Defines a dependency from a private Github repository
13+
14+
:param personal_access_token: Github Personal Access Token
15+
:param repo_user: Dependency repository user
16+
:param repo_name: Dependency repository name
17+
:param package_name: Dependency package name
18+
:param package_version: Dependency repository release (tag)
19+
:return: The dependency specification for the install_requires field
20+
"""
21+
22+
return f'{package_name} @ ' \
23+
f'git+https://{personal_access_token}@github.com/' \
24+
f'{repo_user}/{repo_name}.git/@{package_version}#egg={package_name}-0'
25+
26+
27+
@dataclass
28+
class Dependency:
29+
30+
project_name: AnyStr
31+
package_name: AnyStr
32+
release_version: AnyStr
33+
34+
35+
DEPENDENCIES = [
36+
Dependency(project_name='lambda-dispatch-stream', package_name='dispatch_stream', release_version='1.0.0'),
37+
Dependency(project_name='lambda-geocode-property', package_name='geocode_property', release_version='1.0.0'),
38+
Dependency(project_name='lambda-fetch-properties', package_name='fetch_properties', release_version='1.0.0')
39+
]

0 commit comments

Comments
 (0)