Skip to content

add regexp support #33

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
giuliocalzo merged 4 commits into from
Oct 28, 2019
Merged

add regexp support #33

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e0f6005
add regexp support
Oct 16, 2019
74b5354
removed test file
Oct 23, 2019
469c02f
remove duplicated class
Oct 23, 2019
b100402
fix typo
Oct 23, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 6 additions & 8 deletions es-cleanup.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,11 @@
"""
This AWS Lambda function allowed to delete the old Elasticsearch index
"""
import re
import os
import json
import time
import boto3
import datetime

from botocore.auth import SigV4Auth
from botocore.awsrequest import AWSRequest
from botocore.credentials import create_credential_resolver
Expand Down Expand Up @@ -51,14 +50,13 @@ def __init__(self, event, context):

self.cfg = {}
self.cfg["es_endpoint"] = self.get_parameter("es_endpoint")
self.cfg["index"] = self.get_parameter("index", "all").split(",")
self.cfg["skip_index"] = self.get_parameter("skip_index", ".kibana").split(",")
self.cfg["index"] = self.get_parameter("index", ".*")
self.cfg["skip_index"] = self.get_parameter("skip_index", ".kibana*")

self.cfg["delete_after"] = int(self.get_parameter("delete_after", 15))
self.cfg["es_max_retry"] = int(self.get_parameter("es_max_retry", 3))
self.cfg["index_format"] = self.get_parameter(
"index_format", "%Y.%m.%d")
self.cfg["sns_alert"] = self.get_parameter("sns_alert", "")

if not self.cfg["es_endpoint"]:
raise Exception("[es_endpoint] OS variable is not set")
Expand Down Expand Up @@ -168,7 +166,7 @@ def lambda_handler(event, context):
days=int(es.cfg["delete_after"]))
for index in es.get_indices():
print("Found index: {}".format(index["index"]))
if index["index"] in es.cfg["skip_index"]:
if re.search(es.cfg["skip_index"], index["index"]):
# ignore .kibana index
continue

Expand All @@ -177,15 +175,15 @@ def lambda_handler(event, context):
idx_name = idx_split[0]
idx_date = '-'.join(word for word in idx_split[1:])

if idx_name in es.cfg["index"] or "all" in es.cfg["index"]:
if re.search(es.cfg["index"], index["index"]):

if idx_date <= earliest_to_keep.strftime(es.cfg["index_format"]):
print("Deleting index: {}".format(index["index"]))
es.delete_index(index["index"])
else:
print("Keeping index: {}".format(index["index"]))
else:
print("Index {} name {} did not match pattern {}".format(index["index"], idx_name, es.cfg["index"]))
print("Index '{}' name '{}' did not match pattern '{}'".format(index["index"], idx_name, es.cfg["index"]))


if __name__ == '__main__':
Expand Down
5 changes: 3 additions & 2 deletions serverless.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,8 @@ provider:
name: ${env:S3_DEPLOYMENT_BUCKET}
environment:
es_endpoint: search-es-demo-zveqnhnhjqm5flntemgmx5iuya.eu-west-1.es.amazonaws.com
index: all
index: ".*"
skip_index: ".kibana*"
delete_after: "7"
index_format: "%Y.%m.%d"
stackTags: # Optional CF stack tags
Expand All @@ -31,7 +32,7 @@ functions:
handler: es-cleanup.lambda_handler
name: es-cleanup-lambda
description: ES old index removal
runtime: python2.7
runtime: python3.7
memorySize: "128"
timeout: 300
package:
Expand Down