-
Notifications
You must be signed in to change notification settings - Fork 7
/
sftp_handler.py
76 lines (66 loc) · 3.24 KB
/
sftp_handler.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
import logging
import os
import pysftp
from datetime import datetime, timedelta
# Create logger
logger = logging.getLogger("__main__.sftp_handler")
def get_sftp_credentials():
# SFTP credentials
cnopts = pysftp.CnOpts()
cnopts.hostkeys = None
sftp_server = os.getenv("SFTP_SERVER")
sftp_username = os.getenv("SFTP_USERNAME")
sftp_password = os.getenv("SFTP_PASSWORD")
sftp_path = os.getenv("SFTP_PATH")
return cnopts, sftp_server, sftp_username, sftp_password, sftp_path
def sftp_instagram_dump(operation):
cnopts, sftp_server, sftp_username, sftp_password, sftp_path = get_sftp_credentials()
with pysftp.Connection(sftp_server, username=sftp_username, password=sftp_password, cnopts=cnopts) as sftp:
with sftp.cd(sftp_path):
if operation == 'l':
logger.info("Loading dump settings from SFTP")
sftp.get("remote_dump.json", "local_dump.json")
elif operation == 'c':
logger.info("Copying dump settings to SFTP")
sftp.put("local_dump.json", "remote_dump.json")
sftp.close()
def sftp_checkpoint_utils(operation, new_posts = set()):
cnopts, sftp_server, sftp_username, sftp_password, sftp_path = get_sftp_credentials()
current_date = datetime.utcnow().strftime('%Y-%m-%d')
filename = f'sent_posts_{current_date}.txt'
lookup_days = 7
# Connect to SFTP server
with pysftp.Connection(sftp_server, username=sftp_username, password=sftp_password, cnopts=cnopts) as sftp:
if operation == 'r':
with sftp.cd(sftp_path):
# Look for latest checkpoint files
for i in range(lookup_days):
filename = f'sent_posts_{(datetime.utcnow() - timedelta(days=i)).strftime("%Y-%m-%d")}.txt'
if sftp.exists(filename):
logger.info("Reading checkpoint file in SFTP server")
with sftp.open(filename, 'r') as f:
sent_posts = set([line.strip() for line in f.readlines()])
break
elif (datetime.utcnow() - timedelta(days=lookup_days)) > datetime.strptime(filename.split("_")[-1].split(".")[0], '%Y-%m-%d'):
break
else:
sent_posts = set()
return sent_posts
elif operation == 'w':
if new_posts:
with sftp.cd(sftp_path):
logger.info("Updating checkpoint file in SFTP server")
with sftp.open(filename, 'a') as f:
f.write('\n'.join(new_posts))
f.write('\n')
else:
logger.info("No new Instagram posts")
elif operation == 'd':
with sftp.cd(sftp_path):
for file in sftp.listdir():
if file.startswith('sent_posts_'):
mtime = sftp.stat(file).st_mtime
if (datetime.now() - datetime.fromtimestamp(mtime)).days > lookup_days:
logger.info(f"Deleting checkpoint files older than {lookup_days} days from SFTP server")
sftp.remove(file)
sftp.close()