Skip to content

Commit 060fbeb

Browse files
paintdogpaintdog
committed
updating packt_notifer
1 parent 0256efd commit 060fbeb

File tree

5 files changed

+144
-115
lines changed

5 files changed

+144
-115
lines changed

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,6 @@
22

33
This tool scrapes [the Packt website](https://www.packtpub.com/packt/offers/free-learning) and informs you by email about new free learning ebooks (e. g. "Go Design Patterns"). If you allready have this book in your booklist, you will be informed otherwise you will be pleased to download the new ebook.
44

5+
Please add your data in settings.py.
6+
57
It is not a complete automated solution, but it works for me.

‎__build_booklist__.py

Lines changed: 0 additions & 32 deletions
This file was deleted.

‎booklist.txt

Whitespace-only changes.

‎main.pyw

Lines changed: 139 additions & 83 deletions
Original file line numberDiff line numberDiff line change
@@ -1,91 +1,147 @@
1-
from bs4 import BeautifulSoup
2-
from email_notification import email_notification
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
33
import datetime
4+
import logging
45
import os
5-
import requests
6-
from settings import settings
7-
8-
9-
# checking which books I allready have
10-
try:
11-
with open("booklist.txt") as f:
12-
books = [book.strip("\n") for book in f.readlines()]
13-
except FileNotFoundError:
14-
with open("booklist.txt", "w") as f:
15-
f.write()
16-
books = []
17-
18-
try:
19-
url = "https://www.packtpub.com/packt/offers/free-learning"
20-
21-
headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/50.0.2661.102 Safari/537.36'}
22-
r = requests.get(url, headers=headers)
23-
soup = BeautifulSoup(r.text, "html5lib")
24-
25-
# title
26-
title = soup.find("div", attrs={"class":"dotd-title"}).text.strip("\t\n")
27-
28-
# Summary
29-
summary_div = soup.find("div", attrs={"class":"dotd-main-book-summary"})
30-
divs = summary_div.find_all("div")
31-
summary = []
32-
for i, div in enumerate(divs):
33-
if i < 2:
34-
pass
35-
elif i == 2:
36-
summary.append(divs[2].text.strip("\n\t"))
37-
else:
38-
if not 'class="dotd-main-book-form cf"' in str(div):
39-
summary.append(divs[i].text.strip("\n\t"))
40-
summary = "\n".join(summary)
41-
42-
# image
43-
img = soup.find("img", attrs={"class" : "bookimage"})["src"].strip("/")
44-
45-
# print([title])
46-
# print([summary])
47-
# print([img])
48-
# print([url])
49-
50-
except KeyError:
51-
# ToDo -- Specify possible errors
52-
email_notification(settings, "packtbub.com | An error occurs...", f"<b>Please visit {url} manually and fix me...</b>")
53-
exit()
54-
6+
import time
557

56-
if title in books:
57-
58-
subject = f"packtpub.com | {title} | You allready have this book!"
59-
60-
text = """<h2>{title}</h2>
61-
<p>Please visit {url}!</p>
62-
<img src={img}/>
63-
<p>{summary}</p>""".format(title=title, url=url, img=img, summary=summary.replace("\n", "</p><p>"))
64-
65-
email_notification(settings, subject, text)
66-
67-
else:
68-
69-
books.append(title)
70-
71-
subject = f"packtpub.com | {title} | ✭ This book is new! ✭"
8+
import requests
729

73-
text = """<div style="margin-bottom: 10px; padding: 10px; background: Yellow; color: Red;">
74-
<strong>I added this book to your booklist. Please download this book immediately.</strong>
75-
</div>
10+
from email_notification import email_notification
11+
from settings import settings
7612

77-
<h2>{title}</h2>
78-
<p>Please visit {url}!</p>
13+
# LOGGING -------------------------------------------------------------------
14+
filename = "logfile.log"
15+
handler = logging.FileHandler(filename, "a")
16+
frm = logging.Formatter("%(asctime)s [%(levelname)-8s] [%(funcName)-20s] [%(lineno)-4s] %(message)s",
17+
"%d.%m.%Y %H:%M:%S")
18+
handler.setFormatter(frm)
19+
logger = logging.getLogger()
20+
logger.addHandler(handler)
21+
logger.setLevel(logging.DEBUG)
22+
# ---------------------------------------------------------------------------
23+
24+
URL = "https://www.packtpub.com/packt/offers/free-learning"
25+
URL_PRODUCT_ID = "https://services.packtpub.com/free-learning-v1/offers?dateFrom={}T00:00:00.000Z&dateTo={}T00:00:00.000Z"
26+
URL_SUMMARY = "https://static.packt-cdn.com/products/{}/summary"
27+
URL_AUTHOR = "https://static.packt-cdn.com/authors/{}"
28+
URL_COVER = "https://static.packt-cdn.com/products/{}/cover/smaller"
29+
30+
def get_product_id(URL_PRODUCT_ID):
31+
''' get the product id for todays free ebook '''
32+
today = datetime.date.today()
33+
tomorrow = datetime.date.today() + datetime.timedelta(1)
34+
web = requests.get(URL_PRODUCT_ID.format(today, tomorrow))
35+
data = web.json()
36+
product_id = data["data"][0]["productId"]
37+
return product_id
38+
39+
def get_summary_data(product_id, URL_SUMMARY):
40+
''' get the summary data for a product id '''
41+
web = requests.get(URL_SUMMARY.format(product_id))
42+
summary_data = web.json()
43+
return summary_data
44+
45+
def get_cover_url(product_id, URL_COVER):
46+
''' get the url to the cover of a product '''
47+
return URL_COVER.format(product_id)
48+
49+
def get_authors(author_ids, URL_AUTHOR):
50+
''' get the information about the authors '''
51+
author_text = []
52+
for author_id in author_ids:
53+
web = requests.get(URL_AUTHOR.format(author_id))
54+
author_text.append(web.json()["description"])
55+
return "".join(author_text)
56+
57+
def update_books(books):
58+
books.sort(key=str.lower)
59+
with open("booklist.txt", "w") as f:
60+
for book in books:
61+
f.write(book + "\n")
62+
63+
def main():
64+
# checking which books I allready have
65+
try:
66+
with open("booklist.txt") as f:
67+
books = [book.strip("\n") for book in f.readlines()]
68+
except FileNotFoundError:
69+
with open("booklist.txt", "w") as f:
70+
f.write()
71+
books = []
72+
73+
try:
74+
75+
product_id = get_product_id(URL_PRODUCT_ID)
76+
summary_data = get_summary_data(product_id, URL_SUMMARY)
77+
title = summary_data["title"]
78+
image_url = get_cover_url(product_id, URL_COVER)
79+
summary_text = summary_data["about"]
80+
authors = get_authors(summary_data["authors"], URL_AUTHOR)
81+
82+
logging.info("## url: {}".format(URL))
83+
logging.info("## ID: {}".format(product_id))
84+
logging.info("## title: {}".format(title))
85+
logging.info("## summary: {}...".format(summary_text[:40]))
86+
logging.info("## img_url: {}".format(image_url))
87+
88+
except KeyError:
89+
email_notification(settings, "packtbub.com | An error occured...", f"<b>Please visit {url} manually and fix me...</b>")
90+
exit()
91+
92+
if title in books:
93+
subject = f"packtpub.com | {title} | You allready have this book!"
94+
text = """<h2>{title}</h2>
95+
<p>Please visit {url}!</p>
96+
<img src='{img}' width='200px'></img>
97+
<p>Heute: [b]{title}[/b]<br><br>{url}</p>
98+
<p>{summary}</p>""".format(title=title, url=URL, img=image_url, summary=summary_text)
99+
email_notification(settings, subject, text)
100+
101+
elif title == "" or title is None:
102+
# ToDo -- Specify possible errors
103+
email_notification(settings, "packtbub.com | An error occured...", f"<b>Please visit {url} manually and fix me...</b>")
104+
exit()
105+
106+
else:
107+
books.append(title)
108+
subject = f"packtpub.com | {title} | ✭ This book is new! ✭"
109+
text = """<div style="margin-bottom: 10px; padding: 10px; background: Yellow; color: Red;">
110+
<strong>I added this book to your booklist. Please download this book immediately.</strong>
111+
</div>
112+
<h2>{title}</h2>
113+
<p>Please visit {url}!</p>
114+
<img src='{img}' width='200px'></img>
115+
<p>Heute: [b]{title}[/b]<br><br>{url}</p>
116+
<p>{summary}</p>""".format(title=title, url=URL, img=image_url, summary=summary_text)
117+
email_notification(settings, subject, text)
118+
119+
# Updating booklist
120+
update_books(books)
79121

80-
img src="{img}" title="" />
81122

82-
<p>{summary}</p>""".format(title=title, url=url, img=img, summary=summary.replace("\n", "</p><p>"))
123+
if __name__ == "__main__":
83124

84-
email_notification(settings, subject, text)
85-
86-
87-
# Updating booklist
88-
books.sort(key=str.lower)
89-
with open("booklist.txt", "w") as f:
90-
for book in books:
91-
f.write(book + "\n")
125+
logging.info("job started.")
126+
done_flag = False
127+
error_counter = 0
128+
129+
while True:
130+
try:
131+
main()
132+
done_flag = True
133+
except:
134+
logging.exception("Ein Fehler ist aufgetreten.")
135+
136+
from skynet import Report
137+
report = Report()
138+
report.set_title("packt_notifier")
139+
report.set_summary('Please check the logs, an error occured.<br>Please visit <a href="https://www.packtpub.com/packt/offers/free-learning">https://www.packtpub.com/packt/offers/free-learning</a>.')
140+
report.save()
141+
break
142+
143+
if done_flag:
144+
print("DONE")
145+
break
146+
147+
logging.info("job closed.\n\n")

‎settings.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
1+
#!/usr/bin/env python3
2+
# -*- coding: utf-8 -*-
3+
14
settings = {"senderEmail" : "FROM_ABCDEFG@WWW.COM", # your mail adress (From:)
25
"senderPasswort" : "YOUR_PASSWORT", # your password
36
"empfangsEmail" : "TO_ABCDEFG@WWW.COM", # your mail adress (To:)

0 commit comments

Comments
 (0)