Merge branch 'dev' into pyup-update-tox-3.14.0-to-3.20.1

Procfile

@@ -1 +1,2 @@
-bot: python bot/main.py
+bot: python bot/main.py
+debug: sh -c 'tox -e extended && python bot/main.py'

app.json

@@ -47,15 +47,23 @@
     },
     "INCLUDE_REPLY_TO_USER": {
       "description": "Include replies **from other** Twitter users tweeting **TO** your tracked twitter user?\nExample: `True`",
-      "required": false
+      "value": "true"
     },
     "INCLUDE_RETWEET": {
-      "description": "Include retweets?\nExample: `False`",
-      "required": false
+      "description": "Include retweets?\nExample: `False`\nWhether the bot should show retweets.",
+      "value": "true"
+    },
+    "IncludeAttachment": {
+      "description": "Include attachments?\nExample: `False`\nIf the tweet contains media the bot will try to attach an image.",
+      "value": "true"
+    },
+    "IncludeQuote": {
+      "description": "Include retweets?\nExample: `False`\nThis requires INCLUDE_RETWEET True. Shows the original tweet as an additional field in the embed.",
+      "value": "true"
     },
     "INCLUDE_USER_REPLY": {
       "description": "Include replies **FROM** your tracked twitter user to other twitter users\nExample: `True`",
-      "required": false
+      "value": "true"
     },
     "CUSTOM_MESSAGE": {
       "description": "OPTIONAL: Custom message. Useful for pinging @everyone, a role or individual discord members.\nExample: `A new tweet arived pinging role <@&123456> and user <@123456>`\nWith 123456 being the role or user id. You can get them by e.g. typing `\\@Linley#8686` the backslash \\ is important!",

bot/config.yml

@@ -39,6 +39,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER!b}
     IncludeRetweet: ${INCLUDE_RETWEET!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT!b}
+    IncludeQuote: ${INCLUDE_QUOTE!b}
     custom_message: ${CUSTOM_MESSAGE}
     keyword_sets: ${KEYWORDS!ll}
     blackword_sets: ${BLACKWORDS!ll}
@@ -52,6 +54,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_2!b}
     IncludeRetweet: ${INCLUDE_RETWEET_2!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_2!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_2!b}
+    IncludeQuote: ${INCLUDE_QUOTE_2!b}
     custom_message: ${CUSTOM_MESSAGE_2}
     keyword_sets: ${KEYWORDS_2!ll}
     blackword_sets: ${BLACKWORDS_2!ll}
@@ -65,6 +69,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_3!b}
     IncludeRetweet: ${INCLUDE_RETWEET_3!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_3!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_3!b}
+    IncludeQuote: ${INCLUDE_QUOTE_3!b}
     custom_message: ${CUSTOM_MESSAGE_3}
     keyword_sets: ${KEYWORDS_3!ll}
     blackword_sets: ${BLACKWORDS_3!ll}
@@ -78,6 +84,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_4!b}
     IncludeRetweet: ${INCLUDE_RETWEET_4!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_4!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_4!b}
+    IncludeQuote: ${INCLUDE_QUOTE_4!b}
     custom_message: ${CUSTOM_MESSAGE_4}
     keyword_sets: ${KEYWORDS_4!ll}
     blackword_sets: ${BLACKWORDS_4!ll}
@@ -91,6 +99,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_5!b}
     IncludeRetweet: ${INCLUDE_RETWEET_5!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_5!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_5!b}
+    IncludeQuote: ${INCLUDE_QUOTE_5!b}
     custom_message: ${CUSTOM_MESSAGE_5}
     keyword_sets: ${KEYWORDS_5!ll}
     blackword_sets: ${BLACKWORDS_5!ll}
@@ -104,6 +114,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_6!b}
     IncludeRetweet: ${INCLUDE_RETWEET_6!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_6!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_6!b}
+    IncludeQuote: ${INCLUDE_QUOTE_6!b}
     custom_message: ${CUSTOM_MESSAGE_6}
     keyword_sets: ${KEYWORDS_6!ll}
     blackword_sets: ${BLACKWORDS_6!ll}
@@ -117,6 +129,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_7!b}
     IncludeRetweet: ${INCLUDE_RETWEET_7!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_7!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_7!b}
+    IncludeQuote: ${INCLUDE_QUOTE_7!b}
     custom_message: ${CUSTOM_MESSAGE_7}
     keyword_sets: ${KEYWORDS_7!ll}
     blackword_sets: ${BLACKWORDS_7!ll}
@@ -130,6 +144,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_8!b}
     IncludeRetweet: ${INCLUDE_RETWEET_8!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_8!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_8!b}
+    IncludeQuote: ${INCLUDE_QUOTE_8!b}
     custom_message: ${CUSTOM_MESSAGE_8}
     keyword_sets: ${KEYWORDS_8!ll}
     blackword_sets: ${BLACKWORDS_8!ll}
@@ -143,6 +159,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_9!b}
     IncludeRetweet: ${INCLUDE_RETWEET_9!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_9!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_9!b}
+    IncludeQuote: ${INCLUDE_QUOTE_9!b}
     custom_message: ${CUSTOM_MESSAGE_9}
     keyword_sets: ${KEYWORDS_9!ll}
     blackword_sets: ${BLACKWORDS_9!ll}
@@ -156,6 +174,8 @@ Discord:
   - IncludeReplyToUser: ${INCLUDE_REPLY_TO_USER_10!b}
     IncludeRetweet: ${INCLUDE_RETWEET_10!b}
     IncludeUserReply: ${INCLUDE_USER_REPLY_10!b}
+    IncludeAttachment: ${INCLUDE_ATTACHMENT_10!b}
+    IncludeQuote: ${INCLUDE_QUOTE_10!b}
     custom_message: ${CUSTOM_MESSAGE_10}
     keyword_sets: ${KEYWORDS_10!ll}
     blackword_sets: ${BLACKWORDS_10!ll}

bot/main.py

@@ -1,4 +1,4 @@
-#!/usr/local/bin/python
+#!/usr/bin/env python
 # -*- coding: utf-8 -*-
 from tweepy.streaming import StreamListener
 from tweepy import Stream
@@ -52,6 +52,7 @@ def _on_status(self, status):
 
             for wh_url in data_discord.get("webhook_urls", []):
                 p.create_embed()
+                p.attach_field()
                 p.attach_media()
 
                 p.send_message(wh_url)
@@ -71,8 +72,9 @@ def on_status(self, status):
                 f"---------Error---------\n"
                 f"unknown error\n"
                 f"You've found an error. Please contact the owner (https://discord.gg/JV5eUB) "
-                f"and send him what follows below:\n"
+                f"and send him what follows below in a pastebin (no picture!):\n"
                 f"{error}\n"
+                f"{self.config_discord}\n"
                 f"{status}"
                 f"-----------------------"
             )
@@ -123,7 +125,7 @@ def print_error(_error):
                 f"---------Error---------\n"
                 f"unknown error\n"
                 f"You've found an error. Please contact the owner (https://discord.gg/JV5eUB) "
-                f"and send him what follows below:\n"
+                f"and send him what follows below in a pastebin (no picture!):\n"
                 f"{error}\n"
                 f"{config}\n"
                 f"Sleeping for 5 minute then continuing.\n"

bot/utils/processor.py

@@ -209,14 +209,14 @@ def get_text(self):
         else:
             self.text = self.status_tweet["text"]
 
-        for url in self.status_tweet["entities"]["urls"]:
+        for url in self.status_tweet["entities"].get("urls", []):
             if url["expanded_url"] is None:
                 continue
             self.text = self.text.replace(
                 url["url"], "[%s](%s)" % (url["display_url"], url["expanded_url"])
             )
 
-        for userMention in self.status_tweet["entities"]["user_mentions"]:
+        for userMention in self.status_tweet["entities"].get("user_mentions", []):
             self.text = self.text.replace(
                 "@%s" % userMention["screen_name"],
                 "[@%s](https://twitter.com/%s)"
@@ -225,7 +225,7 @@ def get_text(self):
 
         if "extended_tweet" in self.status_tweet:
             for hashtag in sorted(
-                self.status_tweet["extended_tweet"]["entities"]["hashtags"],
+                self.status_tweet["extended_tweet"]["entities"].get("hashtags", []),
                 key=lambda k: k["text"],
                 reverse=True,
             ):
@@ -235,7 +235,9 @@ def get_text(self):
                 )
 
         for hashtag in sorted(
-            self.status_tweet["entities"]["hashtags"], key=lambda k: k["text"], reverse=True
+            self.status_tweet["entities"].get("hashtags", []),
+            key=lambda k: k["text"],
+            reverse=True,
         ):
             self.text = self.text.replace(
                 "#%s" % hashtag["text"],
@@ -250,8 +252,47 @@ def keyword_set_present(self):
     def blackword_set_present(self):
         return blackword_set_present(self.discord_config.get("blackword_sets", [[""]]), self.text)
 
+    def attach_field(self):
+        if self.discord_config.get("IncludeQuote", True) and "quoted_status" in self.status_tweet:
+            if self.status_tweet["quoted_status"].get("text"):
+                text = self.status_tweet["quoted_status"]["text"]
+                for url in self.status_tweet["quoted_status"]["entities"].get("urls", []):
+                    if url["expanded_url"] is None:
+                        continue
+                    text = text.replace(
+                        url["url"], "[%s](%s)" % (url["display_url"], url["expanded_url"])
+                    )
+
+                for userMention in self.status_tweet["quoted_status"]["entities"].get(
+                    "user_mentions", []
+                ):
+                    text = text.replace(
+                        "@%s" % userMention["screen_name"],
+                        "[@%s](https://twitter.com/%s)"
+                        % (userMention["screen_name"], userMention["screen_name"]),
+                    )
+
+                for hashtag in sorted(
+                    self.status_tweet["quoted_status"]["entities"].get("hashtags", []),
+                    key=lambda k: k["text"],
+                    reverse=True,
+                ):
+                    text = text.replace(
+                        "#%s" % hashtag["text"],
+                        "[#%s](https://twitter.com/hashtag/%s)"
+                        % (hashtag["text"], hashtag["text"]),
+                    )
+
+                text = unescape(text)
+                self.embed.add_field(
+                    name=self.status_tweet["quoted_status"]["user"]["screen_name"], value=text
+                )
+
     def attach_media(self):
-        if "retweeted_status" in self.status_tweet:
+        if (
+            self.discord_config.get("IncludeAttachment", True)
+            and "retweeted_status" in self.status_tweet
+        ):
             if (
                 "extended_tweet" in self.status_tweet["retweeted_status"]
                 and "media" in self.status_tweet["retweeted_status"]["extended_tweet"]["entities"]

bot/utils/startup.py

@@ -142,6 +142,8 @@ def pprint(config):
             ("Include Reply To User", instance.get("IncludeReplyToUser", True)),
             ("Include User Reply", instance.get("IncludeUserReply", True)),
             ("Include Retweet", instance.get("IncludeRetweet", True)),
+            ("Include Quote", instance.get("IncludeQuote", True)),
+            ("Include Attachment", instance.get("IncludeAttachment", True)),
             ("Custom Message", True if instance.get("custom_message", "") is not "" else False),
             ("Keyword Sets", True if instance.get("keyword_sets", "") is not "" else False),
             ("Blackword Sets", True if instance.get("blackword_sets", "") is not "" else False),

requirements.txt

@@ -1,8 +1,8 @@
 black==18.9b0
-discord.py==1.5.0
+discord.py==1.5.1
 jsonschema==3.2.0
-python-dotenv==0.14.0
+python-dotenv==0.15.0
 PyYAML==5.3.1
 tox==3.20.1
-tweepy==3.8.0
-virtualenv==20.0.33
+tweepy==3.9.0
+virtualenv==20.1.0

tests/test_config/config_schema.json

@@ -93,6 +93,12 @@
           },
           "IncludeUserReply": {
             "type": "boolean"
+          },
+          "IncludeQuote": {
+            "type": "boolean"
+          },
+          "IncludeAttachment": {
+            "type": "boolean"
           }
         },
         "additionalProperties": false,