From 0526f3d5bf3b5c38749cf0fd062f9638fe2f6580 Mon Sep 17 00:00:00 2001 From: NNTin Date: Sun, 24 Feb 2019 23:18:55 +0000 Subject: [PATCH 1/9] pyup.io integration --- .pyup.yml | 1 + 1 file changed, 1 insertion(+) create mode 100644 .pyup.yml diff --git a/.pyup.yml b/.pyup.yml new file mode 100644 index 00000000..4ff6ddb6 --- /dev/null +++ b/.pyup.yml @@ -0,0 +1 @@ +branch: dev From 0957f9da746d015fda2c0eb632f94bd4a881b9eb Mon Sep 17 00:00:00 2001 From: NNTin Date: Mon, 25 Feb 2019 00:54:14 +0100 Subject: [PATCH 2/9] fixed docker image fail --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 200b7de3..ef0f18a2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -71,5 +71,11 @@ jobs: branch: master repo: NNTin/discord-twitter-bot - stage: docker test - before_script: export DOCKER_TAG="$TRAVIS_BRANCH-${TRAVIS_COMMIT::8}" - script: docker pull "$DOCKER_IMAGE_NAME:$DOCKER_TAG" \ No newline at end of file + before_deploy: + - export DOCKER_TAG="$TRAVIS_BRANCH-${TRAVIS_COMMIT::8}" + deploy: + - provider: script + script: docker pull "$DOCKER_IMAGE_NAME:$DOCKER_TAG" + on: + all_branches: true + repo: NNTin/discord-twitter-bot \ No newline at end of file From ea5f3c58f283452b37fc591dc7a9ec83f1853c82 Mon Sep 17 00:00:00 2001 From: NNTin Date: Fri, 1 Mar 2019 20:29:43 +0100 Subject: [PATCH 3/9] Set up CI with Azure Pipelines --- azure-pipelines.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..1b3a64d8 --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,41 @@ +jobs: + +- job: 'Test' + pool: + vmImage: 'Ubuntu-16.04' + strategy: + matrix: + Python36-32bit-fast: + PYTHON_VERSION: '3.6' + PYTHON_ARCH: 'x86' + Python36-64bit-full: + PYTHON_VERSION: '3.6' + PYTHON_ARCH: 'x64' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '$(PYTHON_VERSION)' + architecture: $(PYTHON_ARCH) + + - script: python -m pip install --upgrade pip && pip install -r requirements.txt + displayName: 'Install dependencies' + + - script: | + python -m unittest discover -s tests/test_discord + python -m unittest discover -s tests/test_python + displayName: 'unittest' + +- job: 'Publish' + dependsOn: 'Test' + pool: + vmImage: 'Ubuntu-16.04' + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + architecture: 'x64' + + - script: echo "placeholder" + displayName: 'Placeholder' From f71bfa744fec1b43fde677ae89b85e3b600af01e Mon Sep 17 00:00:00 2001 From: EntropyInEffect <57723264+EntropyInEffect@users.noreply.github.com> Date: Wed, 13 Nov 2019 12:53:56 -0500 Subject: [PATCH 4/9] Fix retweet handling Previously, retweets would be truncated by the bot once posted. This update adds lines to handle retweets specifically. --- bot/utils/processor.py | 108 ++++++++++++++++++++++++++++------------- 1 file changed, 75 insertions(+), 33 deletions(-) diff --git a/bot/utils/processor.py b/bot/utils/processor.py index 10931f24..62c9d29c 100644 --- a/bot/utils/processor.py +++ b/bot/utils/processor.py @@ -182,7 +182,14 @@ def worth_posting_follow(self): ) def get_text(self): - if "extended_tweet" in self.status_tweet: + if "retweeted_status" in self.status_tweet: + if "extended_tweet" in self.status_tweet["retweeted_status"]: + self.text = self.status_tweet["retweeted_status"]["extended_tweet"]["full_text"] + elif "full_text" in self.status_tweet["retweeted_status"]: + self.text = self.status_tweet["retweeted_status"]["full_text"] + else: + self.text = self.status_tweet["retweeted_status"]["text"] + elif "extended_tweet" in self.status_tweet: self.text = self.status_tweet["extended_tweet"]["full_text"] elif "full_text" in self.status_tweet: self.text = self.status_tweet["full_text"] @@ -231,38 +238,73 @@ def blackword_set_present(self): return blackword_set_present(self.discord_config.get("blackword_sets", [[""]]), self.text) def attach_media(self): - if ( - "extended_tweet" in self.status_tweet - and "media" in self.status_tweet["extended_tweet"]["entities"] - ): - for media in self.status_tweet["extended_tweet"]["entities"]["media"]: - if media["type"] == "photo": - self.embed.set_image(url=media["media_url_https"]) - elif media["type"] == "video": - pass - elif media["type"] == "animated_gif": - pass - - if "media" in self.status_tweet["entities"]: - for media in self.status_tweet["entities"]["media"]: - if media["type"] == "photo": - self.embed.set_image(url=media["media_url_https"]) - elif media["type"] == "video": - pass - elif media["type"] == "animated_gif": - pass - - if ( - "extended_entities" in self.status_tweet - and "media" in self.status_tweet["extended_entities"] - ): - for media in self.status_tweet["extended_entities"]["media"]: - if media["type"] == "photo": - self.embed.set_image(url=media["media_url_https"]) - elif media["type"] == "video": - pass - elif media["type"] == "animated_gif": - pass + if "retweeted_status" in self.status_tweet: + if ( + "extended_tweet" in self.status_tweet["retweeted_status"] + and "media" in self.status_tweet["retweeted_status"]["extended_tweet"]["entities"] + ): + for media in self.status_tweet["retweeted_status"]["extended_tweet"]["entities"]["media"]: + if media["type"] == "photo": + self.embed.set_image(url=media["media_url_https"]) + elif media["type"] == "video": + pass + elif media["type"] == "animated_gif": + pass + + if "media" in self.status_tweet["retweeted_status"]["entities"]: + for media in self.status_tweet["retweeted_status"]["entities"]["media"]: + if media["type"] == "photo": + self.embed.set_image(url=media["media_url_https"]) + elif media["type"] == "video": + pass + elif media["type"] == "animated_gif": + pass + + if ( + "extended_entities" in self.status_tweet["retweeted_status"] + and "media" in self.status_tweet["retweeted_status"]["extended_entities"] + ): + for media in self.status_tweet["retweeted_status"]["extended_entities"]["media"]: + if media["type"] == "photo": + self.embed.set_image(url=media["media_url_https"]) + elif media["type"] == "video": + pass + elif media["type"] == "animated_gif": + pass + else: + if ( + "extended_tweet" in self.status_tweet + and "media" in self.status_tweet["extended_tweet"]["entities"] + ): + for media in self.status_tweet["extended_tweet"]["entities"]["media"]: + if media["type"] == "photo": + self.embed.set_image(url=media["media_url_https"]) + elif media["type"] == "video": + pass + elif media["type"] == "animated_gif": + pass + + if "media" in self.status_tweet["entities"]: + for media in self.status_tweet["entities"]["media"]: + if media["type"] == "photo": + self.embed.set_image(url=media["media_url_https"]) + elif media["type"] == "video": + pass + elif media["type"] == "animated_gif": + pass + + if ( + "extended_entities" in self.status_tweet + and "media" in self.status_tweet["extended_entities"] + ): + for media in self.status_tweet["extended_entities"]["media"]: + if media["type"] == "photo": + self.embed.set_image(url=media["media_url_https"]) + elif media["type"] == "video": + pass + elif media["type"] == "animated_gif": + pass + def create_embed(self): self.embed = Embed( From ffeb252b9dd7676ee848dee9aac694480660c98e Mon Sep 17 00:00:00 2001 From: NNTin Date: Fri, 29 Nov 2019 09:09:39 +0000 Subject: [PATCH 5/9] fix stylecheck error --- bot/utils/processor.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bot/utils/processor.py b/bot/utils/processor.py index 62c9d29c..3f7e8236 100644 --- a/bot/utils/processor.py +++ b/bot/utils/processor.py @@ -243,7 +243,9 @@ def attach_media(self): "extended_tweet" in self.status_tweet["retweeted_status"] and "media" in self.status_tweet["retweeted_status"]["extended_tweet"]["entities"] ): - for media in self.status_tweet["retweeted_status"]["extended_tweet"]["entities"]["media"]: + for media in self.status_tweet["retweeted_status"]["extended_tweet"]["entities"][ + "media" + ]: if media["type"] == "photo": self.embed.set_image(url=media["media_url_https"]) elif media["type"] == "video": @@ -304,7 +306,6 @@ def attach_media(self): pass elif media["type"] == "animated_gif": pass - def create_embed(self): self.embed = Embed( From 31a06841940c69d01a9b526265c2f77f8ee01686 Mon Sep 17 00:00:00 2001 From: NNTin Date: Mon, 1 Jun 2020 14:44:22 +0200 Subject: [PATCH 6/9] RegEx fix: Discord domain changed Discord changed their domain from discordapp.com to discord.com. Adjusted code to accept both. --- bot/utils/processor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bot/utils/processor.py b/bot/utils/processor.py index 3f7e8236..f99ece64 100644 --- a/bot/utils/processor.py +++ b/bot/utils/processor.py @@ -51,7 +51,7 @@ 0x005C73, 0x7C29A6, ] -WH_REGEX = r"discordapp\.com\/api\/webhooks\/(?P\d+)\/(?P.+)" +WH_REGEX = r"discord(app)?\.com\/api\/webhooks\/(?P\d+)\/(?P.+)" def worth_posting_location(location, coordinates): From 4fc250063fe29cbaa8bcc9912f59e3308f3a300c Mon Sep 17 00:00:00 2001 From: NNTin Date: Mon, 5 Oct 2020 11:54:21 +0200 Subject: [PATCH 7/9] Extend processing (#200) * include retweet logic applied to tracking hashtags and location boxes * fix stylecheck error --- bot/utils/processor.py | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/bot/utils/processor.py b/bot/utils/processor.py index f99ece64..c71580e2 100644 --- a/bot/utils/processor.py +++ b/bot/utils/processor.py @@ -54,22 +54,28 @@ WH_REGEX = r"discord(app)?\.com\/api\/webhooks\/(?P\d+)\/(?P.+)" -def worth_posting_location(location, coordinates): +def worth_posting_location(location, coordinates, retweeted, include_retweet): location = [location[i : i + 4] for i in range(0, len(location), 4)] for box in location: for coordinate in coordinates: if box[0] < coordinate[0] < box[2] and box[1] < coordinate[1] < box[3]: + if not include_retweet and retweeted: + return False return True return False -def worth_posting_track(track, hashtags, text): +def worth_posting_track(track, hashtags, text, retweeted, include_retweet): for t in track: if t.startswith("#"): if t[1:] in map(lambda x: x["text"], hashtags): + if not include_retweet and retweeted: + return False return True elif t in text: + if not include_retweet and retweeted: + return False return True return False @@ -151,7 +157,10 @@ def worth_posting_location(self): coordinates.append(c) return worth_posting_location( - location=self.discord_config.get("location", []), coordinates=coordinates + location=self.discord_config.get("location", []), + coordinates=coordinates, + retweeted=self.status_tweet["retweeted"] or "retweeted_status" in self.status_tweet, + include_retweet=self.discord_config.get("IncludeRetweet", True), ) def worth_posting_track(self): @@ -167,7 +176,11 @@ def worth_posting_track(self): ) return worth_posting_track( - track=self.discord_config.get("track", []), hashtags=hashtags, text=self.text + track=self.discord_config.get("track", []), + hashtags=hashtags, + text=self.text, + retweeted=self.status_tweet["retweeted"] or "retweeted_status" in self.status_tweet, + include_retweet=self.discord_config.get("IncludeRetweet", True), ) def worth_posting_follow(self): From b05d514af86533cabe0f8e285dd2503912317f43 Mon Sep 17 00:00:00 2001 From: NNTin Date: Tue, 6 Oct 2020 01:54:10 +0200 Subject: [PATCH 8/9] Several minor changes (#201) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update python-dotenv from 0.10.1 to 0.10.3 * Update discord.py from 1.1.1 to 1.2.3 * Update tweepy from 3.7.0 to 3.8.0 * Update websockets from 6.0 to 8.0.2 * Update jsonschema from 3.0.1 to 3.0.2 * Update oauthlib from 3.0.1 to 3.1.0 * Update tox from 3.9.0 to 3.14.0 * Update virtualenv from 16.4.3 to 16.7.5 * Update pluggy from 0.9.0 to 0.13.0 * Update certifi from 2019.3.9 to 2019.9.11 * Update aiohttp from 3.5.4 to 3.6.1 * Update pysocks from 1.6.8 to 1.7.1 * Update urllib3 from 1.24.1 to 1.25.6 * Update attrs from 19.1.0 to 19.2.0 * fix requirements.txt * reduce dependencies * updated remaining dependencies * Update development branch to master branch (#169) * Fix retweet handling Previously, retweets would be truncated by the bot once posted. This update adds lines to handle retweets specifically. * fix stylecheck error * RegEx fix: Discord domain changed Discord changed their domain from discordapp.com to discord.com. Adjusted code to accept both. Co-authored-by: EntropyInEffect <57723264+EntropyInEffect@users.noreply.github.com> * Update pyyaml from 3.13 to 5.3.1 (#153) Merging into dev branch. Did not test manually but since the CI checks passed which does cover the .yml file parsing it should probably work. Will revert commit if it broke. ¯\_(ツ)_/¯ * Update azure-pipelines.yml (#195) Using latest OS images * Update image in azure pipeline * Extend processing (#200) * include retweet logic applied to tracking hashtags and location boxes * fix stylecheck error Co-authored-by: pyup-bot Co-authored-by: EntropyInEffect <57723264+EntropyInEffect@users.noreply.github.com> --- azure-pipelines.yml | 16 ++++++++-------- bot/utils/processor.py | 21 +++++++++++++++++---- requirements.txt | 36 +++++++----------------------------- 3 files changed, 32 insertions(+), 41 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 1b8b168e..2e8873ea 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,19 +9,19 @@ jobs: matrix: Python36-Windows-Style: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'vs2017-win2016' + IMAGE_NAME: 'windows-latest' TOXENV: 'style' Python36-Windows: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'vs2017-win2016' + IMAGE_NAME: 'windows-latest' TOXENV: 'py36' Python36-Linux: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'ubuntu-16.04' + IMAGE_NAME: 'ubuntu-latest' TOXENV: 'py36' Python36-MacOSX: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'macos-10.13' + IMAGE_NAME: 'macOS-latest' TOXENV: 'py36' pool: @@ -60,15 +60,15 @@ jobs: matrix: Python36-Windows: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'vs2017-win2016' + IMAGE_NAME: 'windows-latest' TOXENV: 'extended' Python36-Linux: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'ubuntu-16.04' + IMAGE_NAME: 'ubuntu-latest' TOXENV: 'extended' Python36-MacOSX: PYTHON_VERSION: '3.6' - IMAGE_NAME: 'macos-10.13' + IMAGE_NAME: 'macOS-latest' TOXENV: 'extended' pool: vmImage: '$(IMAGE_NAME)' @@ -206,4 +206,4 @@ jobs: ) displayName: 'Creating release manifest and pushing' env: - DOCKER_CLI_EXPERIMENTAL: 'enabled' \ No newline at end of file + DOCKER_CLI_EXPERIMENTAL: 'enabled' diff --git a/bot/utils/processor.py b/bot/utils/processor.py index f99ece64..c71580e2 100644 --- a/bot/utils/processor.py +++ b/bot/utils/processor.py @@ -54,22 +54,28 @@ WH_REGEX = r"discord(app)?\.com\/api\/webhooks\/(?P\d+)\/(?P.+)" -def worth_posting_location(location, coordinates): +def worth_posting_location(location, coordinates, retweeted, include_retweet): location = [location[i : i + 4] for i in range(0, len(location), 4)] for box in location: for coordinate in coordinates: if box[0] < coordinate[0] < box[2] and box[1] < coordinate[1] < box[3]: + if not include_retweet and retweeted: + return False return True return False -def worth_posting_track(track, hashtags, text): +def worth_posting_track(track, hashtags, text, retweeted, include_retweet): for t in track: if t.startswith("#"): if t[1:] in map(lambda x: x["text"], hashtags): + if not include_retweet and retweeted: + return False return True elif t in text: + if not include_retweet and retweeted: + return False return True return False @@ -151,7 +157,10 @@ def worth_posting_location(self): coordinates.append(c) return worth_posting_location( - location=self.discord_config.get("location", []), coordinates=coordinates + location=self.discord_config.get("location", []), + coordinates=coordinates, + retweeted=self.status_tweet["retweeted"] or "retweeted_status" in self.status_tweet, + include_retweet=self.discord_config.get("IncludeRetweet", True), ) def worth_posting_track(self): @@ -167,7 +176,11 @@ def worth_posting_track(self): ) return worth_posting_track( - track=self.discord_config.get("track", []), hashtags=hashtags, text=self.text + track=self.discord_config.get("track", []), + hashtags=hashtags, + text=self.text, + retweeted=self.status_tweet["retweeted"] or "retweeted_status" in self.status_tweet, + include_retweet=self.discord_config.get("IncludeRetweet", True), ) def worth_posting_follow(self): diff --git a/requirements.txt b/requirements.txt index b6939b42..e519ea4c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,30 +1,8 @@ -aiohttp==3.5.4 -async-timeout==3.0.1 -attrs==19.1.0 -certifi==2019.3.9 -chardet==3.0.4 -discord.py==1.1.1 -idna==2.8 -idna-ssl==1.1.0 -multidict==4.5.2 -oauthlib==3.0.1 -PySocks==1.6.8 -python-dotenv==0.10.1 -PyYAML==3.13 -requests==2.21.0 -requests-oauthlib==1.2.0 -six==1.12.0 -tweepy==3.7.0 -urllib3==1.24.1 -websockets==6.0 -yarl==1.3.0 -toml==0.10.0 -appdirs==1.4.3 black==18.9b0 -Click==7.0 -filelock==3.0.10 -jsonschema==3.0.1 -pluggy==0.9.0 -py==1.8.0 -tox==3.9.0 -virtualenv==16.4.3 \ No newline at end of file +discord.py==1.2.3 +jsonschema==3.0.2 +python-dotenv==0.10.3 +PyYAML==5.3.1 +tox==3.14.0 +tweepy==3.8.0 +virtualenv==16.7.5 From 4fa3d41d5970d310cadd057bf26248ea3e7c3776 Mon Sep 17 00:00:00 2001 From: pyup-bot Date: Tue, 19 Nov 2019 13:50:21 +0100 Subject: [PATCH 9/9] Update jsonschema from 3.0.2 to 3.2.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index e519ea4c..b31836a4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ black==18.9b0 discord.py==1.2.3 -jsonschema==3.0.2 +jsonschema==3.2.0 python-dotenv==0.10.3 PyYAML==5.3.1 tox==3.14.0