Skip to content

Commit d616d5a

Browse files
authored
Merge pull request #45 from ClementJ18/develop
v0.11.0 release
2 parents 536b423 + 2c77147 commit d616d5a

File tree

278 files changed

+265062
-397113
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

278 files changed

+265062
-397113
lines changed

.gitattributes

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
* text eol=lf

.github/workflows/python-package.yml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,4 +56,11 @@ jobs:
5656
run: |
5757
bash run_tests.sh full --test-group-count 3 --test-group=3 --reruns 3 --reruns-delay 15
5858
if: github.event.pull_request.base.ref == 'master'
59+
- name: Upload test results
60+
uses: actions/upload-artifact@v4
61+
with:
62+
name: Test result
63+
path: report.html
64+
65+
5966

.readthedocs.yml

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,13 @@
1+
version: 2
2+
13
build:
2-
image: latest
4+
os: ubuntu-22.04
5+
tools:
6+
python: "3.9"
7+
8+
sphinx:
9+
configuration: docs/source/conf.py
310

411
python:
5-
version: 3.8
6-
setup_py_install: true
12+
install:
13+
- requirements: requirements-docs.txt

docs/source/changelog.rst

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,26 @@ The page attempt to keep a clear list of breaking/non-breaking changes and new f
88
:local:
99
:backlinks: none
1010

11+
12+
v0.11.0
13+
-------
14+
Bug Fixes
15+
##########
16+
* Fixed an edge case in comment parsing
17+
18+
New Features
19+
#############
20+
* Added `FrontPage.get_poll` to get the poll on the front page
21+
* Ratelimiting login request to to 1/5s
22+
* Ratelimited requests that are asked to raise will now raise `moddb.errors.Ratelimited`
23+
* Profiles now display the aggregated download count of mods and games (if they have one) under `download_count`
24+
25+
26+
Removed Features
27+
#################
28+
* Removed `FrontPage.poll` to reduce the number of requests used when calling `front_page()`
29+
30+
1131
v0.10.0
1232
-------
1333
Bug Fixes

docs/source/conf.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@
2727
# The short X.Y version
2828
version = ""
2929
# The full version, including alpha/beta/rc tags
30-
release = "0.10.0"
30+
release = "0.11.0"
3131

3232

3333
# -- General configuration ---------------------------------------------------

moddb/__init__.py

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,23 @@
88

99
SESSION = requests.Session()
1010

11-
__version__ = "0.10.0"
11+
__version__ = "0.11.0"
12+
13+
__all__ = [
14+
"front_page",
15+
"login",
16+
"logout",
17+
"parse_page",
18+
"parse_results",
19+
"rss",
20+
"search",
21+
"search_tags",
22+
"Client",
23+
"Thread",
24+
"BASE_URL",
25+
"LOGGER",
26+
"Object",
27+
"get_page",
28+
"request",
29+
"soup",
30+
]

moddb/boxes.py

Lines changed: 113 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,11 @@
22

33
import collections
44
import datetime
5+
import logging
56
import re
67
import sys
78
from typing import TYPE_CHECKING, Any, List, Tuple, Union
89

9-
import toolz
1010
from bs4 import BeautifulSoup
1111

1212
from .enums import (
@@ -40,6 +40,7 @@
4040
join,
4141
normalize,
4242
time_mapping,
43+
unroll_number,
4344
)
4445

4546
if TYPE_CHECKING:
@@ -177,6 +178,9 @@ class Profile:
177178
the plaftorms the software was built for.
178179
status : Status
179180
Exclusive to Games, Mods, Addons, Engines, Hardware .Whether the thing is released, unreleased, ect...
181+
download_count: int
182+
Total count of all downloads on the page, this adds up downloads of all files and addons. Exclusive
183+
to mods and games.
180184
181185
"""
182186

@@ -226,7 +230,12 @@ def __init__(self, html: BeautifulSoup):
226230
"facebook": share[3]["href"],
227231
}
228232
except (AttributeError, IndexError):
229-
LOGGER.info("Something funky about share box of %s %s", page_type.name, _name)
233+
LOGGER.info(
234+
"Something funky about share box of %s %s",
235+
page_type.name,
236+
_name,
237+
exc_info=LOGGER.level >= logging.DEBUG,
238+
)
230239
self.share = None
231240

232241
if page_type in [SearchCategory.developers, SearchCategory.groups]:
@@ -251,7 +260,12 @@ def __init__(self, html: BeautifulSoup):
251260
self.icon = profile_raw.find("h5", string="Icon").parent.span.img["src"]
252261
except AttributeError:
253262
self.icon = None
254-
LOGGER.info("%s '%s' does not have an icon", page_type, _name)
263+
LOGGER.info(
264+
"%s '%s' does not have an icon",
265+
page_type,
266+
_name,
267+
exc_info=LOGGER.level >= logging.DEBUG,
268+
)
255269

256270
if page_type in [
257271
SearchCategory.games,
@@ -284,7 +298,12 @@ def __init__(self, html: BeautifulSoup):
284298
d = profile_raw.find("h5", string="Release date").parent.span.time
285299
self.release = get_date(d["datetime"])
286300
except KeyError:
287-
LOGGER.info("%s %s has not been released", page_type.name, _name)
301+
LOGGER.info(
302+
"%s %s has not been released",
303+
page_type.name,
304+
_name,
305+
exc_info=LOGGER.level >= logging.DEBUG,
306+
)
288307
self.release = None
289308

290309
if "Coming" in d.string:
@@ -308,7 +327,12 @@ def __init__(self, html: BeautifulSoup):
308327
self.homepage = html.find("h5", string="Homepage").parent.span.a["href"]
309328
except AttributeError:
310329
self.homepage = None
311-
LOGGER.info("%s %s has no homepage", page_type.name, _name)
330+
LOGGER.info(
331+
"%s %s has no homepage",
332+
page_type.name,
333+
_name,
334+
exc_info=LOGGER.level >= logging.DEBUG,
335+
)
312336

313337
if page_type in [SearchCategory.games, SearchCategory.addons]:
314338
engine = profile_raw.find("h5", string="Engine")
@@ -353,6 +377,20 @@ def __init__(self, html: BeautifulSoup):
353377
category = html.find("h3").string.strip().lower().replace(" & ", "_")
354378
self.category = GroupCategory[category]
355379

380+
if page_type in [SearchCategory.games, SearchCategory.mods]:
381+
try:
382+
self.download_count = unroll_number(
383+
html.find("a", class_="downloadautotoggle").span.string
384+
)
385+
except AttributeError:
386+
self.download_count = 0
387+
LOGGER.info(
388+
"%s %s has no download count",
389+
page_type.name,
390+
_name,
391+
exc_info=LOGGER.level >= logging.DEBUG,
392+
)
393+
356394
def __repr__(self):
357395
return f"<Profile category={self.category.name}>"
358396

@@ -392,12 +430,12 @@ def __init__(self, html: BeautifulSoup):
392430
try:
393431
self.scope = Scope(int(html.find("h5", string="Project").parent.a["href"][-1]))
394432
except AttributeError:
395-
LOGGER.info("Has no scope")
433+
LOGGER.info("Has no scope", exc_info=LOGGER.level >= logging.DEBUG)
396434

397435
try:
398436
self.boxart = html.find("h5", string="Boxart").parent.span.a.img["src"]
399437
except AttributeError:
400-
LOGGER.info("Has no boxart")
438+
LOGGER.info("Has no boxart", exc_info=LOGGER.level >= logging.DEBUG)
401439

402440
def __repr__(self):
403441
return (
@@ -427,12 +465,12 @@ class Thumbnail:
427465
"""
428466

429467
def __init__(self, **attrs):
430-
self.url = join(attrs.get("url"))
431-
self.name = attrs.get("name", None)
432-
self.image = attrs.get("image", None)
433-
self.summary = attrs.get("summary", None)
434-
self.date = attrs.get("date", None)
435-
self.type = attrs.get("type")
468+
self.url: str = join(attrs.get("url"))
469+
self.name: str | None = attrs.get("name", None)
470+
self.image: str | None = attrs.get("image", None)
471+
self.summary: str | None = attrs.get("summary", None)
472+
self.date: datetime.datetime | None = attrs.get("date", None)
473+
self.type: ThumbnailType = attrs.get("type")
436474

437475
def __repr__(self):
438476
return f"<Thumbnail name={self.name} type={self.type.name}>"
@@ -478,7 +516,7 @@ def _parse_results(html):
478516
)
479517
except (TypeError, KeyError):
480518
# parse as a title-content pair of articles
481-
LOGGER.info("Parsing articles as key-pair list")
519+
LOGGER.info("Parsing articles as key-value pair list", exc_info=LOGGER.level >= logging.DEBUG)
482520
for title, content in zip(search_raws[::2], search_raws[1::2]):
483521
date = title.find("time")
484522
url = title.find("h4").a
@@ -532,14 +570,40 @@ def _parse_comments(html):
532570
try:
533571
comments[-1].children[-1].children.append(comment)
534572
except IndexError:
535-
comments[-1].children.append(MissingComment(1))
536-
comments[-1].children[-1].children.append(comment)
573+
try:
574+
comments[-1].children.append(MissingComment(1))
575+
comments[-1].children[-1].children.append(comment)
576+
except IndexError:
577+
comments.append(MissingComment(0))
578+
comments[-1].children.append(MissingComment(1))
579+
comments[-1].children[-1].children.append(comment)
537580
else:
538581
comments.append(comment)
539582

540583
return comments, current_page, total_page, total_results
541584

542585

586+
class CommentAuthor(Thumbnail):
587+
"""Represents the thumbnail of a user having left a comment on a page. Functions the same as a
588+
thumbnail but with an extra attribute.
589+
590+
Attributes
591+
-----------
592+
comment_count : int
593+
Number of comments the user has posted
594+
"""
595+
596+
def __init__(self, **attrs):
597+
super().__init__(**attrs)
598+
599+
self.comment_count: int = attrs.get("comment_count", 0)
600+
601+
def __repr__(self):
602+
return (
603+
f"<Thumbnail name={self.name} type={self.type.name} comment_count={self.comment_count}>"
604+
)
605+
606+
543607
class Comment:
544608
"""A moddb comment object.
545609
@@ -591,11 +655,19 @@ class Comment:
591655
def __init__(self, html: BeautifulSoup):
592656
author = html.find("a", class_="avatar")
593657
self.id = int(html["id"])
594-
self.author = Thumbnail(
658+
comment_count = int(
659+
html.find("span", class_="heading")
660+
.text.strip()
661+
.split("-")[-1]
662+
.replace("comments", "")
663+
.replace(",", "")
664+
)
665+
self.author = CommentAuthor(
595666
name=author["title"],
596667
url=author["href"],
597668
image=author.img["src"],
598669
type=ThumbnailType.member,
670+
comment_count=comment_count,
599671
)
600672
self.date = get_date(html.find("time")["datetime"])
601673
actions = html.find("span", class_="actions")
@@ -620,6 +692,7 @@ def __init__(self, html: BeautifulSoup):
620692
"Comment %s by %s has no content, likely embed",
621693
self.id,
622694
self.author.name,
695+
exc_info=LOGGER.level >= logging.DEBUG,
623696
)
624697
self.content = None
625698

@@ -772,21 +845,37 @@ def __init__(self, html: BeautifulSoup):
772845
try:
773846
self.gender = profile_raw.find("h5", string="Gender").parent.span.string.strip()
774847
except AttributeError:
775-
LOGGER.info("Member %s has not publicized their gender", self.name)
848+
LOGGER.info(
849+
"Member %s has not publicized their gender",
850+
self.name,
851+
exc_info=LOGGER.level >= logging.DEBUG,
852+
)
776853
self.gender = None
777854

778855
try:
779856
self.homepage = html.find("h5", string="Homepage").parent.span.a["href"]
780857
except AttributeError:
781858
self.homepage = None
782-
LOGGER.info("Member %s has no homepage", self.name)
859+
LOGGER.info(
860+
"Member %s has no homepage", self.name, exc_info=LOGGER.level >= logging.DEBUG
861+
)
783862

784-
self.country = profile_raw.find("h5", string="Country").parent.span.string.strip()
863+
try:
864+
self.country = profile_raw.find("h5", string="Country").parent.span.string.strip()
865+
except AttributeError:
866+
self.country = None
867+
LOGGER.info(
868+
"Member %s country is not visible (happens when not logged in)",
869+
self.name,
870+
exc_info=LOGGER.level >= logging.DEBUG,
871+
)
785872

786873
try:
787874
self.follow = join(profile_raw.find("h5", string="Member watch").parent.span.a["href"])
788875
except AttributeError:
789-
LOGGER.info("Can't watch yourself, narcissist...")
876+
LOGGER.info(
877+
"Can't watch yourself, narcissist...", exc_info=LOGGER.level >= logging.DEBUG
878+
)
790879
self.follow = None
791880

792881
def __repr__(self):
@@ -855,7 +944,7 @@ def get(parent):
855944
except AttributeError:
856945
self.rank = 0
857946
self.total = 0
858-
LOGGER.info("Member %s has no rank", name)
947+
LOGGER.info("Member %s has no rank", name, exc_info=LOGGER.level >= logging.DEBUG)
859948

860949
def __repr__(self):
861950
return f"<MemberStatistics rank={self.rank}/{self.total}>"
@@ -1106,13 +1195,13 @@ def get_all_results(self):
11061195
results.extend(search)
11071196
LOGGER.info("Parsed page %s/%s", search.current_page, search.total_pages)
11081197

1109-
def key_check(element):
1198+
def key(element):
11101199
if isinstance(element, Comment):
11111200
return element.id
11121201
else:
11131202
return element.name
11141203

1115-
search._results = list(toolz.unique(results, key=key_check))
1204+
search._results = list({key(e): e for e in results}.values())
11161205
return search
11171206

11181207
def __repr__(self):

0 commit comments

Comments
 (0)