2
2
3
3
import collections
4
4
import datetime
5
+ import logging
5
6
import re
6
7
import sys
7
8
from typing import TYPE_CHECKING , Any , List , Tuple , Union
8
9
9
- import toolz
10
10
from bs4 import BeautifulSoup
11
11
12
12
from .enums import (
40
40
join ,
41
41
normalize ,
42
42
time_mapping ,
43
+ unroll_number ,
43
44
)
44
45
45
46
if TYPE_CHECKING :
@@ -177,6 +178,9 @@ class Profile:
177
178
the plaftorms the software was built for.
178
179
status : Status
179
180
Exclusive to Games, Mods, Addons, Engines, Hardware .Whether the thing is released, unreleased, ect...
181
+ download_count: int
182
+ Total count of all downloads on the page, this adds up downloads of all files and addons. Exclusive
183
+ to mods and games.
180
184
181
185
"""
182
186
@@ -226,7 +230,12 @@ def __init__(self, html: BeautifulSoup):
226
230
"facebook" : share [3 ]["href" ],
227
231
}
228
232
except (AttributeError , IndexError ):
229
- LOGGER .info ("Something funky about share box of %s %s" , page_type .name , _name )
233
+ LOGGER .info (
234
+ "Something funky about share box of %s %s" ,
235
+ page_type .name ,
236
+ _name ,
237
+ exc_info = LOGGER .level >= logging .DEBUG ,
238
+ )
230
239
self .share = None
231
240
232
241
if page_type in [SearchCategory .developers , SearchCategory .groups ]:
@@ -251,7 +260,12 @@ def __init__(self, html: BeautifulSoup):
251
260
self .icon = profile_raw .find ("h5" , string = "Icon" ).parent .span .img ["src" ]
252
261
except AttributeError :
253
262
self .icon = None
254
- LOGGER .info ("%s '%s' does not have an icon" , page_type , _name )
263
+ LOGGER .info (
264
+ "%s '%s' does not have an icon" ,
265
+ page_type ,
266
+ _name ,
267
+ exc_info = LOGGER .level >= logging .DEBUG ,
268
+ )
255
269
256
270
if page_type in [
257
271
SearchCategory .games ,
@@ -284,7 +298,12 @@ def __init__(self, html: BeautifulSoup):
284
298
d = profile_raw .find ("h5" , string = "Release date" ).parent .span .time
285
299
self .release = get_date (d ["datetime" ])
286
300
except KeyError :
287
- LOGGER .info ("%s %s has not been released" , page_type .name , _name )
301
+ LOGGER .info (
302
+ "%s %s has not been released" ,
303
+ page_type .name ,
304
+ _name ,
305
+ exc_info = LOGGER .level >= logging .DEBUG ,
306
+ )
288
307
self .release = None
289
308
290
309
if "Coming" in d .string :
@@ -308,7 +327,12 @@ def __init__(self, html: BeautifulSoup):
308
327
self .homepage = html .find ("h5" , string = "Homepage" ).parent .span .a ["href" ]
309
328
except AttributeError :
310
329
self .homepage = None
311
- LOGGER .info ("%s %s has no homepage" , page_type .name , _name )
330
+ LOGGER .info (
331
+ "%s %s has no homepage" ,
332
+ page_type .name ,
333
+ _name ,
334
+ exc_info = LOGGER .level >= logging .DEBUG ,
335
+ )
312
336
313
337
if page_type in [SearchCategory .games , SearchCategory .addons ]:
314
338
engine = profile_raw .find ("h5" , string = "Engine" )
@@ -353,6 +377,20 @@ def __init__(self, html: BeautifulSoup):
353
377
category = html .find ("h3" ).string .strip ().lower ().replace (" & " , "_" )
354
378
self .category = GroupCategory [category ]
355
379
380
+ if page_type in [SearchCategory .games , SearchCategory .mods ]:
381
+ try :
382
+ self .download_count = unroll_number (
383
+ html .find ("a" , class_ = "downloadautotoggle" ).span .string
384
+ )
385
+ except AttributeError :
386
+ self .download_count = 0
387
+ LOGGER .info (
388
+ "%s %s has no download count" ,
389
+ page_type .name ,
390
+ _name ,
391
+ exc_info = LOGGER .level >= logging .DEBUG ,
392
+ )
393
+
356
394
def __repr__ (self ):
357
395
return f"<Profile category={ self .category .name } >"
358
396
@@ -392,12 +430,12 @@ def __init__(self, html: BeautifulSoup):
392
430
try :
393
431
self .scope = Scope (int (html .find ("h5" , string = "Project" ).parent .a ["href" ][- 1 ]))
394
432
except AttributeError :
395
- LOGGER .info ("Has no scope" )
433
+ LOGGER .info ("Has no scope" , exc_info = LOGGER . level >= logging . DEBUG )
396
434
397
435
try :
398
436
self .boxart = html .find ("h5" , string = "Boxart" ).parent .span .a .img ["src" ]
399
437
except AttributeError :
400
- LOGGER .info ("Has no boxart" )
438
+ LOGGER .info ("Has no boxart" , exc_info = LOGGER . level >= logging . DEBUG )
401
439
402
440
def __repr__ (self ):
403
441
return (
@@ -427,12 +465,12 @@ class Thumbnail:
427
465
"""
428
466
429
467
def __init__ (self , ** attrs ):
430
- self .url = join (attrs .get ("url" ))
431
- self .name = attrs .get ("name" , None )
432
- self .image = attrs .get ("image" , None )
433
- self .summary = attrs .get ("summary" , None )
434
- self .date = attrs .get ("date" , None )
435
- self .type = attrs .get ("type" )
468
+ self .url : str = join (attrs .get ("url" ))
469
+ self .name : str | None = attrs .get ("name" , None )
470
+ self .image : str | None = attrs .get ("image" , None )
471
+ self .summary : str | None = attrs .get ("summary" , None )
472
+ self .date : datetime . datetime | None = attrs .get ("date" , None )
473
+ self .type : ThumbnailType = attrs .get ("type" )
436
474
437
475
def __repr__ (self ):
438
476
return f"<Thumbnail name={ self .name } type={ self .type .name } >"
@@ -478,7 +516,7 @@ def _parse_results(html):
478
516
)
479
517
except (TypeError , KeyError ):
480
518
# parse as a title-content pair of articles
481
- LOGGER .info ("Parsing articles as key-pair list" )
519
+ LOGGER .info ("Parsing articles as key-value pair list" , exc_info = LOGGER . level >= logging . DEBUG )
482
520
for title , content in zip (search_raws [::2 ], search_raws [1 ::2 ]):
483
521
date = title .find ("time" )
484
522
url = title .find ("h4" ).a
@@ -532,14 +570,40 @@ def _parse_comments(html):
532
570
try :
533
571
comments [- 1 ].children [- 1 ].children .append (comment )
534
572
except IndexError :
535
- comments [- 1 ].children .append (MissingComment (1 ))
536
- comments [- 1 ].children [- 1 ].children .append (comment )
573
+ try :
574
+ comments [- 1 ].children .append (MissingComment (1 ))
575
+ comments [- 1 ].children [- 1 ].children .append (comment )
576
+ except IndexError :
577
+ comments .append (MissingComment (0 ))
578
+ comments [- 1 ].children .append (MissingComment (1 ))
579
+ comments [- 1 ].children [- 1 ].children .append (comment )
537
580
else :
538
581
comments .append (comment )
539
582
540
583
return comments , current_page , total_page , total_results
541
584
542
585
586
+ class CommentAuthor (Thumbnail ):
587
+ """Represents the thumbnail of a user having left a comment on a page. Functions the same as a
588
+ thumbnail but with an extra attribute.
589
+
590
+ Attributes
591
+ -----------
592
+ comment_count : int
593
+ Number of comments the user has posted
594
+ """
595
+
596
+ def __init__ (self , ** attrs ):
597
+ super ().__init__ (** attrs )
598
+
599
+ self .comment_count : int = attrs .get ("comment_count" , 0 )
600
+
601
+ def __repr__ (self ):
602
+ return (
603
+ f"<Thumbnail name={ self .name } type={ self .type .name } comment_count={ self .comment_count } >"
604
+ )
605
+
606
+
543
607
class Comment :
544
608
"""A moddb comment object.
545
609
@@ -591,11 +655,19 @@ class Comment:
591
655
def __init__ (self , html : BeautifulSoup ):
592
656
author = html .find ("a" , class_ = "avatar" )
593
657
self .id = int (html ["id" ])
594
- self .author = Thumbnail (
658
+ comment_count = int (
659
+ html .find ("span" , class_ = "heading" )
660
+ .text .strip ()
661
+ .split ("-" )[- 1 ]
662
+ .replace ("comments" , "" )
663
+ .replace ("," , "" )
664
+ )
665
+ self .author = CommentAuthor (
595
666
name = author ["title" ],
596
667
url = author ["href" ],
597
668
image = author .img ["src" ],
598
669
type = ThumbnailType .member ,
670
+ comment_count = comment_count ,
599
671
)
600
672
self .date = get_date (html .find ("time" )["datetime" ])
601
673
actions = html .find ("span" , class_ = "actions" )
@@ -620,6 +692,7 @@ def __init__(self, html: BeautifulSoup):
620
692
"Comment %s by %s has no content, likely embed" ,
621
693
self .id ,
622
694
self .author .name ,
695
+ exc_info = LOGGER .level >= logging .DEBUG ,
623
696
)
624
697
self .content = None
625
698
@@ -772,21 +845,37 @@ def __init__(self, html: BeautifulSoup):
772
845
try :
773
846
self .gender = profile_raw .find ("h5" , string = "Gender" ).parent .span .string .strip ()
774
847
except AttributeError :
775
- LOGGER .info ("Member %s has not publicized their gender" , self .name )
848
+ LOGGER .info (
849
+ "Member %s has not publicized their gender" ,
850
+ self .name ,
851
+ exc_info = LOGGER .level >= logging .DEBUG ,
852
+ )
776
853
self .gender = None
777
854
778
855
try :
779
856
self .homepage = html .find ("h5" , string = "Homepage" ).parent .span .a ["href" ]
780
857
except AttributeError :
781
858
self .homepage = None
782
- LOGGER .info ("Member %s has no homepage" , self .name )
859
+ LOGGER .info (
860
+ "Member %s has no homepage" , self .name , exc_info = LOGGER .level >= logging .DEBUG
861
+ )
783
862
784
- self .country = profile_raw .find ("h5" , string = "Country" ).parent .span .string .strip ()
863
+ try :
864
+ self .country = profile_raw .find ("h5" , string = "Country" ).parent .span .string .strip ()
865
+ except AttributeError :
866
+ self .country = None
867
+ LOGGER .info (
868
+ "Member %s country is not visible (happens when not logged in)" ,
869
+ self .name ,
870
+ exc_info = LOGGER .level >= logging .DEBUG ,
871
+ )
785
872
786
873
try :
787
874
self .follow = join (profile_raw .find ("h5" , string = "Member watch" ).parent .span .a ["href" ])
788
875
except AttributeError :
789
- LOGGER .info ("Can't watch yourself, narcissist..." )
876
+ LOGGER .info (
877
+ "Can't watch yourself, narcissist..." , exc_info = LOGGER .level >= logging .DEBUG
878
+ )
790
879
self .follow = None
791
880
792
881
def __repr__ (self ):
@@ -855,7 +944,7 @@ def get(parent):
855
944
except AttributeError :
856
945
self .rank = 0
857
946
self .total = 0
858
- LOGGER .info ("Member %s has no rank" , name )
947
+ LOGGER .info ("Member %s has no rank" , name , exc_info = LOGGER . level >= logging . DEBUG )
859
948
860
949
def __repr__ (self ):
861
950
return f"<MemberStatistics rank={ self .rank } /{ self .total } >"
@@ -1106,13 +1195,13 @@ def get_all_results(self):
1106
1195
results .extend (search )
1107
1196
LOGGER .info ("Parsed page %s/%s" , search .current_page , search .total_pages )
1108
1197
1109
- def key_check (element ):
1198
+ def key (element ):
1110
1199
if isinstance (element , Comment ):
1111
1200
return element .id
1112
1201
else :
1113
1202
return element .name
1114
1203
1115
- search ._results = list (toolz . unique ( results , key = key_check ))
1204
+ search ._results = list ({ key ( e ): e for e in results }. values ( ))
1116
1205
return search
1117
1206
1118
1207
def __repr__ (self ):
0 commit comments