Skip to content

Commit

Permalink
Add custom headers in Image Download func
Browse files Browse the repository at this point in the history
  • Loading branch information
@mvdctop
mvdctop committed May 21, 2022
1 parent 47110c5 commit daf431b
Show file tree
Hide file tree
Showing 2 changed files with 42 additions and 30 deletions.
12 changes: 7 additions & 5 deletions ADC_function.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def getXpathSingle(htmlcode, xpath):

G_USER_AGENT = r'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.133 Safari/537.36'

def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None, encoding: str = None):
def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None, encoding: str = None, json_headers = None):
"""
网页请求核心函数
"""
Expand All @@ -38,6 +38,8 @@ def get_html(url, cookies: dict = None, ua: str = None, return_type: str = None,
errors = ""

headers = {"User-Agent": ua or G_USER_AGENT} # noqa
if json_headers != None:
headers.update(json_headers)

for i in range(configProxy.retry):
try:
Expand Down Expand Up @@ -518,15 +520,15 @@ def download_one_file(args) -> str:
wrapped for map function
"""

(url, save_path) = args
filebytes = get_html(url, return_type='content')
(url, save_path, json_data) = args
filebytes = get_html(url, return_type='content', json_headers=json_data['headers'])
if isinstance(filebytes, bytes) and len(filebytes):
with save_path.open('wb') as fpbyte:
if len(filebytes) == fpbyte.write(filebytes):
return str(save_path)


def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel: int = 0):
def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel: int = 0, json_data=None):
"""
download files in parallel 多线程下载文件
Expand All @@ -545,7 +547,7 @@ def parallel_download_files(dn_list: typing.Iterable[typing.Sequence], parallel:
and fullpath and isinstance(fullpath, (str, Path)) and len(str(fullpath)):
fullpath = Path(fullpath)
fullpath.parent.mkdir(parents=True, exist_ok=True)
mp_args.append((url, fullpath))
mp_args.append((url, fullpath, json_data))
if not len(mp_args):
return []
if not isinstance(parallel, int) or parallel not in range(1, 200):
Expand Down
60 changes: 35 additions & 25 deletions core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,11 +71,11 @@ def get_info(json_data): # 返回json里的数据
return title, studio, year, outline, runtime, director, actor_photo, release, number, cover, trailer, website, series, label


def small_cover_check(path, filename, cover_small, movie_path):
def small_cover_check(path, filename, cover_small, movie_path, json_data=None):
full_filepath = Path(path) / filename
if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(str(full_filepath)):
return
download_file_with_filename(cover_small, filename, path, movie_path)
download_file_with_filename(cover_small, filename, path, movie_path, json_data)
print('[+]Image Downloaded! ' + full_filepath.name)


Expand Down Expand Up @@ -113,7 +113,7 @@ def create_folder(json_data): # 创建文件夹
# =====================资源下载部分===========================

# path = examle:photo , video.in the Project Folder!
def download_file_with_filename(url, filename, path, filepath):
def download_file_with_filename(url, filename, path, filepath, json_headers=None):
conf = config.getInstance()
configProxy = conf.proxy()

Expand All @@ -127,8 +127,9 @@ def download_file_with_filename(url, filename, path, filepath):
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
proxies = configProxy.proxies()
headers = {
'User-Agent': G_USER_AGENT}
headers = {'User-Agent': G_USER_AGENT}
if json_headers != None:
headers.update(json_headers)
r = requests.get(url, headers=headers, timeout=configProxy.timeout, proxies=proxies)
if r == '':
print('[-]Movie Download Data not found!')
Expand All @@ -143,8 +144,9 @@ def download_file_with_filename(url, filename, path, filepath):
except:
print(f"[-]Fatal error! Can not make folder '{path}'")
os._exit(0)
headers = {
'User-Agent': G_USER_AGENT}
headers = {'User-Agent': G_USER_AGENT}
if json_headers != None:
headers.update(json_headers)
r = requests.get(url, timeout=configProxy.timeout, headers=headers)
if r == '':
print('[-]Movie Download Data not found!')
Expand Down Expand Up @@ -226,13 +228,13 @@ def actor_photo_download(actors, save_dir, number):


# 剧照下载成功，否则移动到failed
def extrafanart_download(data, path, number, filepath):
def extrafanart_download(data, path, number, filepath, json_data=None):
if config.getInstance().extrafanart_thread_pool_download():
return extrafanart_download_threadpool(data, path, number)
extrafanart_download_one_by_one(data, path, filepath)
return extrafanart_download_threadpool(data, path, number, json_data)
extrafanart_download_one_by_one(data, path, filepath, json_data)


def extrafanart_download_one_by_one(data, path, filepath):
def extrafanart_download_one_by_one(data, path, filepath, json_data=None):
tm_start = time.perf_counter()
j = 1
conf = config.getInstance()
Expand All @@ -244,13 +246,13 @@ def extrafanart_download_one_by_one(data, path, filepath):
jpg_fullpath = os.path.join(path, jpg_filename)
if download_only_missing_images and not file_not_exist_or_empty(jpg_fullpath):
continue
if download_file_with_filename(url, jpg_filename, path, filepath) == 'failed':
if download_file_with_filename(url, jpg_filename, path, filepath, json_data) == 'failed':
moveFailedFolder(filepath)
return
for i in range(configProxy.retry):
if file_not_exist_or_empty(jpg_fullpath):
print('[!]Image Download Failed! Trying again. [{}/3]', i + 1)
download_file_with_filename(url, jpg_filename, path, filepath)
download_file_with_filename(url, jpg_filename, path, filepath, json_data)
continue
else:
break
Expand All @@ -262,7 +264,7 @@ def extrafanart_download_one_by_one(data, path, filepath):
print(f'[!]Extrafanart download one by one mode runtime {time.perf_counter() - tm_start:.3f}s')


def extrafanart_download_threadpool(url_list, save_dir, number):
def extrafanart_download_threadpool(url_list, save_dir, number,json_data=None):
tm_start = time.perf_counter()
conf = config.getInstance()
extrafanart_dir = Path(save_dir) / conf.get_extrafanart()
Expand All @@ -278,7 +280,7 @@ def extrafanart_download_threadpool(url_list, save_dir, number):
parallel = min(len(dn_list), conf.extrafanart_thread_pool_download())
if parallel > 100:
print('[!]Warrning: Parallel download thread too large may cause website ban IP!')
result = parallel_download_files(dn_list, parallel)
result = parallel_download_files(dn_list, parallel, json_data)
failed = 0
for i, r in enumerate(result, start=1):
if not r:
Expand All @@ -300,19 +302,27 @@ def image_ext(url):


# 封面是否下载成功，否则移动到failed
def image_download(cover, fanart_path, thumb_path, path, filepath):
def image_download(cover, fanart_path, thumb_path, path, filepath, json_data):
full_filepath = os.path.join(path, fanart_path)
if config.getInstance().download_only_missing_images() and not file_not_exist_or_empty(full_filepath):
return
if download_file_with_filename(cover, fanart_path, path, filepath) == 'failed':
moveFailedFolder(filepath)
return
if "headers" in json_data:
if download_file_with_filename(cover, fanart_path, path, filepath, json_data['headers']) == 'failed':
moveFailedFolder(filepath)
return
else:
if download_file_with_filename(cover, fanart_path, path, filepath) == 'failed':
moveFailedFolder(filepath)
return

configProxy = config.getInstance().proxy()
for i in range(configProxy.retry):
if file_not_exist_or_empty(full_filepath):
print('[!]Image Download Failed! Trying again. [{}/3]', i + 1)
download_file_with_filename(cover, fanart_path, path, filepath)
if "headers" in json_data:
download_file_with_filename(cover, fanart_path, path, filepath, json_data['headers'])
else:
download_file_with_filename(cover, fanart_path, path, filepath)
continue
else:
break
Expand Down Expand Up @@ -853,10 +863,10 @@ def core_main(movie_path, number_th, oCC):

# 检查小封面, 如果image cut为3，则下载小封面
if imagecut == 3:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path)
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data)

# creatFolder会返回番号路径
image_download( cover, fanart_path,thumb_path, path, movie_path)
image_download( cover, fanart_path,thumb_path, path, movie_path, json_data)

if not multi_part or part.lower() == '-cd1':
try:
Expand All @@ -866,7 +876,7 @@ def core_main(movie_path, number_th, oCC):

# 下载剧照 data, path, filepath
if conf.is_extrafanart() and json_data.get('extrafanart'):
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path)
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path, json_data)

# 下载演员头像 KODI .actors 目录位置
if conf.download_actor_photo_for_kodi():
Expand Down Expand Up @@ -907,7 +917,7 @@ def core_main(movie_path, number_th, oCC):

# 检查小封面, 如果image cut为3，则下载小封面
if imagecut == 3:
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path)
small_cover_check(path, poster_path, json_data.get('cover_small'), movie_path, json_data)

# creatFolder会返回番号路径
image_download( cover, fanart_path, thumb_path, path, movie_path)
Expand All @@ -920,7 +930,7 @@ def core_main(movie_path, number_th, oCC):

# 下载剧照 data, path, filepath
if conf.is_extrafanart() and json_data.get('extrafanart'):
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path)
extrafanart_download(json_data.get('extrafanart'), path, number, movie_path, json_data)

# 下载演员头像 KODI .actors 目录位置
if conf.download_actor_photo_for_kodi():
Expand Down

0 comments on commit daf431b

Please sign in to comment.