scraper.py

import json
import secrets
import config
from lxml import etree
from pathlib import Path

from ADC_function import delete_all_elements_in_list, delete_all_elements_in_str, file_modification_days, load_cookies, translate
from scrapinglib.api import search

def get_data_from_json(file_number, oCC, specified_source, specified_url):
    """
    iterate through all services and fetch the data 从JSON返回元数据
    """

    actor_mapping_data = etree.parse(str(Path.home() / '.local' / 'share' / 'mdc' / 'mapping_actor.xml'))
    info_mapping_data = etree.parse(str(Path.home() / '.local' / 'share' / 'mdc' / 'mapping_info.xml'))

    conf = config.getInstance()
    # default fetch order list, from the beginning to the end
    sources = conf.sources()

    # TODO 准备参数
    # - 清理 ADC_function, webcrawler
    proxies = None
    configProxy = conf.proxy()
    if configProxy.enable:
        proxies = configProxy.proxies()
    
    javdb_sites = conf.javdb_sites().split(',')
    for i in javdb_sites:
        javdb_sites[javdb_sites.index(i)] = "javdb" + i
    javdb_sites.append("javdb")
    # 不加载过期的cookie，javdb登录界面显示为7天免登录，故假定cookie有效期为7天
    has_json = False
    for cj in javdb_sites:
        javdb_site = cj
        cookie_json = javdb_site + '.json'
        cookies_dict, cookies_filepath = load_cookies(cookie_json)
        if isinstance(cookies_dict, dict) and isinstance(cookies_filepath, str):
            cdays = file_modification_days(cookies_filepath)
            if cdays < 7:
                javdb_cookies = cookies_dict
                has_json = True
                break
            elif cdays != 9999:
                print(f'[!]Cookies file {cookies_filepath} was updated {cdays} days ago, it will not be used for HTTP requests.')
    if not has_json:
        javdb_site = secrets.choice(javdb_sites)
        javdb_cookies = None

    cacert =None
    if conf.cacert_file():
        cacert = conf.cacert_file()

    json_data = search(file_number, sources, proxies=proxies, verify=cacert,
                        dbsite=javdb_site, dbcookies=javdb_cookies,
                        morestoryline=conf.is_storyline(),
                        specifiedSource=specified_source, specifiedUrl=specified_url)
    # Return if data not found in all sources
    if not json_data:
        print('[-]Movie Number not found!')
        return None

    # 增加number严格判断，避免提交任何number，总是返回"本橋実来 ADZ335"，这种返回number不一致的数据源故障
    # 目前选用number命名规则是javdb.com Domain Creation Date: 2013-06-19T18:34:27Z
    # 然而也可以跟进关注其它命名规则例如airav.wiki Domain Creation Date: 2019-08-28T07:18:42.0Z
    # 如果将来javdb.com命名规则下不同Studio出现同名碰撞导致无法区分，可考虑更换规则，更新相应的number分析和抓取代码。
    if str(json_data.get('number')).upper() != file_number.upper():
        try:
            if json_data.get('allow_number_change'):
                pass
        except:
            print('[-]Movie number has changed! [{}]->[{}]'.format(file_number, str(json_data.get('number'))))
            return None

    # ================================================网站规则添加结束================================================

    title = json_data.get('title')
    actor_list = str(json_data.get('actor')).strip("[ ]").replace("'", '').split(',')  # 字符串转列表
    actor_list = [actor.strip() for actor in actor_list]  # 去除空白
    director = json_data.get('director')
    release = json_data.get('release')
    number = json_data.get('number')
    studio = json_data.get('studio')
    source = json_data.get('source')
    runtime = json_data.get('runtime')
    outline = json_data.get('outline')
    label = json_data.get('label')
    series = json_data.get('series')
    year = json_data.get('year')

    if json_data.get('cover_small'):
        cover_small = json_data.get('cover_small')
    else:
        cover_small = ''

    if json_data.get('trailer'):
        trailer = json_data.get('trailer')
    else:
        trailer = ''

    if json_data.get('extrafanart'):
        extrafanart = json_data.get('extrafanart')
    else:
        extrafanart = ''

    imagecut = json_data.get('imagecut')
    tag = str(json_data.get('tag')).strip("[ ]").replace("'", '').replace(" ", '').split(',')  # 字符串转列表 @
    while 'XXXX' in tag:
        tag.remove('XXXX')
    while 'xxx' in tag:
        tag.remove('xxx')
    actor = str(actor_list).strip("[ ]").replace("'", '').replace(" ", '')

    if title == '' or number == '':
        print('[-]Movie Number or Title not found!')
        return None

    # if imagecut == '3':
    #     DownloadFileWithFilename()

    # ====================处理异常字符====================== #\/:*?"<>|
    actor = special_characters_replacement(actor)
    actor_list = [special_characters_replacement(a) for a in actor_list]
    title = special_characters_replacement(title)
    label = special_characters_replacement(label)
    outline = special_characters_replacement(outline)
    series = special_characters_replacement(series)
    studio = special_characters_replacement(studio)
    director = special_characters_replacement(director)
    tag = [special_characters_replacement(t) for t in tag]
    release = release.replace('/', '-')
    tmpArr = cover_small.split(',')
    if len(tmpArr) > 0:
        cover_small = tmpArr[0].strip('\"').strip('\'')
    # ====================处理异常字符 END================== #\/:*?"<>|

    # 返回处理后的json_data
    json_data['title'] = title
    json_data['original_title'] = title
    json_data['actor'] = actor
    json_data['release'] = release
    json_data['cover_small'] = cover_small
    json_data['tag'] = tag
    json_data['year'] = year
    json_data['actor_list'] = actor_list
    json_data['trailer'] = trailer
    json_data['extrafanart'] = extrafanart
    json_data['label'] = label
    json_data['outline'] = outline
    json_data['series'] = series
    json_data['studio'] = studio
    json_data['director'] = director

    if conf.is_translate():
        translate_values = conf.translate_values().split(",")
        for translate_value in translate_values:
            if json_data[translate_value] == "":
                continue
            if translate_value == "title":
                title_dict = json.loads(
                    (Path.home() / '.local' / 'share' / 'mdc' / 'c_number.json').read_text(encoding="utf-8"))
                try:
                    json_data[translate_value] = title_dict[number]
                    continue
                except:
                    pass
            if conf.get_translate_engine() == "azure":
                t = translate(
                    json_data[translate_value],
                    target_language="zh-Hans",
                    engine=conf.get_translate_engine(),
                    key=conf.get_translate_key(),
                )
            else:
                t = translate(json_data[translate_value])
            if len(t):
                json_data[translate_value] = special_characters_replacement(t)

    if oCC:
        cc_vars = conf.cc_convert_vars().split(",")
        ccm = conf.cc_convert_mode()
        def convert_list(mapping_data,language,vars):
            total = []
            for i in vars:
                if len(mapping_data.xpath('a[contains(@keyword, $name)]/@' + language, name=f",{i},")) != 0:
                    i = mapping_data.xpath('a[contains(@keyword, $name)]/@' + language, name=f",{i},")[0]
                total.append(i)
            return total
        def convert(mapping_data,language,vars):
            if len(mapping_data.xpath('a[contains(@keyword, $name)]/@' + language, name=vars)) != 0:
                return mapping_data.xpath('a[contains(@keyword, $name)]/@' + language, name=vars)[0]
            else:
                raise IndexError('keyword not found')
        for cc in cc_vars:
            if json_data[cc] == "" or len(json_data[cc]) == 0:
                continue
            if cc == "actor":
                try:
                    if ccm == 1:
                        json_data['actor_list'] = convert_list(actor_mapping_data, "zh_cn", json_data['actor_list'])
                        json_data['actor'] = convert(actor_mapping_data, "zh_cn", json_data['actor'])
                    elif ccm == 2:
                        json_data['actor_list'] = convert_list(actor_mapping_data, "zh_tw", json_data['actor_list'])
                        json_data['actor'] = convert(actor_mapping_data, "zh_tw", json_data['actor'])
                    elif ccm == 3:
                        json_data['actor_list'] = convert_list(actor_mapping_data, "jp", json_data['actor_list'])
                        json_data['actor'] = convert(actor_mapping_data, "jp", json_data['actor'])
                except:
                    json_data['actor_list'] = [oCC.convert(aa) for aa in json_data['actor_list']]
                    json_data['actor'] = oCC.convert(json_data['actor'])
            elif cc == "tag":
                try:
                    if ccm == 1:
                        json_data[cc] = convert_list(info_mapping_data, "zh_cn", json_data[cc])
                        json_data[cc] = delete_all_elements_in_list("删除", json_data[cc])
                    elif ccm == 2:
                        json_data[cc] = convert_list(info_mapping_data, "zh_tw", json_data[cc])
                        json_data[cc] = delete_all_elements_in_list("删除", json_data[cc])
                    elif ccm == 3:
                        json_data[cc] = convert_list(info_mapping_data, "jp", json_data[cc])
                        json_data[cc] = delete_all_elements_in_list("删除", json_data[cc])
                except:
                    json_data[cc] = [oCC.convert(t) for t in json_data[cc]]
            else:
                try:
                    if ccm == 1:
                        json_data[cc] = convert(info_mapping_data, "zh_cn", json_data[cc])
                        json_data[cc] = delete_all_elements_in_str("删除", json_data[cc])
                    elif ccm == 2:
                        json_data[cc] = convert(info_mapping_data, "zh_tw", json_data[cc])
                        json_data[cc] = delete_all_elements_in_str("删除", json_data[cc])
                    elif ccm == 3:
                        json_data[cc] = convert(info_mapping_data, "jp", json_data[cc])
                        json_data[cc] = delete_all_elements_in_str("删除", json_data[cc])
                except IndexError:
                    json_data[cc] = oCC.convert(json_data[cc])
                except:
                    pass

    naming_rule=""
    for i in conf.naming_rule().split("+"):
        if i not in json_data:
            naming_rule += i.strip("'").strip('"')
        else:
            item = json_data.get(i)
            naming_rule += item if type(item) is not list else "&".join(item)

    json_data['naming_rule'] = naming_rule
    return json_data


def special_characters_replacement(text) -> str:
    if not isinstance(text, str):
        return text
    return (text.replace('\\', '∖').     # U+2216 SET MINUS @ Basic Multilingual Plane
                replace('/', '∕').       # U+2215 DIVISION SLASH @ Basic Multilingual Plane
                replace(':', '꞉').       # U+A789 MODIFIER LETTER COLON @ Latin Extended-D
                replace('*', '∗').       # U+2217 ASTERISK OPERATOR @ Basic Multilingual Plane
                replace('?', '？').      # U+FF1F FULLWIDTH QUESTION MARK @ Basic Multilingual Plane
                replace('"', '＂').      # U+FF02 FULLWIDTH QUOTATION MARK @ Basic Multilingual Plane
                replace('<', 'ᐸ').       # U+1438 CANADIAN SYLLABICS PA @ Basic Multilingual Plane
                replace('>', 'ᐳ').       # U+1433 CANADIAN SYLLABICS PO @ Basic Multilingual Plane
                replace('|', 'ǀ').       # U+01C0 LATIN LETTER DENTAL CLICK @ Basic Multilingual Plane
                replace('&lsquo;', '‘'). # U+02018 LEFT SINGLE QUOTATION MARK
                replace('&rsquo;', '’'). # U+02019 RIGHT SINGLE QUOTATION MARK
                replace('&hellip;','…').
                replace('&amp;', '＆')
            )