Skip to content

Commit

Permalink
Create config.py
Browse files Browse the repository at this point in the history
  • Loading branch information
68cdrBxM8YdoJ committed Apr 18, 2020
1 parent 1f4b7e6 commit 8332d65
Show file tree
Hide file tree
Showing 4 changed files with 263 additions and 182 deletions.
66 changes: 14 additions & 52 deletions ADC_function.py
Original file line number Diff line number Diff line change
@@ -1,39 +1,7 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
from configparser import ConfigParser
import os
import re
import time
import sys
from lxml import etree
import sys
import io
# sys.stdout = io.TextIOWrapper(sys.stdout.buffer, errors = 'replace', line_buffering = True)
# sys.setdefaultencoding('utf-8')

config_file = 'config.ini'
config = ConfigParser()

if os.path.exists(config_file):
try:
config.read(config_file, encoding='UTF-8')
except Exception as e:
print('[-]'+e)
print('[-]Config.ini read failed! Please use the offical file!')


def get_network_settings():
try:
proxy = config["proxy"]["proxy"]
timeout = int(config["proxy"]["timeout"])
retry_count = int(config["proxy"]["retry"])
assert timeout > 0
assert retry_count > 0
except:
raise ValueError("[-]Proxy config error! Please check the config.")
return proxy, timeout, retry_count
import config


def get_data_state(data: dict) -> bool: # 元数据获取失败检测
Expand All @@ -45,27 +13,18 @@ def get_data_state(data: dict) -> bool: # 元数据获取失败检测

return True

def ReadMediaWarehouse():
return config['media']['media_warehouse']

def UpdateCheckSwitch():
check=str(config['update']['update_check'])
if check == '1':
return '1'
elif check == '0':
return '0'
elif check == '':
return '0'

def getXpathSingle(htmlcode,xpath):
html = etree.fromstring(htmlcode, etree.HTMLParser())
result1 = str(html.xpath(xpath)).strip(" ['']")
return result1

def get_html(url,cookies = None):#网页请求核心
proxy, timeout, retry_count = get_network_settings()
i = 0
while i < retry_count:

# 网页请求核心
def get_html(url, cookies=None):
proxy, timeout, retry_count = config.Config().proxy()

for i in range(retry_count):
try:
if not proxy == '':
proxies = {"http": "http://" + proxy,"https": "https://" + proxy}
Expand All @@ -78,14 +37,15 @@ def get_html(url,cookies = None):#网页请求核心
getweb = requests.get(str(url), headers=headers, timeout=timeout, cookies=cookies)
getweb.encoding = 'utf-8'
return getweb.text
except:
i += 1
print('[-]Connect retry '+str(i)+'/'+str(retry_count))
except requests.exceptions.ProxyError:
print("[-]Connect retry {}/{}".format(i + 1, retry_count))
print('[-]Connect Failed! Please check your Proxy or Network!')
input("Press ENTER to exit!")
exit()


def post_html(url: str, query: dict) -> requests.Response:
proxy, timeout, retry_count = get_network_settings()
proxy, timeout, retry_count = config.Config().proxy()

if proxy:
proxies = {"http": "http://" + proxy, "https": "https://" + proxy}
Expand All @@ -99,3 +59,5 @@ def post_html(url: str, query: dict) -> requests.Response:
except requests.exceptions.ProxyError:
print("[-]Connect retry {}/{}".format(i+1, retry_count))
print("[-]Connect Failed! Please check your Proxy or Network!")
input("Press ENTER to exit!")
exit()
72 changes: 29 additions & 43 deletions AV_Data_Capture.py
Original file line number Diff line number Diff line change
@@ -1,33 +1,20 @@
#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import glob
import os
import time
import re
from ADC_function import *
from core import *
import json
import shutil
from configparser import ConfigParser
import argparse
from core import *


def check_update(current_version):
if UpdateCheckSwitch() == "1":
data = json.loads(get_html("https://api.github.com/repos/yoshiko2/AV_Data_Capture/releases/latest"))
data = json.loads(get_html("https://api.github.com/repos/yoshiko2/AV_Data_Capture/releases/latest"))

remote_version = data["tag_name"]
download_url = data["html_url"]
remote_version = data["tag_name"]
download_url = data["html_url"]

if current_version != remote_version:
line1 = "* New update " + remote_version + " *"
print("[*]" + line1.center(54))
print("[*]" + "↓ Download ↓".center(54))
print("[*] " + download_url)
print("[*]======================================================")

if current_version != remote_version:
line1 = "* New update " + remote_version + " *"
print("[*]" + line1.center(54))
print("[*]" + "↓ Download ↓".center(54))
print("[*] " + download_url)
print("[*]======================================================")
else:
print("[+]Update Check disabled!")

def argparse_function(switch):
parser = argparse.ArgumentParser()
Expand Down Expand Up @@ -59,7 +46,7 @@ def movie_lists(root, escape_folder):
return total


def CreatFailedFolder(failed_folder):
def create_failed_folder(failed_folder):
if not os.path.exists(failed_folder + '/'): # 新建failed文件夹
try:
os.makedirs(failed_folder + '/')
Expand Down Expand Up @@ -101,31 +88,30 @@ def getNumber(filepath,absolute_path = False):

if __name__ == '__main__':
version = '3.2'

config_file = argparse_function(2)
config = ConfigParser()
config.read(argparse_function(2), encoding='UTF-8')
success_folder = config['common']['success_output_folder']
failed_folder = config['common']['failed_output_folder'] # 失败输出目录
escape_folder = config['escape']['folders'] # 多级目录刮削需要排除的目录
escape_folder = re.split('[,,]', escape_folder)
conf = config.Config(path=config_file)

version_print = 'Version ' + version
print('[*]================== AV Data Capture ===================')
print('[*]' + version_print.center(54))
print('[*]======================================================')

check_update(version)
CreatFailedFolder(failed_folder)
if conf.update_check():
check_update(version)

create_failed_folder(conf.failed_folder())
os.chdir(os.getcwd())
movie_list = movie_lists('.', escape_folder)
movie_list = movie_lists(".", re.split("[,,]", conf.escape_folder()))

#========== 野鸡番号拖动 ==========
number_argparse = argparse_function(1)
if not number_argparse == '':
print("[!]Making Data for [" + number_argparse + "], the number is [" + getNumber(number_argparse,absolute_path = True) + "]")
core_main(number_argparse, getNumber(number_argparse,absolute_path = True))
print("[*]======================================================")
CEF(success_folder)
CEF(failed_folder)
CEF(conf.success_folder())
CEF(conf.failed_folder())
print("[+]All finished!!!")
input("[+][+]Press enter key exit, you can check the error messge before you exit.")
os._exit(0)
Expand All @@ -134,7 +120,7 @@ def getNumber(filepath,absolute_path = False):
count = 0
count_all = str(len(movie_list))
print('[+]Find', count_all, 'movies')
if config['common']['soft_link'] == '1':
if conf.soft_link():
print('[!] --- Soft link mode is ENABLE! ----')
for i in movie_list: # 遍历电影列表 交给core处理
count = count + 1
Expand All @@ -145,24 +131,24 @@ def getNumber(filepath,absolute_path = False):
# print("[*]======================================================")
try:
print("[!]Making Data for [" + i + "], the number is [" + getNumber(i) + "]")
core_main(i, getNumber(i), config_file=config_file)
core_main(i, getNumber(i), conf)
print("[*]======================================================")
except Exception as e: # 番号提取异常
print('[-]' + i + ' ERROR :')
print('[-]',e)
if config['common']['soft_link'] == '1':
if conf.soft_link():
print('[-]Link', i, 'to failed folder')
os.symlink(i, str(os.getcwd()) + '/' + failed_folder + '/')
os.symlink(i, str(os.getcwd()) + '/' + conf.failed_folder() + '/')
else:
try:
print('[-]Move ' + i + ' to failed folder')
shutil.move(i, str(os.getcwd()) + '/' + failed_folder + '/')
shutil.move(i, str(os.getcwd()) + '/' + conf.failed_folder() + '/')
except Exception as e2:
print('[!]', e2)
continue

CEF(success_folder)
CEF(failed_folder)
CEF(conf.success_folder())
CEF(conf.failed_folder())
print("[+]All finished!!!")
if argparse_function(3) == True:
os._exit(0)
Expand Down
125 changes: 125 additions & 0 deletions config.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
import os
import configparser


class Config:
def __init__(self, path: str = "config.ini"):
if os.path.exists(path):
self.conf = configparser.ConfigParser()
self.conf.read(path, encoding="utf-8")
else:
print("[-] Config file not found! Use the default settings")
self.conf = self._default_config()

def main_mode(self) -> str:
try:
return self.conf.getint("common", "main_mode")
except ValueError:
self._exit("common:main_mode")

def failed_folder(self) -> str:
return self.conf.get("common", "failed_output_folder")

def success_folder(self) -> str:
return self.conf.get("common", "success_output_folder")

def soft_link(self) -> bool:
return self.conf.getboolean("common", "soft_link")

def proxy(self) -> [str, int, int]:
try:
sec = "proxy"
proxy = self.conf.get(sec, "proxy")
timeout = self.conf.getint(sec, "timeout")
retry = self.conf.getint(sec, "retry")

return proxy, timeout, retry
except ValueError:
self._exit("common")

def naming_rule(self) -> str:
return self.conf.get("Name_Rule", "naming_rule")

def location_rule(self) -> str:
return self.conf.get("Name_Rule", "location_rule")

def update_check(self) -> bool:
try:
return self.conf.getboolean("update", "update_check")
except ValueError:
self._exit("update:update_check")

def sources(self) -> str:
return self.conf.get("priority", "website")

def escape_literals(self) -> str:
return self.conf.get("escape", "literals")

def escape_folder(self) -> str:
return self.conf.get("escape", "folders")

def debug(self) -> bool:
return self.conf.getboolean("debug_mode", "switch")

@staticmethod
def _exit(sec: str) -> None:
print("[-] Read config error! Please check the {} section in config.ini", sec)
input("[-] Press ENTER key to exit.")
exit()

@staticmethod
def _default_config() -> configparser.ConfigParser:
conf = configparser.ConfigParser()

sec1 = "common"
conf.add_section(sec1)
conf.set(sec1, "main_mode", "1")
conf.set(sec1, "failed_output_folder", "failed")
conf.set(sec1, "success_output_folder", "JAV_output")
conf.set(sec1, "soft_link", "0")

sec2 = "proxy"
conf.add_section(sec2)
conf.set(sec2, "proxy", "127.0.0.1:1080")
conf.set(sec2, "timeout", "10")
conf.set(sec2, "retry", "3")

sec3 = "Name_Rule"
conf.add_section(sec3)
conf.set(sec3, "location_rule", "actor + '/' + number")
conf.set(sec3, "naming_rule", "number + '-' + title")

sec4 = "update"
conf.add_section(sec4)
conf.set(sec4, "update_check", "1")

sec5 = "priority"
conf.add_section(sec5)
conf.set(sec5, "website", "javbus,javdb,fanza,xcity,mgstage,fc2,avsox,jav321,xcity")

sec6 = "escape"
conf.add_section(sec6)
conf.set(sec6, "literals", "\()/") # noqa
conf.set(sec6, "folders", "failed, JAV_output")

sec7 = "debug_mode"
conf.add_section(sec7)
conf.set(sec7, "switch", "0")

return conf


if __name__ == "__main__":
config = Config()
print(config.main_mode())
print(config.failed_folder())
print(config.success_folder())
print(config.soft_link())
print(config.proxy())
print(config.naming_rule())
print(config.location_rule())
print(config.update_check())
print(config.sources())
print(config.escape_literals())
print(config.escape_folder())
print(config.debug())
Loading

0 comments on commit 8332d65

Please sign in to comment.