Skip to content

Commit

Permalink
Merge branch 'bkp'
Browse files Browse the repository at this point in the history
  • Loading branch information
@tianxiu2b2t
tianxiu2b2t committed Mar 9, 2024
2 parents 4f3620d + 8961438 commit 7387ec4
Show file tree
Hide file tree
Showing 5 changed files with 36 additions and 19 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ container/config/
container/__pycache__/
.vscode/
.lh/
test/
=======
config/config.properties
__pycache__
Expand Down
18 changes: 11 additions & 7 deletions container/bmclapi_dashboard/static/js/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,10 +127,10 @@ const calc_more_bytes = (...values) => {
hits.push((obj.hits / 10000).toFixed(2))
bandwidth.push((obj.bandwidth * 8 / 1024.0 ** 2).toFixed(2))
}
core_modules_locals["master"]["nodes"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">节点在线: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'个</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: nodes}]})
core_modules_locals["master"]["bytes"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'TiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
core_modules_locals["master"]["req"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
core_modules_locals["master"]["bandwidth"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
core_modules_locals["master"]["nodes"] .setOption({title: {text: "每小时在线节点（个）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">节点在线: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'个</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: nodes}]})
core_modules_locals["master"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'GiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
core_modules_locals["master"]["req"] .setOption({title: {text: "每小时请求分布（万）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
core_modules_locals["master"]["bandwidth"] .setOption({title: {text: "平均每小时出网带宽（Mbps）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
core_modules_locals["master"]["load"] .setOption({series:[{type:"gauge",center:["50%","50%"],radius:"55%",min:0,max:100,axisLine:{lineStyle:{width:10,color:[[.3,"#2CE69B"],[.5,"#42AAFF"],[.7,"#F1BF4B"],[1,"#FF708D"]]}},pointer:{width:5},data:[{value:(100*data.load).toFixed(2)}]}]});
})
}
Expand All @@ -140,7 +140,7 @@ const calc_more_bytes = (...values) => {
core_modules_locals["master"]["bytes"] .setOption(core_modules_locals["master"]["options"])
core_modules_locals["master"]["req"] .setOption(core_modules_locals["master"]["options"])
core_modules_locals["master"].refresh()
setInterval(core_modules_locals["master"].refresh, 300)
core_modules_locals["master"]["timer"] = setInterval(core_modules_locals["master"].refresh, 300000)
}
},
"resize": () => {
Expand All @@ -150,6 +150,12 @@ const calc_more_bytes = (...values) => {
core_modules_locals["master"]["req"] .resize()
core_modules_locals["master"]["load"] .resize()
},
"disconnect": () => {
if ("master" in core_modules_locals) {
clearInterval(core_modules_locals["master"]["timer"])
delete core_modules_locals["master"]
}
},
"page": () => {
return [
ExtendFlex().append(
Expand Down Expand Up @@ -291,8 +297,6 @@ const calc_more_bytes = (...values) => {
document.getElementsByClassName("list")[0].append(element.valueOf())
if (sub) document.getElementsByClassName("list")[0].append(sub.valueOf())
}

// modules
Object.values(core_modules).filter(v => "init" in v).forEach(v => v["init"]())

})();
Expand Down
22 changes: 14 additions & 8 deletions container/cluster.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import asyncio
from dataclasses import dataclass
import glob
import hashlib
import hmac
import io
Expand Down Expand Up @@ -126,13 +127,13 @@ async def check_file(self):
byte = 0
miss = []
for i, file in enumerate(filelist):
filepath = Path(str(self.dir) + f"/{file.hash[:2]}/{file.hash}")
if not filepath.exists() or filepath.stat().st_size != file.size:
filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}"
if not os.path.exists(filepath) or os.path.getsize(filepath) != file.size:
miss.append(file)
...
await asyncio.sleep(0)
b = utils.calc_more_bytes(byte, filesize)
byte += file.size
b = utils.calc_more_bytes(byte, filesize)
print(f"<<<flush>>>Check file {i}/{total} ({b[0]}/{b[1]})")
if not miss:
print(f"<<<flush>>>Checked all files!")
Expand Down Expand Up @@ -161,6 +162,8 @@ async def check_file(self):
eta = self.download_bytes.get_eta()
print(f"<<<flush>>>Downloading files... {self.download_files.get_cur()}/{total} {b[0]}/{b[1]}, eta: {utils.format_time(eta if eta != -1 else None)}, total: {utils.format_time(self.download_bytes.get_total())}, Min: {bit[0]}, Cur: {bit[2]}, Max: {bit[1]}, Files: {self.download_files.get_cur_speed()}/s")
await asyncio.sleep(1)
for timer in timers:
del timer
await self.start_service()
async def start_service(self):
tokens = await token.getToken()
Expand Down Expand Up @@ -264,13 +267,13 @@ async def __call__(self) -> io.BytesIO:
if self.last < time.time():
stat = self.file.stat()
if self.size == stat.st_size and self.last_file == stat.st_mtime:
self.last = time.time() + 600
self.last = time.time() + 1440
return self.buf
self.buf.seek(0, os.SEEK_SET)
async with aiofiles.open(self.file, "rb") as r:
while (data := await r.read(min(config.IO_BUFFER, stat.st_size - self.buf.tell()))) and self.buf.tell() < stat.st_size:
self.buf.write(data)
self.last = time.time() + 600
self.last = time.time() + 1440
self.size = stat.st_size
self.last_file = stat.st_mtime
self.buf.seek(0, os.SEEK_SET)
Expand Down Expand Up @@ -313,16 +316,19 @@ async def _(request: web.Request):
return Path("./bmclapi_dashboard/index.html")
@router.get("/master")
async def _(request: web.Request, url: str):
resp = await aiohttp.ClientSession(URL).get(url)
return resp.content.iter_chunked(config.REQUEST_BUFFER) # type: ignore
content = io.BytesIO()
async with aiohttp.ClientSession(URL) as session:
async with session.get(url) as resp:
content.write(await resp.read())
return content # type: ignore
app.mount(router)

async def clearCache():
global cache
data = cache.copy()
size = 0
for k, v in data.items():
if v.access + 60 < time.time():
if v.access + 1440 < time.time():
cache.pop(k)
else:
size += v.size
Expand Down
2 changes: 2 additions & 0 deletions container/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ class Client:
is_ssl: bool = False
def get_server_port(self):
return self.server_port
def invaild_ip(self):
return self.writer.get_extra_info("peername") == None
def _record_after(self, start_time: float, data) -> bytes:
if self.unchecked:
return data
Expand Down
12 changes: 8 additions & 4 deletions container/web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,7 @@ def get(self, path):
return self.route(path, 'GET')
def post(self, path):
return self.route(path, 'POST')
async def handle(self, request: 'Request', client: Client):
async def handle(self, request: 'Request'):
cur_route = None
method = await request.get_method()
url = request.get_url()
Expand Down Expand Up @@ -190,9 +190,9 @@ async def handle(self, request: 'Request', client: Client):
accept = await request.get_headers("Accept", ) or ""
if "text/html" in accept:
result = ClientErrorResonse.not_found(request)
await Response(content=result or '', headers=Header({ # type: ignore
yield Response(content=result or '', headers=Header({ # type: ignore
"Server": "TTB-Network"
}))(request, client)
}))
def mount(self, router: Router):
self._routes.append(router)
print(f"Serve router at: {router.prefix}")
Expand Down Expand Up @@ -489,7 +489,8 @@ def process_part(boundary: bytes, files: dict[str, list[tempfile._TemporaryFileW
async def handle(data, client: Client):
try:
request: Request = Request(data, client)
await app.handle(request, client)
async for resp in app.handle(request):
await resp(request, client)
await request.skip()
print(request.get_request_time(), "|", request.method.ljust(6), request.get_status_code(), "|", request.get_ip().ljust(16), "|", request.url, request.get_user_agent())
except TimeoutError:
Expand Down Expand Up @@ -530,6 +531,8 @@ async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
timeout = config.TIMEOUT
try:
while (type := await client.readuntil(b"\r\n", timeout=timeout)):
if client.invaild_ip():
break
if b'HTTP/1.1' in type:
await web.handle(type, client)
timeout = 10
Expand All @@ -539,6 +542,7 @@ async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
traceback.print_exc()
async def main():
global cert, server
print(f"Loading...")
load_cert()
import cluster
await cluster.init()
Expand Down

0 comments on commit 7387ec4

Please sign in to comment.