From 330f0819a821fbcef427f2b3961d71798680bcbb Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 9 Mar 2024 21:11:58 +0800 Subject: [PATCH 1/3] Fix somethings --- container/bmclapi_dashboard/static/js/index.js | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/container/bmclapi_dashboard/static/js/index.js b/container/bmclapi_dashboard/static/js/index.js index f2ca83c..c2a5355 100644 --- a/container/bmclapi_dashboard/static/js/index.js +++ b/container/bmclapi_dashboard/static/js/index.js @@ -127,10 +127,10 @@ const calc_more_bytes = (...values) => { hits.push((obj.hits / 10000).toFixed(2)) bandwidth.push((obj.bandwidth * 8 / 1024.0 ** 2).toFixed(2)) } - core_modules_locals["master"]["nodes"] .setOption({tooltip:{formatter: e =>'
节点在线: '+e[0].data+'个
'},series: [{data: nodes}]}) - core_modules_locals["master"]["bytes"] .setOption({tooltip:{formatter: e =>'
流量: '+e[0].data+'TiB
'},series: [{data: bytes}]}) - core_modules_locals["master"]["req"] .setOption({tooltip:{formatter: e =>'
请求： '+e[0].data+'万
'},series: [{data: hits}]}) - core_modules_locals["master"]["bandwidth"] .setOption({tooltip:{formatter: e =>'
带宽： '+e[0].data+'Mbps
'},series: [{data: bandwidth}]}) + core_modules_locals["master"]["nodes"] .setOption({title: {text: "每小时在线节点（个）"},tooltip:{formatter: e =>'
节点在线: '+e[0].data+'个
'},series: [{data: nodes}]}) + core_modules_locals["master"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"},tooltip:{formatter: e =>'
流量: '+e[0].data+'GiB
'},series: [{data: bytes}]}) + core_modules_locals["master"]["req"] .setOption({title: {text: "每小时请求分布（万）"},tooltip:{formatter: e =>'
请求： '+e[0].data+'万
'},series: [{data: hits}]}) + core_modules_locals["master"]["bandwidth"] .setOption({title: {text: "平均每小时出网带宽（Mbps）"},tooltip:{formatter: e =>'
带宽： '+e[0].data+'Mbps
'},series: [{data: bandwidth}]}) core_modules_locals["master"]["load"] .setOption({series:[{type:"gauge",center:["50%","50%"],radius:"55%",min:0,max:100,axisLine:{lineStyle:{width:10,color:[[.3,"#2CE69B"],[.5,"#42AAFF"],[.7,"#F1BF4B"],[1,"#FF708D"]]}},pointer:{width:5},data:[{value:(100*data.load).toFixed(2)}]}]}); }) } @@ -140,7 +140,7 @@ const calc_more_bytes = (...values) => { core_modules_locals["master"]["bytes"] .setOption(core_modules_locals["master"]["options"]) core_modules_locals["master"]["req"] .setOption(core_modules_locals["master"]["options"]) core_modules_locals["master"].refresh() - setInterval(core_modules_locals["master"].refresh, 300) + setInterval(core_modules_locals["master"].refresh, 300000) } }, "resize": () => { @@ -150,6 +150,9 @@ const calc_more_bytes = (...values) => { core_modules_locals["master"]["req"] .resize() core_modules_locals["master"]["load"] .resize() }, + "disconnect": () => { + delete core_modules_locals["master"] + }, "page": () => { return [ ExtendFlex().append( From ce5eb9333dcb7802af3a8afbd3fb2fae53ee4e01 Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sun, 10 Mar 2024 01:15:05 +0800 Subject: [PATCH 2/3] Update --- .gitignore | 1 + .../bmclapi_dashboard/static/js/index.js | 13 ++++------- container/cluster.py | 22 ++++++++++++------- container/utils.py | 2 ++ container/web.py | 12 ++++++---- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/.gitignore b/.gitignore index d01d225..227d91a 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ container/config/ container/__pycache__/ .vscode/ .lh/ +test/ ======= config/config.properties __pycache__ diff --git a/container/bmclapi_dashboard/static/js/index.js b/container/bmclapi_dashboard/static/js/index.js index c2a5355..4792dc9 100644 --- a/container/bmclapi_dashboard/static/js/index.js +++ b/container/bmclapi_dashboard/static/js/index.js @@ -127,10 +127,10 @@ const calc_more_bytes = (...values) => { hits.push((obj.hits / 10000).toFixed(2)) bandwidth.push((obj.bandwidth * 8 / 1024.0 ** 2).toFixed(2)) } - core_modules_locals["master"]["nodes"] .setOption({title: {text: "每小时在线节点（个）"},tooltip:{formatter: e =>'
节点在线: '+e[0].data+'个
'},series: [{data: nodes}]}) - core_modules_locals["master"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"},tooltip:{formatter: e =>'
流量: '+e[0].data+'GiB
'},series: [{data: bytes}]}) - core_modules_locals["master"]["req"] .setOption({title: {text: "每小时请求分布（万）"},tooltip:{formatter: e =>'
请求： '+e[0].data+'万
'},series: [{data: hits}]}) - core_modules_locals["master"]["bandwidth"] .setOption({title: {text: "平均每小时出网带宽（Mbps）"},tooltip:{formatter: e =>'
带宽： '+e[0].data+'Mbps
'},series: [{data: bandwidth}]}) + core_modules_locals["master"]["nodes"] .setOption({title: {text: "每小时在线节点（个）"}, tooltip:{formatter: e =>'
节点在线: '+e[0].data+'个
'},series: [{data: nodes}]}) + core_modules_locals["master"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"}, tooltip:{formatter: e =>'
流量: '+e[0].data+'GiB
'},series: [{data: bytes}]}) + core_modules_locals["master"]["req"] .setOption({title: {text: "每小时请求分布（万）"}, tooltip:{formatter: e =>'
请求： '+e[0].data+'万
'},series: [{data: hits}]}) + core_modules_locals["master"]["bandwidth"] .setOption({title: {text: "平均每小时出网带宽（Mbps）"}, tooltip:{formatter: e =>'
带宽： '+e[0].data+'Mbps
'},series: [{data: bandwidth}]}) core_modules_locals["master"]["load"] .setOption({series:[{type:"gauge",center:["50%","50%"],radius:"55%",min:0,max:100,axisLine:{lineStyle:{width:10,color:[[.3,"#2CE69B"],[.5,"#42AAFF"],[.7,"#F1BF4B"],[1,"#FF708D"]]}},pointer:{width:5},data:[{value:(100*data.load).toFixed(2)}]}]}); }) } @@ -150,9 +150,6 @@ const calc_more_bytes = (...values) => { core_modules_locals["master"]["req"] .resize() core_modules_locals["master"]["load"] .resize() }, - "disconnect": () => { - delete core_modules_locals["master"] - }, "page": () => { return [ ExtendFlex().append( @@ -294,8 +291,6 @@ const calc_more_bytes = (...values) => { document.getElementsByClassName("list")[0].append(element.valueOf()) if (sub) document.getElementsByClassName("list")[0].append(sub.valueOf()) } - - // modules Object.values(core_modules).filter(v => "init" in v).forEach(v => v["init"]()) })(); diff --git a/container/cluster.py b/container/cluster.py index 1d1a1d4..d4a169f 100644 --- a/container/cluster.py +++ b/container/cluster.py @@ -1,5 +1,6 @@ import asyncio from dataclasses import dataclass +import glob import hashlib import hmac import io @@ -126,13 +127,13 @@ async def check_file(self): byte = 0 miss = [] for i, file in enumerate(filelist): - filepath = Path(str(self.dir) + f"/{file.hash[:2]}/{file.hash}") - if not filepath.exists() or filepath.stat().st_size != file.size: + filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}" + if not os.path.exists(filepath) or os.path.getsize(filepath) != file.size: miss.append(file) ... await asyncio.sleep(0) - b = utils.calc_more_bytes(byte, filesize) byte += file.size + b = utils.calc_more_bytes(byte, filesize) print(f"<<>>Check file {i}/{total} ({b[0]}/{b[1]})") if not miss: print(f"<<>>Checked all files!") @@ -161,6 +162,8 @@ async def check_file(self): eta = self.download_bytes.get_eta() print(f"<<>>Downloading files... {self.download_files.get_cur()}/{total} {b[0]}/{b[1]}, eta: {utils.format_time(eta if eta != -1 else None)}, total: {utils.format_time(self.download_bytes.get_total())}, Min: {bit[0]}, Cur: {bit[2]}, Max: {bit[1]}, Files: {self.download_files.get_cur_speed()}/s") await asyncio.sleep(1) + for timer in timers: + del timer await self.start_service() async def start_service(self): tokens = await token.getToken() @@ -264,13 +267,13 @@ async def __call__(self) -> io.BytesIO: if self.last < time.time(): stat = self.file.stat() if self.size == stat.st_size and self.last_file == stat.st_mtime: - self.last = time.time() + 600 + self.last = time.time() + 1440 return self.buf self.buf.seek(0, os.SEEK_SET) async with aiofiles.open(self.file, "rb") as r: while (data := await r.read(min(config.IO_BUFFER, stat.st_size - self.buf.tell()))) and self.buf.tell() < stat.st_size: self.buf.write(data) - self.last = time.time() + 600 + self.last = time.time() + 1440 self.size = stat.st_size self.last_file = stat.st_mtime self.buf.seek(0, os.SEEK_SET) @@ -313,8 +316,11 @@ async def _(request: web.Request): return Path("./bmclapi_dashboard/index.html") @router.get("/master") async def _(request: web.Request, url: str): - resp = await aiohttp.ClientSession(URL).get(url) - return resp.content.iter_chunked(config.REQUEST_BUFFER) # type: ignore + content = io.BytesIO() + async with aiohttp.ClientSession(URL) as session: + async with session.get(url) as resp: + content.write(await resp.read()) + return content # type: ignore app.mount(router) async def clearCache(): @@ -322,7 +328,7 @@ async def clearCache(): data = cache.copy() size = 0 for k, v in data.items(): - if v.access + 60 < time.time(): + if v.access + 1440 < time.time(): cache.pop(k) else: size += v.size diff --git a/container/utils.py b/container/utils.py index eec154c..0a15ac1 100644 --- a/container/utils.py +++ b/container/utils.py @@ -27,6 +27,8 @@ class Client: is_ssl: bool = False def get_server_port(self): return self.server_port + def invaild_ip(self): + return self.writer.get_extra_info("peername") == None def _record_after(self, start_time: float, data) -> bytes: if self.unchecked: return data diff --git a/container/web.py b/container/web.py index 7b94975..4a1131f 100644 --- a/container/web.py +++ b/container/web.py @@ -128,7 +128,7 @@ def get(self, path): return self.route(path, 'GET') def post(self, path): return self.route(path, 'POST') - async def handle(self, request: 'Request', client: Client): + async def handle(self, request: 'Request'): cur_route = None method = await request.get_method() url = request.get_url() @@ -190,9 +190,9 @@ async def handle(self, request: 'Request', client: Client): accept = await request.get_headers("Accept", ) or "" if "text/html" in accept: result = ClientErrorResonse.not_found(request) - await Response(content=result or '', headers=Header({ # type: ignore + yield Response(content=result or '', headers=Header({ # type: ignore "Server": "TTB-Network" - }))(request, client) + })) def mount(self, router: Router): self._routes.append(router) print(f"Serve router at: {router.prefix}") @@ -489,7 +489,8 @@ def process_part(boundary: bytes, files: dict[str, list[tempfile._TemporaryFileW async def handle(data, client: Client): try: request: Request = Request(data, client) - await app.handle(request, client) + async for resp in app.handle(request): + await resp(request, client) await request.skip() print(request.get_request_time(), "|", request.method.ljust(6), request.get_status_code(), "|", request.get_ip().ljust(16), "|", request.url, request.get_user_agent()) except TimeoutError: @@ -530,6 +531,8 @@ async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): timeout = config.TIMEOUT try: while (type := await client.readuntil(b"\r\n", timeout=timeout)): + if client.invaild_ip(): + break if b'HTTP/1.1' in type: await web.handle(type, client) timeout = 10 @@ -539,6 +542,7 @@ async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter): traceback.print_exc() async def main(): global cert, server + print(f"Loading...") load_cert() import cluster await cluster.init() From 89614386db36a2a8d9c1e713348ffb185f916f15 Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sun, 10 Mar 2024 01:21:12 +0800 Subject: [PATCH 3/3] Update --- container/bmclapi_dashboard/static/js/index.js | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/container/bmclapi_dashboard/static/js/index.js b/container/bmclapi_dashboard/static/js/index.js index 4792dc9..ce6c679 100644 --- a/container/bmclapi_dashboard/static/js/index.js +++ b/container/bmclapi_dashboard/static/js/index.js @@ -140,7 +140,7 @@ const calc_more_bytes = (...values) => { core_modules_locals["master"]["bytes"] .setOption(core_modules_locals["master"]["options"]) core_modules_locals["master"]["req"] .setOption(core_modules_locals["master"]["options"]) core_modules_locals["master"].refresh() - setInterval(core_modules_locals["master"].refresh, 300000) + core_modules_locals["master"]["timer"] = setInterval(core_modules_locals["master"].refresh, 300000) } }, "resize": () => { @@ -150,6 +150,12 @@ const calc_more_bytes = (...values) => { core_modules_locals["master"]["req"] .resize() core_modules_locals["master"]["load"] .resize() }, + "disconnect": () => { + if ("master" in core_modules_locals) { + clearInterval(core_modules_locals["master"]["timer"]) + delete core_modules_locals["master"] + } + }, "page": () => { return [ ExtendFlex().append(