Skip to content

Commit

Permalink
Merge branch 'master' into dev/config
Browse files Browse the repository at this point in the history
  • Loading branch information
@SilianZ
SilianZ committed Mar 9, 2024
2 parents b29d35a + 7387ec4 commit 651e1e5
Show file tree
Hide file tree
Showing 5 changed files with 32 additions and 20 deletions.
1 change: 0 additions & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ container/cache/
container/config/
container/__pycache__/
.vscode/
.lh/
config/config.properties
__pycache__
.venv
Expand Down
18 changes: 11 additions & 7 deletions container/bmclapi_dashboard/static/js/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -128,10 +128,10 @@ const calc_more_bytes = (...values) => {
hits.push((obj.hits / 10000).toFixed(2))
bandwidth.push((obj.bandwidth * 8 / 1024.0 ** 2).toFixed(2))
}
core_modules_locals["master"]["nodes"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">节点在线: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'个</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: nodes}]})
core_modules_locals["master"]["bytes"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'TiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
core_modules_locals["master"]["req"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
core_modules_locals["master"]["bandwidth"] .setOption({tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
core_modules_locals["master"]["nodes"] .setOption({title: {text: "每小时在线节点（个）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">节点在线: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'个</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: nodes}]})
core_modules_locals["master"]["bytes"] .setOption({title: {text: "每小时流量分布（GiB）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">流量: </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'GiB</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bytes}]})
core_modules_locals["master"]["req"] .setOption({title: {text: "每小时请求分布（万）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">请求： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'万</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: hits}]})
core_modules_locals["master"]["bandwidth"] .setOption({title: {text: "平均每小时出网带宽（Mbps）"}, tooltip:{formatter: e =>'<div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><div style="margin: 0px 0 0;line-height:1;"><span style="display:inline-block;margin-right:4px;border-radius:10px;width:10px;height:10px;background-color:#0fc6c2;"></span><span style="font-size:14px;color:#666;font-weight:400;margin-left:2px">带宽： </span><span style="float:right;margin-left:20px;font-size:14px;color:#666;font-weight:900">'+e[0].data+'Mbps</span><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div><div style="clear:both"></div></div>'},series: [{data: bandwidth}]})
core_modules_locals["master"]["load"] .setOption({series:[{type:"gauge",center:["50%","50%"],radius:"55%",min:0,max:100,axisLine:{lineStyle:{width:10,color:[[.3,"#2CE69B"],[.5,"#42AAFF"],[.7,"#F1BF4B"],[1,"#FF708D"]]}},pointer:{width:5},data:[{value:(100*data.load).toFixed(2)}]}]});
})
}
Expand All @@ -141,7 +141,7 @@ const calc_more_bytes = (...values) => {
core_modules_locals["master"]["bytes"] .setOption(core_modules_locals["master"]["options"])
core_modules_locals["master"]["req"] .setOption(core_modules_locals["master"]["options"])
core_modules_locals["master"].refresh()
setInterval(core_modules_locals["master"].refresh, 300)
core_modules_locals["master"]["timer"] = setInterval(core_modules_locals["master"].refresh, 300000)
}
},
"resize": () => {
Expand All @@ -151,6 +151,12 @@ const calc_more_bytes = (...values) => {
core_modules_locals["master"]["req"] .resize()
core_modules_locals["master"]["load"] .resize()
},
"disconnect": () => {
if ("master" in core_modules_locals) {
clearInterval(core_modules_locals["master"]["timer"])
delete core_modules_locals["master"]
}
},
"page": () => {
return [
ExtendFlex().append(
Expand Down Expand Up @@ -289,8 +295,6 @@ const calc_more_bytes = (...values) => {
document.getElementsByClassName("list")[0].append(element.valueOf())
if (sub) document.getElementsByClassName("list")[0].append(sub.valueOf())
}

// modules
Object.values(core_modules).filter(v => "init" in v).forEach(v => v["init"]())

})();
Expand Down
19 changes: 11 additions & 8 deletions container/cluster.py
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import asyncio
from dataclasses import dataclass
import glob
import hashlib
import hmac
import io
Expand Down Expand Up @@ -131,12 +132,11 @@ async def check_file(self):
pbar = tqdm(total=total, unit=' file(s)', unit_scale=True)
pbar.set_description("Checking files")
for i, file in enumerate(filelist):
filepath = Path(str(self.dir) + f"/{file.hash[:2]}/{file.hash}")
if not filepath.exists() or filepath.stat().st_size != file.size:
filepath = str(self.dir) + f"/{file.hash[:2]}/{file.hash}"
if not os.path.exists(filepath) or os.path.getsize(filepath) != file.size:
miss.append(file)
...
await asyncio.sleep(0)
b = utils.calc_more_bytes(byte, filesize)
byte += file.size
pbar.update(1)
if not miss:
Expand Down Expand Up @@ -272,13 +272,13 @@ async def __call__(self) -> io.BytesIO:
if self.last < time.time():
stat = self.file.stat()
if self.size == stat.st_size and self.last_file == stat.st_mtime:
self.last = time.time() + 600
self.last = time.time() + 1440
return self.buf
self.buf.seek(0, os.SEEK_SET)
async with aiofiles.open(self.file, "rb") as r:
while (data := await r.read(min(config.IO_BUFFER, stat.st_size - self.buf.tell()))) and self.buf.tell() < stat.st_size:
self.buf.write(data)
self.last = time.time() + 600
self.last = time.time() + 1440
self.size = stat.st_size
self.last_file = stat.st_mtime
self.buf.seek(0, os.SEEK_SET)
Expand Down Expand Up @@ -321,16 +321,19 @@ async def _(request: web.Request):
return Path("./bmclapi_dashboard/index.html")
@router.get("/master")
async def _(request: web.Request, url: str):
resp = await aiohttp.ClientSession(URL).get(url)
return resp.content.iter_chunked(config.REQUEST_BUFFER) # type: ignore
content = io.BytesIO()
async with aiohttp.ClientSession(URL) as session:
async with session.get(url) as resp:
content.write(await resp.read())
return content # type: ignore
app.mount(router)

async def clearCache():
global cache
data = cache.copy()
size = 0
for k, v in data.items():
if v.access + 60 < time.time():
if v.access + 1440 < time.time():
cache.pop(k)
else:
size += v.size
Expand Down
2 changes: 2 additions & 0 deletions container/utils.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ class Client:
is_ssl: bool = False
def get_server_port(self):
return self.server_port
def invaild_ip(self):
return self.writer.get_extra_info("peername") == None
def _record_after(self, start_time: float, data) -> bytes:
if self.unchecked:
return data
Expand Down
12 changes: 8 additions & 4 deletions container/web.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,7 @@ def get(self, path):
return self.route(path, 'GET')
def post(self, path):
return self.route(path, 'POST')
async def handle(self, request: 'Request', client: Client):
async def handle(self, request: 'Request'):
cur_route = None
method = await request.get_method()
url = request.get_url()
Expand Down Expand Up @@ -191,9 +191,9 @@ async def handle(self, request: 'Request', client: Client):
accept = await request.get_headers("Accept", ) or ""
if "text/html" in accept:
result = ClientErrorResonse.not_found(request)
await Response(content=result or '', headers=Header({ # type: ignore
yield Response(content=result or '', headers=Header({ # type: ignore
"Server": "TTB-Network"
}))(request, client)
}))
def mount(self, router: Router):
self._routes.append(router)
logger.info(f"Serve router at: {router.prefix}")
Expand Down Expand Up @@ -490,7 +490,8 @@ def process_part(boundary: bytes, files: dict[str, list[tempfile._TemporaryFileW
async def handle(data, client: Client):
try:
request: Request = Request(data, client)
await app.handle(request, client)
async for resp in app.handle(request):
await resp(request, client)
await request.skip()
logger.info(request.method.ljust(6), request.get_status_code(), "|", request.get_ip().ljust(16), "|", request.url, request.get_user_agent())
except TimeoutError:
Expand Down Expand Up @@ -531,6 +532,8 @@ async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
timeout = config.TIMEOUT
try:
while (type := await client.readuntil(b"\r\n", timeout=timeout)):
if client.invaild_ip():
break
if b'HTTP/1.1' in type:
await web.handle(type, client)
timeout = 10
Expand All @@ -540,6 +543,7 @@ async def _handle(reader: asyncio.StreamReader, writer: asyncio.StreamWriter):
traceback.print_exc()
async def main():
global cert, server
print(f"Loading...")
load_cert()
import cluster
await cluster.init()
Expand Down

0 comments on commit 651e1e5

Please sign in to comment.