From e3f462f0b7339a0b581a3e57ae0e97014a410649 Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 27 Apr 2024 19:32:34 +0800 Subject: [PATCH 1/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E6=9C=AC=E5=9C=B0?= =?UTF-8?q?=E5=AD=98=E5=82=A8=E6=97=B6=E5=80=99=E8=AF=BB=E5=8F=96=E6=95=B0?= =?UTF-8?q?=E6=8D=AE=E9=87=8F=E8=BF=87=E5=A4=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/api.py | 3 +++ core/cluster.py | 15 ++++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/core/api.py b/core/api.py index 7b44dc7..a574afc 100644 --- a/core/api.py +++ b/core/api.py @@ -57,6 +57,9 @@ def is_url(self): if not isinstance(self.path, str): return False return self.path.startswith("http://") or self.path.startswith("https://") + + def is_path(self): + return isinstance(self.path, Path) def get_path(self) -> str | Path: return self.path diff --git a/core/cluster.py b/core/cluster.py index 6262151..e19bb1b 100644 --- a/core/cluster.py +++ b/core/cluster.py @@ -528,13 +528,14 @@ async def get(self, hash: str, offset: int = 0) -> File: file.cache = True return file path = Path(str(self.dir) + f"/{hash[:2]}/{hash}") - buf = io.BytesIO() - async with aiofiles.open(path, "rb") as r: - while data := await r.read(IO_BUFFER): - buf.write(data) - file = File(path, hash, buf.tell(), time.time(), time.time()) - file.set_data(buf.getbuffer()) + file = File(path, hash, 0) if CACHE_ENABLE: + buf = io.BytesIO() + async with aiofiles.open(path, "rb") as r: + while data := await r.read(IO_BUFFER): + buf.write(data) + file = File(path, hash, buf.tell(), time.time(), time.time()) + file.set_data(buf.getbuffer()) self.cache[hash] = file file.cache = False return file @@ -1399,7 +1400,7 @@ async def _(request: web.Request, hash: str): if data.is_url() and isinstance(data.get_path(), str): return web.RedirectResponse(str(data.get_path())).set_headers(name) return web.Response( - data.get_data().getbuffer(), headers=data.headers or {} + data.get_data().getbuffer() if not data.is_path() else data.get_path(), headers=data.headers or {} ).set_headers(name) dir = Path("./bmclapi_dashboard/") From 48f1985f603b74dec7cafffa034a623efb3d27d4 Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 27 Apr 2024 20:21:53 +0800 Subject: [PATCH 2/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dweb.py=E5=8F=91=E9=80=81?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=AF=BB=E5=8F=96=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/web.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/web.py b/core/web.py index 26dafd0..8c3adf7 100644 --- a/core/web.py +++ b/core/web.py @@ -904,7 +904,7 @@ async def __call__( async with aiofiles.open(content, "rb") as r: cur_length: int = 0 await r.seek(start_bytes, os.SEEK_SET) - while data := await r.read(min(IO_BUFFER, length - cur_length)): + while data := await r.read(max(0, min(IO_BUFFER, length - cur_length))): cur_length += len(data) client.write(data) await client.drain() From 26b7f68eb8d1f38723b25531b95df4040cbffebe Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 27 Apr 2024 20:47:59 +0800 Subject: [PATCH 3/7] =?UTF-8?q?=E5=BC=83=E7=94=A8=E4=BA=86Minecraft=20VarI?= =?UTF-8?q?nt=E5=8D=8F=E8=AE=AE=EF=BC=88-1=E8=BF=9B=E5=85=A5=E6=AD=BB?= =?UTF-8?q?=E5=BE=AA=E7=8E=AF=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bmclapi_dashboard/static/js/index.min.js | 32 +++++++++++------------- core/utils.py | 28 ++++++++++----------- 2 files changed, 28 insertions(+), 32 deletions(-) diff --git a/bmclapi_dashboard/static/js/index.min.js b/bmclapi_dashboard/static/js/index.min.js index 8c293fd..250c300 100644 --- a/bmclapi_dashboard/static/js/index.min.js +++ b/bmclapi_dashboard/static/js/index.min.js @@ -1152,14 +1152,13 @@ class WebSocketClient { class MinecraftUtils { static getVarInt(data) { let r = []; - while (true) { - if ((data & 0xFFFFFF80) === 0) { - r.push(data); - break; - } - r.push(data & 0x7F | 0x80); - data >>= 7; + data = (data << 1) ^ (data >> 63) + while ((data & ~0x7F) != 0) { + r.push((data & 0x7f) | 0x80) + data >>= 7 } + r.push(data); + console.log(data) return r; } static getVarIntLength(data) { @@ -1295,16 +1294,15 @@ class DataInputStream extends BytesBuffer { return (new DataView(new Uint8Array(this.readBytes(4)))).getFloat32() } readVarInt() { - let i = 0; - let j = 0; - let k; - while (true) { - k = this.read(1)[0]; - i |= (k & 0x7F) << j * 7; - j += 1; - if ((k & 0x80) !== 128) break; - } - return i >= 2 ** 31 - 1 ? i - 2 ** 31 * 2 : i; + let b = this.read(1)[0] + let n = b & 0x7F + let shift = 7 + while ((b & 0x80) != 0) { + b = self.read(1)[0] + n |= (b & 0x7F) << shift + shift += 7 + } + return (n >> 1) ^ -(n & 1) } readString(maximum = null, encoding = 'utf-8') { return new TextDecoder(encoding).decode(new Uint8Array(this.read(maximum == null ? this.readVarInt() : maximum))); diff --git a/core/utils.py b/core/utils.py index 06b2716..b4a2597 100644 --- a/core/utils.py +++ b/core/utils.py @@ -10,6 +10,7 @@ import re import sys import time +import avro from typing import ( Any, Coroutine, @@ -523,12 +524,11 @@ class MinecraftUtils: @staticmethod def getVarInt(data: int): r: bytes = b"" - while 1: - if data & 0xFFFFFF80 == 0: - r += data.to_bytes(1, "big") - break - r += (data & 0x7F | 0x80).to_bytes(1, "big") + data = (data << 1) ^ (data >> 63) + while (data & ~0x7F) != 0: + r += ((data & 0x7f) | 0x80).to_bytes(1, "big") data >>= 7 + r += data.to_bytes(1, "big") return r @staticmethod @@ -621,16 +621,14 @@ def readLong(self) -> int: return value - 2**64 if value > 2**63 - 1 else value def readVarInt(self) -> int: - i: int = 0 - j: int = 0 - k: int - while 1: - k = int.from_bytes(self.read(1), byteorder="big") - i |= (k & 0x7F) << j * 7 - j += 1 - if (k & 0x80) != 128: - break - return i + b = ord(self.read(1)) + n = b & 0x7F + shift = 7 + while (b & 0x80) != 0: + b = ord(self.read(1)) + n |= (b & 0x7F) << shift + shift += 7 + return (n >> 1) ^ -(n & 1) def readString( self, maximun: Optional[int] = None, encoding: Optional[str] = None From 07df9e7acea839fe36673a449166d1c99820d863 Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 27 Apr 2024 20:50:33 +0800 Subject: [PATCH 4/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E4=B8=80=E4=B8=AApy?= =?UTF-8?q?=E8=BD=ACjs=E7=9A=84=E9=97=AE=E9=A2=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- bmclapi_dashboard/static/js/index.min.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bmclapi_dashboard/static/js/index.min.js b/bmclapi_dashboard/static/js/index.min.js index 250c300..ad21891 100644 --- a/bmclapi_dashboard/static/js/index.min.js +++ b/bmclapi_dashboard/static/js/index.min.js @@ -1158,7 +1158,6 @@ class MinecraftUtils { data >>= 7 } r.push(data); - console.log(data) return r; } static getVarIntLength(data) { @@ -1298,7 +1297,7 @@ class DataInputStream extends BytesBuffer { let n = b & 0x7F let shift = 7 while ((b & 0x80) != 0) { - b = self.read(1)[0] + b = this.read(1)[0] n |= (b & 0x7F) << shift shift += 7 } From ff8a614981757c6d23d5785431b7542a8d7cfe2d Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 27 Apr 2024 21:15:39 +0800 Subject: [PATCH 5/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=B9=E5=B0=B1?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=A7=A3=E6=9E=90=E7=9A=84=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/stats.py | 36 +++++++++++++++++++++++++++++++----- 1 file changed, 31 insertions(+), 5 deletions(-) diff --git a/core/stats.py b/core/stats.py index 6a8bb02..5a95350 100644 --- a/core/stats.py +++ b/core/stats.py @@ -556,11 +556,24 @@ def stats_pro(day): t, ): hour = (q[0] + get_utc_offset()) if not format_day else (q[0] + get_utc_offset()) // 24 - data = DataInputStream(zstd.decompress(q[1])) - for ip, c in {data.readString(): data.readVarInt() for _ in range(data.readVarInt())}.items(): - if hour not in s_ip: - s_ip[hour] = defaultdict(int) - s_ip[hour][ip] += c + try: + data = DataInputStream(zstd.decompress(q[1])) + for ip, c in {data.readString(): data.readVarInt() for _ in range(data.readVarInt())}.items(): + if hour not in s_ip: + s_ip[hour] = defaultdict(int) + s_ip[hour][ip] += c + except: + new_data = DataOutputStream() + data_ip = {data.readString(): old_data_read_varint(data) for _ in range(old_data_read_varint(data))} + new_data.writeVarInt(len(data_ip)) + for ip, c in data_ip.items(): + new_data.writeString(ip) + new_data.writeVarInt(c) + if hour not in s_ip: + s_ip[hour] = defaultdict(int) + s_ip[hour][ip] += c + execute("update access_ip set data = ? where hour = ?", zstd.compress(new_data.io.getbuffer()), hour) + addresses: defaultdict[location.IPInfo, int] = defaultdict(int) for ips in s_ip.values(): for address, count in ips.items(): @@ -575,3 +588,16 @@ def stats_pro(day): "bytes": file_bytes, "downloads": file_download } + + +def old_data_read_varint(input: DataInputStream): + i: int = 0 + j: int = 0 + k: int + while 1: + k = int.from_bytes(input.read(1), byteorder="big") + i |= (k & 0x7F) << j * 7 + j += 1 + if (k & 0x80) != 128: + break + return i \ No newline at end of file From 0df69542d05bc5ace0bdfa9019b051a6c852970f Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Sat, 27 Apr 2024 21:21:42 +0800 Subject: [PATCH 6/7] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=AF=B9=E6=97=A7?= =?UTF-8?q?=E6=95=B0=E6=8D=AE=E8=A7=A3=E6=9E=90=E7=9A=84=E9=94=99=E8=AF=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/stats.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/stats.py b/core/stats.py index 5a95350..3d096da 100644 --- a/core/stats.py +++ b/core/stats.py @@ -564,7 +564,7 @@ def stats_pro(day): s_ip[hour][ip] += c except: new_data = DataOutputStream() - data_ip = {data.readString(): old_data_read_varint(data) for _ in range(old_data_read_varint(data))} + data_ip = {data.readString(old_data_read_varint(data)): old_data_read_varint(data) for _ in range(old_data_read_varint(data))} new_data.writeVarInt(len(data_ip)) for ip, c in data_ip.items(): new_data.writeString(ip) From a2085fcf1c48b5b5dc44b5a73479f34c2c1af649 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sat, 27 Apr 2024 13:50:12 +0000 Subject: [PATCH 7/7] =?UTF-8?q?:sparkles:=20=E6=9B=B4=E6=96=B0=20VERSION?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- VERSION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/VERSION b/VERSION index f8ba16b..dadf289 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -v1.10.3-6b2e6c5 +v1.10.4-bb3ac7a