From 8a42dbadde5a523deeecc12f5b6ff332e72c670a Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Mon, 15 Apr 2024 23:54:35 +0800 Subject: [PATCH 1/2] =?UTF-8?q?=E4=BF=AE=E5=A4=8D206=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=94=99=E8=AF=AF=EF=BC=8C=E4=BF=AE=E5=A4=8D=E7=BB=9F=E8=AE=A1?= =?UTF-8?q?=E6=95=B0=E6=8D=AEstorage.bin?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/cluster.py | 31 ++++++++++++++++++++----------- core/dashboard.py | 8 ++++++++ core/stats.py | 13 +++++++++---- core/web.py | 9 +++++---- main.py | 7 +++---- 5 files changed, 45 insertions(+), 23 deletions(-) diff --git a/core/cluster.py b/core/cluster.py index 8ac7627..25ab11b 100644 --- a/core/cluster.py +++ b/core/cluster.py @@ -739,17 +739,18 @@ async def get(self, file: str, offset: int = 0) -> File: file, size=int(resp.headers.get("Content-Length", 0)), ) - f.headers = {} - for field in ( - "ETag", - "Last-Modified", - "Content-Length", - "Content-Range", - ): - if field not in resp.headers: - continue - f.headers[field] = resp.headers.get(field) if resp.status == 200: + f.headers = {} + for field in ( + "ETag", + "Last-Modified", + "Content-Length", + "Content-Range", + "Accept-Ranges" + ): + if field not in resp.headers: + continue + f.headers[field] = resp.headers.get(field) f.set_data(await resp.read()) f.expiry = time.time() + CACHE_TIME elif resp.status // 100 == 3: @@ -908,7 +909,8 @@ async def get(self, hash: str, offset: int, ip: str, ua: str = "") -> Optional[F if not exists: return None file = await storage.get(hash, offset) - self._storage_stats[storage].hit(file, offset, ip, ua) + if file is not None: + self._storage_stats[storage].hit(file, offset, ip, ua) return file def get_storage_stat(self, storage): @@ -1009,7 +1011,10 @@ async def enable(self): if self.want_enable or self.enabled: logger.tdebug("cluster.debug.cluster.enable_again") return + timeoutTimer = None async def _(err, ack): + if timeoutTimer is not None: + timeoutTimer.block() self.want_enable = False if err: logger.terror( @@ -1027,6 +1032,9 @@ async def _(err, ack): ) await dashboard.set_status("cluster.enabled" + (".trusted" if self.trusted else "")) await self.keepalive() + async def _timeout(): + self.want_enable = False + await self.retry() self.want_enable = True self.keepalive_failed = 0 await self.channel_lock.wait() @@ -1061,6 +1069,7 @@ async def _(err, ack): }, callback=_ ) + timeoutTimer = Timer.delay(_timeout, delay=120) async def keepalive(self): def _clear(): if self.keepalive_timer is not None: diff --git a/core/dashboard.py b/core/dashboard.py index 0b1b8b9..d20a24b 100644 --- a/core/dashboard.py +++ b/core/dashboard.py @@ -177,7 +177,15 @@ async def process(type: str, data: Any): ) ) return data + if type == "global_stats": + return { + "ip": { + }, + "ua": { + + } + } async def get_cache_stats() -> StatsCache: stat = StatsCache() diff --git a/core/stats.py b/core/stats.py index ff15a24..e0f9041 100644 --- a/core/stats.py +++ b/core/stats.py @@ -222,7 +222,6 @@ def read_storage(): with open("./cache/storage.bin", "rb") as r: f = FileDataInputStream(r) last_hour = f.readVarInt() - GlobalStats.from_binary(f.read(f.readVarInt())) for _ in range(f.readVarInt()): storage = StorageStats(f.readString()) ( @@ -244,15 +243,18 @@ def read_storage(): ) storages[storage.get_name()] = storage + try: + blength = f.readVarInt() + bdata = f.read(bdata) + except: + return + GlobalStats.from_binary(bdata) def write_storage(): global storages, globalStats f = DataOutputStream() f.writeVarInt(last_hour) - data = globalStats.get_binary() - f.writeVarInt(len(data)) - f.write(data) f.writeVarInt(len(storages)) for storage in storages.values(): f.writeString(storage.get_name()) @@ -266,6 +268,9 @@ def write_storage(): storage._last_bytes, ): f.writeVarInt(field) + data = globalStats.get_binary() + f.writeVarInt(len(data)) + f.write(data) with open("./cache/storage.bin", "wb") as w: w.write(f.io.getbuffer()) diff --git a/core/web.py b/core/web.py index c9fd124..fa47fa6 100644 --- a/core/web.py +++ b/core/web.py @@ -830,7 +830,7 @@ async def __call__( r"bytes=(\d+)-", range_str, re.S ) end_bytes = length - 1 - if range_match: + if range_match and self.status_code == 200: start_bytes = int(range_match.group(1)) if range_match else 0 if range_match.lastindex == 2: end_bytes = int(range_match.group(2)) @@ -843,17 +843,18 @@ async def __call__( ) self.set_headers( { - "Accept-ranges": "bytes", + "Accept-Ranges": "bytes", "Content-Range": f"bytes {start_bytes}-{end_bytes}/{length}", } ) self.status_code = 206 if start_bytes > 0 else 200 + print(length, start_bytes) length = length - start_bytes if isinstance(content, io.BytesIO) and len(content.getbuffer()) != 0: self.content_type = self.content_type or self._get_content_type(content) compression: Compressor = compressor( await request.get_headers("Accept-Encoding", ""), - content.getbuffer()[start_bytes:length], + content.getbuffer()[start_bytes:end_bytes + 1], ) if compression.type is None: content = compression.data @@ -901,7 +902,7 @@ async def __call__( client.write(content.data) await client.writer.drain() elif isinstance(content, io.BytesIO): - client.write(content.getbuffer()[start_bytes:length]) + client.write(content.getbuffer()[start_bytes:end_bytes + 1]) await client.writer.drain() elif isinstance(content, Path): async with aiofiles.open(content, "rb") as r: diff --git a/main.py b/main.py index 8417158..d3ce7d0 100644 --- a/main.py +++ b/main.py @@ -18,10 +18,9 @@ ) if __name__ == "__main__": - if sys.version_info <= (3, 8): - print(f"Not support version: {sys.version}") - if sys.version_info == (3, 9): - print(f"Warning version: {sys.version}") + if sys.version_info <= (3, 9): + print(f"Not support version: {sys.version}. Please update python to 3.10+") + exit(-1) import core core.init() From e1f6a6f8926836978a85ebf441b0c182a30c349e Mon Sep 17 00:00:00 2001 From: tianxiu2b2t Date: Mon, 15 Apr 2024 23:59:25 +0800 Subject: [PATCH 2/2] =?UTF-8?q?=E6=97=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- core/web.py | 1 - 1 file changed, 1 deletion(-) diff --git a/core/web.py b/core/web.py index fa47fa6..06b08c9 100644 --- a/core/web.py +++ b/core/web.py @@ -848,7 +848,6 @@ async def __call__( } ) self.status_code = 206 if start_bytes > 0 else 200 - print(length, start_bytes) length = length - start_bytes if isinstance(content, io.BytesIO) and len(content.getbuffer()) != 0: self.content_type = self.content_type or self._get_content_type(content)