diff --git a/mars/xlog/crypt/decode_mars_nocrypt_log_file.py b/mars/xlog/crypt/decode_mars_nocrypt_log_file.py index db0661d31..adf49f152 100644 --- a/mars/xlog/crypt/decode_mars_nocrypt_log_file.py +++ b/mars/xlog/crypt/decode_mars_nocrypt_log_file.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 import sys import os @@ -50,7 +50,8 @@ def IsGoodLogBuffer(_buffer, _offset, count): headerLen = 1 + 2 + 1 + 1 + 4 + crypt_key_len if _offset + headerLen + 1 + 1 > len(_buffer): return (False, 'offset:%d > len(buffer):%d'%(_offset, len(_buffer))) - length = struct.unpack_from("I", buffer(_buffer, _offset+headerLen-4-crypt_key_len, 4))[0] + pos = _offset+headerLen-4-crypt_key_len + length = struct.unpack_from("I", memoryview(_buffer)[pos:pos+4])[0] if _offset + headerLen + length + 1 > len(_buffer): return (False, 'log length:%d, end pos %d > len(buffer):%d'%(length, _offset + headerLen + length + 1, len(_buffer))) if MAGIC_END!=_buffer[_offset + headerLen + length]: return (False, 'log length:%d, buffer[%d]:%d != MAGIC_END'%(length, _offset + headerLen + length, _buffer[_offset + headerLen + length])) @@ -81,7 +82,7 @@ def DecodeBuffer(_buffer, _offset, _outbuffer): if -1==fixpos: return -1 else: - _outbuffer.extend("[F]decode_log_file.py decode error len=%d, result:%s \n"%(fixpos, ret[1])) + _outbuffer.extend(b"[F]decode_log_file.py decode error len=%d, result:%s \n"%(fixpos, ret[1])) _offset += fixpos magic_start = _buffer[_offset] @@ -91,20 +92,24 @@ def DecodeBuffer(_buffer, _offset, _outbuffer): or MAGIC_SYNC_ZSTD_START==magic_start or MAGIC_SYNC_NO_CRYPT_ZSTD_START==magic_start or MAGIC_ASYNC_ZSTD_START==magic_start or MAGIC_ASYNC_NO_CRYPT_ZSTD_START==magic_start: crypt_key_len = 64 else: - _outbuffer.extend('in DecodeBuffer _buffer[%d]:%d != MAGIC_NUM_START'%(_offset, magic_start)) + _outbuffer.extend(b'in DecodeBuffer _buffer[%d]:%d != MAGIC_NUM_START'%(_offset, magic_start)) return -1 headerLen = 1 + 2 + 1 + 1 + 4 + crypt_key_len - length = struct.unpack_from("I", buffer(_buffer, _offset+headerLen-4-crypt_key_len, 4))[0] + pos = _offset+headerLen-4-crypt_key_len + length = struct.unpack_from("I", memoryview(_buffer)[pos:pos+4])[0] tmpbuffer = bytearray(length) - seq=struct.unpack_from("H", buffer(_buffer, _offset+headerLen-4-crypt_key_len-2-2, 2))[0] - begin_hour=struct.unpack_from("c", buffer(_buffer, _offset+headerLen-4-crypt_key_len-1-1, 1))[0] - end_hour=struct.unpack_from("c", buffer(_buffer, _offset+headerLen-4-crypt_key_len-1, 1))[0] + pos = _offset+headerLen-4-crypt_key_len-2-2 + seq=struct.unpack_from("H", memoryview(_buffer)[pos:pos+2])[0] + pos = _offset+headerLen-4-crypt_key_len-1-1 + begin_hour=struct.unpack_from("c", memoryview(_buffer)[pos:pos+1])[0] + pos = _offset+headerLen-4-crypt_key_len-1 + end_hour=struct.unpack_from("c", memoryview(_buffer)[pos:pos+1])[0] global lastseq if seq != 0 and seq != 1 and lastseq != 0 and seq != (lastseq+1): - _outbuffer.extend("[F]decode_log_file.py log seq:%d-%d is missing\n" %(lastseq+1, seq-1)) + _outbuffer.extend(b"[F]decode_log_file.py log seq:%d-%d is missing\n" %(lastseq+1, seq-1)) if seq != 0: lastseq = seq @@ -118,14 +123,14 @@ def DecodeBuffer(_buffer, _offset, _outbuffer): print("use wrong decode script") elif MAGIC_ASYNC_NO_CRYPT_ZSTD_START == _buffer[_offset]: decompressor = zstd.ZstdDecompressor() - tmpbuffer = next(decompressor.read_from(ZstdDecompressReader(str(tmpbuffer)), 100000, 1000000)) + tmpbuffer = next(decompressor.read_to_iter(ZstdDecompressReader(bytes(tmpbuffer)), 100000, 1000000)) elif MAGIC_COMPRESS_START==_buffer[_offset] or MAGIC_COMPRESS_NO_CRYPT_START==_buffer[_offset]: decompressor = zlib.decompressobj(-zlib.MAX_WBITS) - tmpbuffer = decompressor.decompress(str(tmpbuffer)) + tmpbuffer = decompressor.decompress(tmpbuffer) elif MAGIC_COMPRESS_START1==_buffer[_offset]: decompress_data = bytearray() while len(tmpbuffer) > 0: - single_log_len = struct.unpack_from("H", buffer(tmpbuffer, 0, 2))[0] + single_log_len = struct.unpack_from("H", memoryview(tmpbuffer)[0:2])[0] decompress_data.extend(tmpbuffer[2:single_log_len+2]) tmpbuffer[:] = tmpbuffer[single_log_len+2:len(tmpbuffer)] @@ -136,7 +141,7 @@ def DecodeBuffer(_buffer, _offset, _outbuffer): pass # _outbuffer.extend('seq:%d, hour:%d-%d len:%d decompress:%d\n' %(seq, ord(begin_hour), ord(end_hour), length, len(tmpbuffer))) - except Exception, e: + except Exception as e: traceback.print_exc() _outbuffer.extend("[F]decode_log_file.py decompress err, " + str(e) + "\n") return _offset+headerLen+length+1