mitmproxy + python 實(shí)現(xiàn)游戲協(xié)議測試

本文側(cè)重介紹如何使用 python 和 mitmproxy 實(shí)現(xiàn)攔截?cái)?shù)據(jù)包、重發(fā)數(shù)據(jù)包，以及解析 protobuf 數(shù)據(jù)內(nèi)容，對于相關(guān)依賴的安裝不做介紹。

一、游戲協(xié)議安全測試內(nèi)容

參考https://testerhome.com/topics/29053，這篇文章講的很清楚。

二、實(shí)現(xiàn)原理

想直接使用的同學(xué)可以跳到第三部分。

mitmproxy 作為代理，可以獲取客戶端與服務(wù)端通信的數(shù)據(jù)，并且可以攔截、修改和自主發(fā)送數(shù)據(jù)。當(dāng)配合其證書使用時，還可以解密 wss 連接中的 websocket 數(shù)據(jù)。

• Websotcket 數(shù)據(jù)處理源碼分析

在 http 代理的過程中若發(fā)現(xiàn) upgrade websocket 請求，則創(chuàng)建 WebSocketLayer 實(shí)例，并調(diào)用其<u style="box-sizing: border-box; outline-style: none; --tw-border-opacity:1; border-color: rgb(229 231 235/var(--tw-border-opacity));">call</u>方法。

# mitmproxy/proxy/protocol/http.py
"""以下為Httplayer的_process_flow方法的部分代碼"""
if f.response.status_code == 101:
    # Handle a successful HTTP 101 Switching Protocols Response,
    # received after e.g. a WebSocket upgrade request.
    # Check for WebSocket handshake
    is_websocket = (
        websockets.check_handshake(f.request.headers) and
        websockets.check_handshake(f.response.headers)
    )
    if is_websocket and not self.config.options.websocket:
        self.log(
          "Client requested WebSocket connection, but the protocol is disabled.",
          "info"
        )

    if is_websocket and self.config.options.websocket:
        layer = WebSocketLayer(self, f)
    else:
        layer = self.ctx.next_layer(self)
    layer()

WebSocketLayer 初始化時會創(chuàng)建用于此次 websocket 通信的編解碼器。

# mitmproxy/proxy/protocol/websocket.py
"""WebSocketLayer類的init方法，省略部分代碼"""
def __init__(self, ctx, handshake_flow):
    super().__init__(ctx)
    self.handshake_flow = handshake_flow

    self.connections: dict[object, WSConnection] = {}

    client_extensions = []
    server_extensions = []
    # 判斷交互數(shù)據(jù)是否使用deflate壓縮
    if 'Sec-WebSocket-Extensions' in handshake_flow.response.headers:
        if PerMessageDeflate.name in handshake_flow.response.headers['Sec-WebSocket-Extensions']:
            client_extensions = [PerMessageDeflate()]
            server_extensions = [PerMessageDeflate()]
    # self.client_conn和self.server_conn繼承自ctx,即原h(huán)ttp的client和server，原理為父類的__getattr__(self, name)方法返回的是getattr(self.ctx, name)。WSConnection是一個websocket協(xié)議編解碼器，實(shí)際不會發(fā)送任何網(wǎng)絡(luò)IO，文檔地址：https://python-hyper.org/projects/wsproto/en/latest/basic-usage.html
    # 負(fù)責(zé)和解碼server收到信息和編碼server發(fā)送的信息
    self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER)
    # 負(fù)責(zé)和解碼client收到信息和編碼client發(fā)送的信息
    self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT)

    # 構(gòu)造發(fā)送給Server的websocket的握手請求
    request = Request(extensions=client_extensions,host=handshake_flow.request.host,target=handshake_flow.request.path)
    # send()方法只會構(gòu)造一個適用于對應(yīng)conn的數(shù)據(jù)，并不會真正發(fā)送數(shù)據(jù)，recv_data()會將信息解碼，需要通過next(conn.events())獲取解碼后數(shù)據(jù)
    # 按上所說，下面兩行代碼的操作是將握手請求按client編碼后發(fā)送給server編碼器，然后讓server編碼器解碼
    data = self.connections[self.server_conn].send(request)
    self.connections[self.client_conn].receive_data(data)

    event = next(self.connections[self.client_conn].events())
    assert isinstance(event, events.Request)
    # 返回給客戶端接受連接響應(yīng)
    data = self.connections[self.client_conn].send(AcceptConnection(extensions=server_extensions))
    self.connections[self.server_conn].receive_data(data)
    assert isinstance(next(self.connections[self.server_conn].events()), events.AcceptConnection)

WebSocketLayer 實(shí)例的<u style="box-sizing: border-box; outline-style: none; --tw-border-opacity:1; border-color: rgb(229 231 235/var(--tw-border-opacity));">call</u>方法負(fù)責(zé)處理后續(xù) websocket 通信

# mitmproxy/proxy/protocol/websocket.py
"""WebSocketLayer類的call方法，省略部分代碼"""
def __call__(self):
    self.flow = WebSocketFlow(self.client_conn, self.server_conn, self.handshake_flow)
    self.flow.metadata['websocket_handshake'] = self.handshake_flow.id
    self.handshake_flow.metadata['websocket_flow'] = self.flow.id
    # 調(diào)用addons中的websocket_start(self, flow)對flow進(jìn)行處理
    self.channel.ask("websocket_start", self.flow)

    conns = [c.connection for c in self.connections.keys()]
    close_received = False

    try:
        while not self.channel.should_exit.is_set():
            # 往client或server插入信息，self.flow._inject_messages_client/self.flow._inject_messages_server是隊(duì)列，后續(xù)實(shí)現(xiàn)在連接中主動發(fā)消息就是通過往隊(duì)列中插入數(shù)據(jù)實(shí)現(xiàn)
            self._inject_messages(self.client_conn, self.flow._inject_messages_client)
            self._inject_messages(self.server_conn, self.flow._inject_messages_server)
        # select監(jiān)視原h(huán)ttp的client和server連接的可讀事件
            r = tcp.ssl_read_select(conns, 0.1)
            for conn in r:
                source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn
                other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn
                is_server = (source_conn == self.server_conn)

                frame = websockets.Frame.from_file(source_conn.rfile)
                # 將從conn中獲取的數(shù)據(jù)放入編解碼器，此方法并沒有返回值，所以data是None
                data = self.connections[source_conn].receive_data(bytes(frame))
                # data是None，不解此舉有何意義
                source_conn.send(data)

                if close_received:
                    return
        # 處理編解碼器中解碼后的數(shù)據(jù)，event由pop取出，后續(xù)不會再用到。
                for event in self.connections[source_conn].events():
                    if not self._handle_event(event, source_conn, other_conn, is_server):
                        if not close_received:
                            close_received = True
    except (socket.error, exceptions.TcpException, SSL.Error) as e:
        s = 'server' if is_server else 'client'
        self.flow.error = flow.Error("WebSocket connection closed unexpectedly by {}: {}".format(s, repr(e)))
        # 調(diào)用addons中的websocket_start(self, flow)對flow進(jìn)行處理
        self.channel.tell("websocket_start", self.flow)
    finally:
        self.flow.ended = True
        # 調(diào)用addons中的websocket_end(self, flow)對flow進(jìn)行處理
        self.channel.tell("websocket_end", self.flow)

WebSocketLayer 實(shí)例中處理 Message Event 的方法

# mitmproxy/proxy/protocol/websocket.py
"""WebSocketLayer類的_handle_message方法，_handle_event中，若isinstance(event, events.Message)，則會調(diào)用此函數(shù)"""
def _handle_message(self, event, source_conn, other_conn, is_server):
    fb = self.server_frame_buffer if is_server else self.client_frame_buffer
    fb.append(event.data)

    if event.message_finished:
        original_chunk_sizes = [len(f) for f in fb]

        if isinstance(event, events.TextMessage):
            message_type = wsproto.frame_protocol.Opcode.TEXT
            payload = ''.join(fb)
        else:
            message_type = wsproto.frame_protocol.Opcode.BINARY
            payload = b''.join(fb)

        fb.clear()

        websocket_message = WebSocketMessage(message_type, not is_server, payload)
        length = len(websocket_message.content)
        self.flow.messages.append(websocket_message)
        # 調(diào)用addons中的websocket_message(self, flow)對flow進(jìn)行處理
        self.channel.ask("websocket_message", self.flow)

        # WebsocketMessage的屬性killed用于判斷該信息是否需要被轉(zhuǎn)發(fā)，可在websocket_message函數(shù)中調(diào)用message的kill()方法置為True
        if not self.flow.stream and not websocket_message.killed:
            def get_chunk(payload):
                if len(payload) == length:
                    # message has the same length, we can reuse the same sizes
                    pos = 0
                    for s in original_chunk_sizes:
                        yield (payload[pos:pos + s], True if pos + s == length else False)
                        pos += s
                else:
                    # just re-chunk everything into 4kB frames
                    # header len = 4 bytes without masking key and 8 bytes with masking key
                    chunk_size = 4092 if is_server else 4088
                    chunks = range(0, len(payload), chunk_size)
                    for i in chunks:
                        yield (payload[i:i + chunk_size], True if i + chunk_size >= len(payload) else False)

            # 將收到的信息重新編碼后向?qū)Χ税l(fā)送
            for chunk, final in get_chunk(websocket_message.content):
                data = self.connections[other_conn].send(Message(data=chunk, message_finished=final))
                other_conn.send(data)

    if self.flow.stream:
        data = self.connections[other_conn].send(Message(data=event.data, message_finished=event.message_finished))
        other_conn.send(data)
    return True

• Tcp 數(shù)據(jù)處理源碼分析

TCP 數(shù)據(jù)處理觸發(fā)條件

# mitmproxy/proxy/root_context.py
"""RootContext類_next_layer方法，省略部分代碼"""
"""
4\. Check for --tcp
判斷Option中tcp_hosts, 類型是一個列表，包含需要轉(zhuǎn)換成tcp流信息的server address正則表達(dá)式，例如['192\.168\.\d+\.\d+']
"""
if self.config.check_tcp(top_layer.server_conn.address):
    return protocol.RawTCPLayer(top_layer)

"""
6\. Check for raw tcp mode
判斷Option中rawtcp，類型是bool，若為true，則將不能處理的流轉(zhuǎn)換成tcp流處理，建議開啟，默認(rèn)是false
"""
is_ascii = (
    len(d) == 3 and
    # expect A-Za-z
    all(65 <= x <= 90 or 97 <= x <= 122 for x in d)
)
if self.config.options.rawtcp and not is_ascii:
    return protocol.RawTCPLayer(top_layer)

TCP 信息處理 RawTCPLayer 類源碼

class RawTCPLayer(base.Layer):
    chunk_size = 4096

    def __init__(self, ctx, ignore=False):
        self.ignore = ignore
        super().__init__(ctx)

    def __call__(self):
        self.connect()

        if not self.ignore:
            f = tcp.TCPFlow(self.client_conn, self.server_conn, self)
            # 調(diào)用addons中的tcp_start(self, flow)對flow進(jìn)行處理
            self.channel.ask("tcp_start", f)

        # 創(chuàng)建一個長度為4096的空bytearray
        buf = memoryview(bytearray(self.chunk_size))

        client = self.client_conn.connection
        server = self.server_conn.connection
        conns = [client, server]

        # https://github.com/openssl/openssl/issues/6234
        for conn in conns:
            if isinstance(conn, SSL.Connection) and hasattr(SSL._lib, "SSL_clear_mode"):
                SSL._lib.SSL_clear_mode(conn._ssl, SSL._lib.SSL_MODE_AUTO_RETRY)

        try:
            while not self.channel.should_exit.is_set():
                r = mitmproxy.net.tcp.ssl_read_select(conns, 10)
                for conn in r:
                    dst = server if conn == client else client
                    try:
                        # 將從conn中recv的數(shù)據(jù)存入buf，返回size
                        size = conn.recv_into(buf, self.chunk_size)
                    except (SSL.WantReadError, SSL.WantWriteError):
                        continue
                    if not size:
                        conns.remove(conn)
                        # Shutdown connection to the other peer
                        if isinstance(conn, SSL.Connection):
                            # We can't half-close a connection, so we just close everything here.
                            # Sockets will be cleaned up on a higher level.
                            return
                        else:
                            dst.shutdown(socket.SHUT_WR)

                        if len(conns) == 0:
                            return
                        continue
            # 將recv的數(shù)據(jù)轉(zhuǎn)成TCPMessage
                    tcp_message = tcp.TCPMessage(dst == server, buf[:size].tobytes())
                    if not self.ignore:
                        f.messages.append(tcp_message)
                        # 調(diào)用addons中的tcp_message(self, flow)對flow進(jìn)行處理
                        self.channel.ask("tcp_message", f)
                    # 發(fā)送tcp_message中的content
                    dst.sendall(tcp_message.content)

        except (socket.error, exceptions.TcpException, SSL.Error) as e:
            if not self.ignore:
                f.error = flow.Error("TCP connection closed unexpectedly: {}".format(repr(e)))
                # 調(diào)用addons中的tcp_error(self, flow)對flow進(jìn)行處理
                self.channel.tell("tcp_error", f)
        finally:
            if not self.ignore:
                # 調(diào)用addons中的tcp_end(self, flow)對flow進(jìn)行處理
                self.channel.tell("tcp_end", f)

三、開啟 mitmproxy 并加載 addon

首先需要安裝兩個庫：mitmproxy 和 mitmdump

1、編寫 websocket 的 addon

"""
簡略版用于websocket的Addon
后續(xù)改進(jìn)可以增加判斷host，避免攔截到不需要處理的連接，或者將Queue改成redis
"""
import asyncio
from multiprocessing import Queue

import mitmproxy.websocket

class WebsocketAddon:
    def __init__(self, input_q: Queue = Queue(), output_q: Queue = Queue()):
        self._input_q = input_q
        self._output_q = output_q

    async def inject(self, flow: mitmproxy.websocket.WebSocketFlow):
        while not flow.ended and not flow.error:
            # 增加間隔，否則會阻塞event
            await asyncio.sleep(0.5)
            while not self._input_q.empty():
                # WebSocketFlow的內(nèi)置方法，用于主動插入信息，這里我只主動插入client->server的信息
                flow.inject_message(flow.server_conn, self._input_q.get())

    def websocket_start(self, flow: mitmproxy.websocket.WebSocketFlow):
        # 加入發(fā)送websocket消息的task，參考了官方的示例腳本，地址：https://docs.mitmproxy.org/stable/addons-examples/#websocket-inject-message
        asyncio.get_event_loop().create_task(self.inject(flow))

    def websocket_message(self, flow: mitmproxy.websocket.WebSocketFlow):
        message = flow.messages[-1]
        self._output_q.put({
          'from_client': message.from_client, 
          'data': message.content
        })
        # message.kill()可以讓Layer不轉(zhuǎn)發(fā)該條信息，我這里的目的是攔截掉所有客戶端發(fā)送的數(shù)據(jù)，由自己編輯后再發(fā)送
        if message.from_client:
            message.kill()

2、編寫 socket 的 addon

"""
簡略版用于socket的Addon
和websocket版差別不大，插入數(shù)據(jù)和攔截?cái)?shù)據(jù)有區(qū)別
"""
import asyncio
from multiprocessing import Queue

import mitmproxy.tcp

class SocketAddon:
    def __init__(self, input_q: Queue = Queue(), output_q: Queue = Queue()):
        self._input_q = input_q
        self._output_q = output_q

    async def inject(self, flow: mitmproxy.websocket.WebSocketFlow):
        while flow.live and not flow.error:
            await asyncio.sleep(0.5)
            while not self._input_q.empty():
                    # 直接向?qū)Χ税l(fā)送socket信息完成插入
                    flow.server_conn.connection.sendall(payload)

    def websocket_start(self, flow: mitmproxy.websocket.WebSocketFlow):
        asyncio.get_event_loop().create_task(self.inject(flow))

    def websocket_message(self, flow: mitmproxy.websocket.WebSocketFlow):
        message = flow.messages[-1]
        self._output_q.put({
          'from_client': message.from_client, 
          'data': message.content
        })
        if message.from_client:
           # socket發(fā)送0字節(jié)，conn.sendall(b'')將不會發(fā)送任何數(shù)據(jù)
           message.content = b''

3、開啟 mitmproxy 并完成處理函數(shù)

import multiprocessing

from mitmdump import Options, DumpMaster

def start_proxy(input_q: multiprocessing.Queue(), output_q: multiprocessing.Queue()):
        addons = [
        # 自主選擇是使用Websocket還是Socket
            WebsocketAddon(input_q, output_q)
        # SocketAddon(input_q, output_q)
        ]
        opts = Options(listen_host='0.0.0.0', listen_port=1080, scripts=None, mode='socks5',
                     rawtcp=True,
                   # 需要轉(zhuǎn)換tcp數(shù)據(jù)成的ip正則
                   tcp_hosts=['.*'],
                   flow_detail=0, termlog_verbosity='error', show_clientconnect_log=True, )
        m = DumpMaster(opts)
        m.addons.add(*addons)
        m.run()

def deal_client_message_func(client_message: [bytes, str]):
        if type(client_message) is bytes:
        return client_message.decode('utf-8').encode('gbk')
    elif type(client_message) is str:
        return f"test {client_message}"

def simple_handel_message_func(input_q: multiprocessing.Queue(), output_q: multiprocessing.Queue()):
    while True:
            if not output_q.empty()
            message = output_q.get()
                print(f"{'客戶端' if message['from_client'] else '服務(wù)端'} 包內(nèi)容：{message['data']}")
            if message['from_client']:
                input_q.push(deal_client_message_func(message['data']))

def main():
    input_queue = multiprocessing.Queue()
    output_queue = multiprocessing.Queue()
    # 使用子進(jìn)程啟動proxy
    multiprocessing.Process(target=start_proxy, args=(input_queue, output_queue)).start()
    simple_handel_message_func(input_queue, output_queue)

四、總結(jié)

對于想實(shí)現(xiàn)開頭文中所提到的功能還需要實(shí)現(xiàn)客戶端，以及對于 protobuf 協(xié)議的編解碼，這里限于篇幅不再討論，后續(xù)有機(jī)會再更新。

另外，之所以 mitmproxy 選擇 socks5 模式，是因?yàn)?socks 協(xié)議支持代理除了 http、https 以外更多種類的協(xié)議，windows 開啟 socks5 代理的工具：proxifer，android 開啟 socks5 代理工具：postern。