遍历百万级Redis的键值的大结局

最新推荐文章于 2020-12-30 04:53:31 发布

小屋子大侠

最新推荐文章于 2020-12-30 04:53:31 发布

阅读量456

点赞数

分类专栏： python 文章标签： python redis

本文链接：https://blog.csdn.net/qq_33339479/article/details/106928610

版权

python 专栏收录该内容

67 篇文章 7 订阅

订阅专栏

背景

上次改完利用条件变量的形式来进行rdbtool和socket接受的数据联合分析，我再想能不能通过协程来实现避免条件变量这种调用系统调用的方式，当然如果算一下因为每一次接受的socket的数据都尽量的大的话其他调用条件变量的次数或许在整个性能消耗里面占比比较小的，这个方式只是想自己探索一下。

协程的改造之路

greenlet的基本使用

from greenlet import greenlet

def test1():
    print(12)
    gr2.switch()
    print(34)

def test2():
    print(56)
    gr1.switch()
    print(78)

gr1 = greenlet(test1)
gr2 = greenlet(test2)
gr1.switch()

这是greenlet官网提供的示例，输入的结果大家自行运行一下，从该实例代码可以看出greenlet保存的是执行函数的上下文信息，在调度的过程中会还原已经保存的信息，greenlet底层其实就是调用的是汇编代码来保存上下文信息，大家有兴趣可自行查看。

模拟读写过程

首先编写一个server脚本，代码如下；

import socket


def run_server():
    sock = socket.socket()
    sock.bind(("127.0.0.1", 6000))
    sock.listen(5)

    while True:
        try:
            conn, addr = sock.accept()
        except Exception as e:
            print(e)
            return

        Flag = True
        while Flag:
            try:
                data = conn.recv(1024)
            except Exception as e:
                print(e)
                Flag = False
                continue

            print("recv  ", data)
            try:
                conn.send(b"1234567890qwertyuiopasdfghjklzxcvbnm")
            except Exception as e:
                print(e)
                Flag = False
                continue


if __name__ == '__main__':
    run_server()

server代码相对简单并未做很鉴权的处理，仅是模拟一下redis的server。

利用greenlet来进行数据的读写；

import socket

from greenlet import greenlet


def read_data(data=None):
    # socket读到了数据 然后来解析该数据  如果数据不够则需要继续读取数据
    total_len = 0
    if data:
        print("read_data  ", data)
        total_len += len(data)

    while True:
        recv_data = gr_read.switch()
        print("current recv ", recv_data)
        if recv_data:
            total_len += len(data)

        if total_len >= 20:
            # 关闭连接 进行收尾工作 如果接受的数据大于20则完成任务
            return


def read_socket(host="127.0.0.1", port=6000):
    # 从socket中读取数据，然后切换到read_data中奖读取到的数据 交给read_data来解析
    conn = socket.socket()
    conn.connect((host, port))

    total = 0
    conn.send(b"1")
    while True:
        r = conn.recv(4)
        print(r)
        gr_consumer.switch(r)
  
gr_read = greenlet(read_socket)
gr_consumer = greenlet(read_data)
gr_read.switch()

运行结果如下；

b'1234'
read_data   b'1234'
b'5678'
current recv  b'5678'
b'90qw'
current recv  b'90qw'
b'erty'
current recv  b'erty'
b'uiop'
current recv  b'uiop'

因为server默认传回的内容是长度大于20的数据的，所有client会主动停止。如上的思路大致跑通之后，由于greenlet只能够对当前执行的函数栈进行恢复与调度，如果使用yield来进行操作的话，只能够先通过greenlet调度，再在greenlet的流程中包含yield的流程，修改代码如下；

import socket

from greenlet import greenlet


class Buff(object):
    def __init__(self):
        self.read_length = 0
        self.buff = b""
        self.flag = True
        self.parse_func = self.parse()
        next(self.parse_func)

    def add(self, data):
        self.buff += data

    def start(self):
        self.parse_func.send(None)

    def parse(self):
        print("start parse")
        while self.flag:
            read_three = self.read_n(5)
            print("parse read    ", read_three)
            if isinstance(read_three, bytes):
                continue
            yield read_three

    def wait_read(self):
        n = self.read_length
        while True:
            yield
            if len(self.buff) >= n:
                r = self.buff[:n]
                self.buff = self.buff[n:]
                return r

    def read_n(self, n):
        print("reand n  ", len(self.buff), n)
        if len(self.buff) >= n:
            r = self.buff[:n]
            self.buff = self.buff[n:]
            return r
        else:
            self.read_length = n
            return self.wait_read()


buff = Buff()


def read_data(data=None):
    # socket读到了数据 然后来解析该数据  如果数据不够则需要继续读取数据
    total_len = 0
    if data:
        print("read_data  ", data)
        total_len += len(data)
        buff.add(data)

    while True:
        recv_data = gr_read.switch()
        print("current recv ", recv_data)
        if recv_data:
            total_len += len(data)
            buff.add(recv_data)
            buff.start()

        if total_len >= 20:
            # 关闭连接 进行收尾工作
            return


def read_socket(host="127.0.0.1", port=6000):
    # 从socket中读取数据，然后切换到read_data中奖读取到的数据 交给read_data来解析
    conn = socket.socket()
    conn.connect((host, port))

    total = 0
    conn.send(b"1")
    while True:
        r = conn.recv(4)
        print(r)
        gr_consumer.switch(r)


if __name__ == '__main__':
    gr_read = greenlet(read_socket)
    gr_consumer = greenlet(read_data)
    gr_read.switch()

通过调用greenlet中的buff实现的parse的协程，从而完成当解析的数据不够的时候，则切换到接受数据的协程，然后再接收到数据之后再切换到解析的函数过程中执行（解析仅仅就是读出数据而已，具体业务可能是具体的场景），从而完成了两个协程交替执行读数据解析的任务。

rdb分析脚本改造

import socket
import logging
import time

from greenlet import greenlet
from rdbtools import RdbParser, KeyValsOnlyCallback
from rdbtools.encodehelpers import ESCAPE_CHOICES

logger = logging.getLogger(__package__)


start = time.time()


redis_ip = "192.168.10.202"
redis_port = 6371
key_size = 412


def encode_command(*args, buf=None):
    if buf is None:
        buf = bytearray()
    buf.extend(b'*%d\r\n' % len(args))

    try:
        for arg in args:
            if isinstance(arg, str):
                arg = arg.encode("utf-8")
            buf.extend(b'$%d\r\n%s\r\n' % (len(arg), arg))
    except KeyError:
        raise TypeError("Argument {!r} expected to be of bytearray, bytes,"
                        " float, int, or str type".format(arg))
    return buf


class RecvBuff(object):

    def __init__(self):
        self.buff = b""
        self.length = 0
        self.total_length = 0
        self.is_done = False
        self.read_length = 0
        self.gr_read = None
        self.gr_consumer = None

    def add(self, data):
        self.buff += data

    def wait_from_socket(self):
        n = self.read_length
        while True:
            time.sleep(1)
            self.gr_read.switch()
            if len(self.buff) >= n:
                r = self.buff[:n]
                self.buff = self.buff[n:]
                self.length += n
                if self.length == self.total_length:
                    self.is_done = True
                    return
                return r

    def consumer_length(self, n):
        if len(self.buff) >= n:
            r = self.buff[:n]
            self.buff = self.buff[n:]
            self.length += n
            if self.length == self.total_length:
                self.is_done = True
                raise
            return r
        else:
            self.read_length = n
            r = self.wait_from_socket()
            print("consumer length return  ", r)
            return r


recv_buff = RecvBuff()


def rdb_work():
    class Writer(object):

        def write(self, value):
            if b" " in value:
                index = value.index(b" ")
                length = len(value)
                if length - index - 1 >= key_size:
                    print(value, index, length)

    out_file_obj = Writer()
    callback = {
        'justkeyvals': lambda f: KeyValsOnlyCallback(f, string_escape=ESCAPE_CHOICES[0]),
    }["justkeyvals"](out_file_obj)
    parser = RdbParser(callback)

    def parse(self, filename=None):
        class Reader(object):
            def __init__(self, buff):
                self.buff = buff

            def __enter__(self):
                return self

            def __exit__(self, exc_type, exc_val, exc_tb):
                pass

            def read(self, n):
                if n <= 0:
                    return
                # res = self.buff.consumer_length(n)
                while True:
                    if len(self.buff.buff) < n:
                        self.buff.gr_read.switch()
                    else:
                        break
                res = self.buff.consumer_length(n)
                return res

            def close(self):
                pass
        f = Reader(recv_buff)
        self.parse_fd(f)
    setattr(parser, "parse", parse)

    print("start rdb work")
    parser.parse(parser)
    recv_buff.is_done = True
    print("finish rdb work")


class RedisServer(object):

    def __init__(self, host=None, port=None):
        self.host = host or "127.0.0.1"
        self.port = port or 6379
        self.conn = None
        self.recv_buff = recv_buff

    def init(self):
        try:
            self.conn = socket.socket()
            self.conn.connect((self.host, self.port))
        except Exception as e:
            logger.exception(e)
            self.conn = None
            return

    def slave_sync(self):
        self.send_sync()
        total_read_length = 0
        # 首先先出去sync返回的数据 b'$9337614\r\n'
        while True:
            data = self.conn.recv(1024 * 1)
            if b"$" == data[:1]:
                length = len(data)
                for i in range(length-1):
                    if b"\r\n" == data[i:(i + 2)]:
                        break

                self.recv_buff.total_length = int(data[1:(i-2)].decode())
                left_data = data[(i+2):]
                total_read_length += len(left_data)
                print("recv  length  ", len(left_data))
                if left_data:
                    self.recv_buff.add(left_data)
                # 切换到启动消费的协程
                rdb_green.switch()
                print("stop  first  rdb work")
                break
            if b"\n" == data:
                continue

        while True:
            try:
                data = self.conn.recv(1024 * 8)
            except Exception as e:
                print("recv error : {0}".format(e))
                return
            if data:
                self.recv_buff.add(data)
                # 切换到消费的协程
                rdb_green.switch()
                if self.recv_buff.is_done:
                    print("recv buff done")
                    return

    def send_sync(self):
        data = encode_command("SYNC")
        try:
            self.conn.send(data)
        except Exception as e:
            return


def main():
    rs = RedisServer(redis_ip, redis_port)
    recv_buff.gr_read = greenlet(rs.slave_sync)
    global rdb_green
    rdb_green = greenlet(rdb_work)
    rs.init()
    recv_buff.gr_read.switch()
    end = time.time()
    print("finish use time {0}  second  ".format(end - start))


if __name__ == '__main__':
    import cProfile
    cProfile.run("main()")

该脚本的改造过程中，一定要将rdb_work和rs.slave_sync的协程的切换过程一定要放在函数中，因为该函数记录了当前rdbtool解析的时候的上下文信息，如果在recv_buff中新开一个协程在该类中切换就失去了rdb_work中的上下文调用栈的信息，从而导致失败。

首先查看一下运行的性能数据；

         16281244 function calls (16278933 primitive calls) in 7.568 seconds

   Ordered by: standard name

   ncalls  tottime  percall  cumtime  percall filename:lineno(function)
      2/1    0.000    0.000    7.563    7.563 <string>:1(<module>)
        1    0.000    0.000    0.000    0.000 callbacks.py:179(__init__)
   100012    0.103    0.000    0.218    0.000 callbacks.py:188(_start_key)
       12    0.000    0.000    0.000    0.000 callbacks.py:196(_end_key)
      468    0.000    0.000    0.001    0.000 callbacks.py:199(_write_comma)
   100000    0.182    0.000    2.944    0.000 callbacks.py:204(set)
       12    0.000    0.000    0.000    0.000 callbacks.py:208(start_hash)
      468    0.001    0.000    0.024    0.000 callbacks.py:212(hset)
       12    0.000    0.000    0.000    0.000 callbacks.py:216(end_hash)
   200948    0.100    0.000    0.165    0.000 compat.py:16(isnumber)
   200948    0.225    0.000    2.292    0.000 encodehelpers.py:126(apply_escape_bytes)
  4118558    1.141    0.000    1.466    0.000 encodehelpers.py:142(<genexpr>)
  4018078    0.325    0.000    0.325    0.000 encodehelpers.py:20(bval)
        2    0.000    0.000    0.000    0.000 enum.py:284(__call__)
        2    0.000    0.000    0.000    0.000 enum.py:526(__new__)
        1    0.000    0.000    0.000    0.000 enum.py:836(__and__)
      469    0.000    0.000    0.002    0.000 parser.py:1019(lzf_decompress)
        8    0.000    0.000    0.000    0.000 parser.py:103(aux_field)
        3    0.000    0.000    0.000    0.000 parser.py:1069(read_signed_char)
   502905    0.308    0.000    1.516    0.000 parser.py:1072(read_unsigned_char)
        3    0.000    0.000    0.000    0.000 parser.py:1081(read_signed_int)
   100013    0.063    0.000    0.297    0.000 parser.py:1087(read_unsigned_int_be)
        1    0.000    0.000    0.000    0.000 parser.py:112(start_database)
        1    0.000    0.000    0.000    0.000 parser.py:141(db_size)
        1    0.000    0.000    0.000    0.000 parser.py:342(end_database)
        1    0.000    0.000    0.000    0.000 parser.py:354(end_rdb)
        1    0.000    0.000    0.000    0.000 parser.py:377(__init__)
        1    0.334    0.334    7.032    7.032 parser.py:396(parse_fd)
   301929    0.324    0.000    1.555    0.000 parser.py:468(read_length_with_encoding)
   100965    0.046    0.000    0.789    0.000 parser.py:490(read_length)
   200964    0.151    0.000    1.759    0.000 parser.py:493(read_string)
   100012    0.148    0.000    3.845    0.000 parser.py:531(read_object)
        1    0.000    0.000    0.000    0.000 parser.py:78(__init__)
   100480    0.051    0.000    2.183    0.000 parser.py:84(encode_key)
   100468    0.045    0.000    0.205    0.000 parser.py:92(encode_value)
        1    0.000    0.000    0.000    0.000 parser.py:954(verify_magic_string)
        1    0.000    0.000    0.000    0.000 parser.py:958(verify_version)
        1    0.000    0.000    0.000    0.000 parser.py:96(start_rdb)
        1    0.000    0.000    0.000    0.000 parser.py:964(init_filter)
   200024    0.259    0.000    0.426    0.000 parser.py:996(matches_filter)
        1    0.000    0.000    0.000    0.000 re.py:232(compile)
        1    0.000    0.000    0.000    0.000 re.py:271(_compile)
        1    0.000    0.000    0.000    0.000 socket.py:139(__init__)
        1    0.000    0.000    0.000    0.000 sre_compile.py:423(_simple)
        1    0.000    0.000    0.000    0.000 sre_compile.py:536(_compile_info)
        2    0.000    0.000    0.000    0.000 sre_compile.py:595(isstring)
        1    0.000    0.000    0.000    0.000 sre_compile.py:598(_code)
      2/1    0.000    0.000    0.000    0.000 sre_compile.py:71(_compile)
        1    0.000    0.000    0.000    0.000 sre_compile.py:759(compile)
        2    0.000    0.000    0.000    0.000 sre_parse.py:111(__init__)
        4    0.000    0.000    0.000    0.000 sre_parse.py:160(__len__)
        8    0.000    0.000    0.000    0.000 sre_parse.py:164(__getitem__)
        1    0.000    0.000    0.000    0.000 sre_parse.py:168(__setitem__)
        1    0.000    0.000    0.000    0.000 sre_parse.py:172(append)
      2/1    0.000    0.000    0.000    0.000 sre_parse.py:174(getwidth)
        1    0.000    0.000    0.000    0.000 sre_parse.py:224(__init__)
        3    0.000    0.000    0.000    0.000 sre_parse.py:233(__next)
        2    0.000    0.000    0.000    0.000 sre_parse.py:249(match)
        2    0.000    0.000    0.000    0.000 sre_parse.py:254(get)
        2    0.000    0.000    0.000    0.000 sre_parse.py:286(tell)
        1    0.000    0.000    0.000    0.000 sre_parse.py:417(_parse_sub)
        1    0.000    0.000    0.000    0.000 sre_parse.py:475(_parse)
        1    0.000    0.000    0.000    0.000 sre_parse.py:76(__init__)
        2    0.000    0.000    0.000    0.000 sre_parse.py:81(groups)
        1    0.000    0.000    0.000    0.000 sre_parse.py:903(fix_flags)
        1    0.000    0.000    0.000    0.000 sre_parse.py:919(parse)
        1    0.000    0.000    7.032    7.032 t.py:103(parse)
        1    0.000    0.000    0.000    0.000 t.py:104(Reader)
        1    0.000    0.000    0.000    0.000 t.py:105(__init__)
        1    0.000    0.000    0.000    0.000 t.py:108(__enter__)
        1    0.000    0.000    0.000    0.000 t.py:111(__exit__)
   803885    0.564    0.000    2.067    0.000 t.py:114(read)
        1    0.000    0.000    0.000    0.000 t.py:140(__init__)
        1    0.000    0.000    0.005    0.005 t.py:146(init)
        1    0.000    0.000    0.001    0.001 t.py:194(send_sync)
        1    0.000    0.000    7.563    7.563 t.py:202(main)
        1    0.000    0.000    0.000    0.000 t.py:24(encode_command)
     1152    0.002    0.000    0.002    0.000 t.py:51(add)
   803885    0.971    0.000    1.041    0.000 t.py:68(consumer_length)
        1    0.000    0.000    7.032    7.032 t.py:87(rdb_work)
        1    0.000    0.000    0.000    0.000 t.py:88(Writer)
   300959    0.236    0.000    0.294    0.000 t.py:90(write)
        1    0.000    0.000    0.000    0.000 t.py:99(<lambda>)
        1    0.000    0.000    0.000    0.000 {built-in method _sre.compile}
   602924    0.164    0.000    0.164    0.000 {built-in method _struct.unpack}
        2    0.000    0.000    0.000    0.000 {built-in method builtins.__build_class__}
   100480    0.413    0.000    1.879    0.000 {built-in method builtins.all}
      2/1    0.000    0.000    7.568    7.568 {built-in method builtins.exec}
   902896    0.137    0.000    0.137    0.000 {built-in method builtins.isinstance}
1709421/1709419    0.170    0.000    0.170    0.000 {built-in method builtins.len}
        4    0.000    0.000    0.000    0.000 {built-in method builtins.min}
      473    0.010    0.000    0.010    0.000 {built-in method builtins.print}
        1    0.000    0.000    0.000    0.000 {built-in method builtins.setattr}
      469    0.001    0.000    0.001    0.000 {built-in method lzf.decompress}
        1    0.000    0.000    0.000    0.000 {built-in method time.time}
   302879    0.033    0.000    0.033    0.000 {method 'append' of 'list' objects}
        1    0.005    0.005    0.005    0.005 {method 'connect' of '_socket.socket' objects}
        1    0.000    0.000    0.000    0.000 {method 'decode' of 'bytes' objects}
        1    0.000    0.000    0.000    0.000 {method 'disable' of '_lsprof.Profiler' objects}
   100013    0.033    0.000    0.033    0.000 {method 'encode' of 'str' objects}
        2    0.000    0.000    0.000    0.000 {method 'extend' of 'bytearray' objects}
        1    0.000    0.000    0.000    0.000 {method 'extend' of 'list' objects}
        1    0.000    0.000    0.000    0.000 {method 'format' of 'str' objects}
   100480    0.037    0.000    0.037    0.000 {method 'index' of 'bytes' objects}
        1    0.000    0.000    0.000    0.000 {method 'items' of 'dict' objects}
   100012    0.085    0.000    0.085    0.000 {method 'match' of 're.Pattern' objects}
     1154    0.898    0.001    0.898    0.001 {method 'recv' of '_socket.socket' objects}
        1    0.000    0.000    0.000    0.000 {method 'send' of '_socket.socket' objects}
   2305/0    0.003    0.000    0.000          {method 'switch' of 'greenlet.greenlet' objects}

从指标上来看，性能耗时较大的是rdbtool的encodehelpers中的转换函数这里耗时大约1.14秒，占总耗时7.56秒的15%，read的耗时大于是0.56秒，接受数据recv的耗时大约是0.898秒，从数据来看大部分的性能消耗都发生在rdbtool的代码解析过程中。所以本次性能消耗的主要的地方还是rdbtool工具的本身。

在运行的过程中，测试的机器还是那个虚拟机，尝试用strace来进行跟踪查看一下；

....
recvfrom(3, "0c6996(257_ee359ea8-e52b-490a-9f"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "bd(368_d1fbef4b-39b2-4c1f-8626-3"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "06_8457c6f1-c445-4683-ace0-3d4ea"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "05a3572-2662-431b-8e1a-cd931519c"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "02f-1182-4cef-bc34-721a476b12e9\370"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "3e2f-437a-b03f-afebc0a4ae9d\370\200\0\0358"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "-46d8-9f1c-1f75417cc880\370\200\0\0357@\0(3"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "1-bc1e-9aa6fd70e167\370\200\0\03579\0(383_b"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "97-e199ee70654c\370\200\0\0357\276\0(347_f44b1"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "0d901db4647\370\200\0\0358\330\0(404_2de2885b-"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "6db3a48\370\200\0\0357\35\0(452_431a7d99-9adc"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "73e\370\200\0\0357I\0(491_b3813e18-fc8b-46c"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "\200\0\0358<\0(329_ae4dd41b-0aef-4e1f-ab"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "D\0(393_25c61115-e9d4-4d77-a5d4-f"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "61_b8c0bcf0-a605-4837-9716-73ae8"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "6ab5b5d-fa14-45ed-b567-b6687fee9"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "0d0-1464-4e43-98bc-202bbc42e722("..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "cef1-4509-83e6-30a57dcf7aa7(486_"..., 8192, 0, NULL, NULL) = 8192
recvfrom(3, "-4288-ae49-a5895974c77e(387_2f36"..., 8192, 0, NULL, NULL) = 6940
write(1, "finish rdb work\n", 16finish rdb work
)       = 16
write(1, "finish use time 7.93073630332946"..., 44finish use time 7.930736303329468  second
) = 44
rt_sigaction(SIGINT, {SIG_DFL, [], SA_RESTORER, 0x7f8a94bc35d0}, {0x57a4c0, [], SA_RESTORER, 0x7f8a94bc35d0}, 8) = 0
sigaltstack(NULL, {ss_sp=0x10fb460, ss_flags=0, ss_size=8192}) = 0
sigaltstack({ss_sp=NULL, ss_flags=SS_DISABLE, ss_size=0}, NULL) = 0
exit_group(0)                           = ?
+++ exited with 0 +++

从strace的跟踪来看，确实脚本的运行过程中发生的主要的系统调用都是recvfrom和write这样的系统调用上，如果还需要在优化的话，一个比较好的方向就是去优化rdbtool的解析过程。

总结

本文还是将条件变量的方式，改为了协程实现的方式，总体上规避了部分条件变量获取时的系统调用的方法，但是对于整个脚本的性能提升相对有效，在测试520万key的遍历的时候，其实并没有太大的优化过程，只是自己在思考这个方向的时候做的一个探索吧。由于本人才疏学浅，如有错误请批评指正。

小屋子大侠

关注

0
点赞
踩
0

收藏

觉得还不错? 一键收藏
0
评论
遍历百万级Redis的键值的大结局

背景上次改完利用条件变量的形式来进行rdbtool和socket接受的数据联合分析，我再想能不能通过协程来实现避免条件变量这种调用系统调用的方式，当然如果算一下因为每一次接受的socket的数据都尽量的大的话其他调用条件变量的次数或许在整个性能消耗里面占比比较小的，这个方式只是想自己探索一下。协程的改造之路greenlet的基本使用from greenlet import greenletdef test1(): print(12) gr2.switch() print(
复制链接

扫一扫

专栏目录