Linux中Python程序CPU占用高排查,Linux中Python程序CPU占用高排查,Linux中Python程序CPU占用高排查
kafka-python==2.0.2和 gevent 新版本 生产机器中发现CPU占用极高,应该是有bug:https://github.com/dpkp/kafka-python/issues/2168 。目前推测是gevent patch后Kafka Consumer 和 heartbeat 占用了大量的 CPU 。
机器环境
- Debian 10
- Python 3.7.3
- kafka-python 2.0.2
- gevent 21.12.0 (greenlet 1.1.2)
测试代码
<span style="color:#333333"><span style="background-color:#f6f6f6"><span style="color:#444444"><span style="background-color:#f6f6f6"><span style="color:#888888"># vim cppla.py</span>
<span style="color:#333333"><strong>from</strong></span> gevent <span style="color:#333333"><strong>import</strong></span> monkey
<span style="color:#333333"><strong>from</strong></span> gevent.pywsgi <span style="color:#333333"><strong>import</strong></span> WSGIServer
monkey.patch_all()
<span style="color:#333333"><strong>from</strong></span> multiprocessing <span style="color:#333333"><strong>import</strong></span> cpu_count, Process
<span style="color:#333333"><strong>from</strong></span> flask <span style="color:#333333"><strong>import</strong></span> Flask, jsonify
<span style="color:#333333"><strong>from</strong></span> kafka <span style="color:#333333"><strong>import</strong></span> KafkaConsumer, KafkaProducer
<span style="color:#333333"><strong>from</strong></span> kafka.errors <span style="color:#333333"><strong>import</strong></span> NoBrokersAvailable, KafkaTimeoutError
app = Flask(__name__)
<span style="color:#1f7199">@app.route("/cppla", methods=['GET'])</span>
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>function_benchmark</strong></span>():
<span style="color:#333333"><strong>return</strong></span> jsonify(
{
<span style="color:#880000">"status"</span>: <span style="color:#880000">"ok"</span>,
}
), <span style="color:#880000">200</span>
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>run</strong></span>():
mulserver = WSGIServer((<span style="color:#880000">'0.0.0.0'</span>, <span style="color:#880000">8080</span>), app)
mulserver.start()
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>server_forever</strong></span>():
mulserver.start_accepting()
mulserver._stop_event.wait()
<span style="color:#333333"><strong>for</strong></span> i <span style="color:#333333"><strong>in</strong></span> range(cpu_count()):
p = Process(target=server_forever)
p.start()
KAFKA_URI = {
<span style="color:#880000">"BOOTSTRAP_SERVERS"</span>: [
<span style="color:#880000">'192.168.1.2:9092'</span>,
<span style="color:#880000">'192.168.1.3:9092'</span>,
<span style="color:#880000">'192.168.1.4:9092'</span>
],
<span style="color:#880000">"TOPIC"</span>: <span style="color:#880000">"test"</span>,
<span style="color:#880000">"GROUP_ID"</span>: <span style="color:#880000">"v1"</span>,
<span style="color:#880000">"KEY"</span>: <span style="color:#880000">"order"</span>
}
<span style="color:#333333"><strong>class</strong></span> <span style="color:#880000"><strong>kafkaClient</strong></span>(object):
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>__init__</strong></span>(self):
print(<span style="color:#880000">"init start "</span>)
self._producer_client = self._createProducer
self._consumer_client = self._createConsumer
print(<span style="color:#880000">"init end "</span>)
<span style="color:#1f7199"> @property</span>
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>_createProducer</strong></span>(self):
<span style="color:#333333"><strong>try</strong></span>:
<span style="color:#333333"><strong>return</strong></span> KafkaProducer(
bootstrap_servers=KAFKA_URI[<span style="color:#880000">"BOOTSTRAP_SERVERS"</span>],
retries=<span style="color:#880000">3</span>
)
<span style="color:#333333"><strong>except</strong></span> NoBrokersAvailable:
print(<span style="color:#880000">"bo brokers"</span>)
<span style="color:#1f7199"> @property</span>
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>_createConsumer</strong></span>(self):
<span style="color:#333333"><strong>try</strong></span>:
<span style="color:#333333"><strong>return</strong></span> KafkaConsumer(
KAFKA_URI[<span style="color:#880000">"TOPIC"</span>],
group_id=KAFKA_URI[<span style="color:#880000">"GROUP_ID"</span>],
bootstrap_servers=KAFKA_URI[<span style="color:#880000">"BOOTSTRAP_SERVERS"</span>],
auto_offset_reset=<span style="color:#880000">"latest"</span>,
enable_auto_commit=<span style="color:#333333"><strong>True</strong></span>,
auto_commit_interval_ms=<span style="color:#880000">5000</span>,
)
<span style="color:#333333"><strong>except</strong></span> NoBrokersAvailable:
print(<span style="color:#880000">"no brokers"</span>)
<span style="color:#1f7199"> @property</span>
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>consumer</strong></span>(self):
print(<span style="color:#880000">"consumer function"</span>)
<span style="color:#333333"><strong>try</strong></span>:
<span style="color:#333333"><strong>for</strong></span> x <span style="color:#333333"><strong>in</strong></span> self._consumer_client:
<span style="color:#333333"><strong>yield</strong></span> {
<span style="color:#880000">"partition"</span>: x.partition,
<span style="color:#880000">"timestamp"</span>: x.timestamp,
<span style="color:#880000">"offset"</span>: x.offset,
<span style="color:#880000">"value"</span>: x.value.decode()
}
<span style="color:#333333"><strong>except</strong></span> Exception <span style="color:#333333"><strong>as</strong></span> e:
print(e)
<span style="color:#333333"><strong>def</strong></span> <span style="color:#880000"><strong>producer</strong></span>(self, msg):
print(<span style="color:#880000">"consumer function"</span>)
<span style="color:#333333"><strong>if</strong></span> <span style="color:#333333"><strong>not</strong></span> self._producer_client:
print(<span style="color:#880000">"mark0"</span>)
<span style="color:#333333"><strong>return</strong></span> <span style="color:#333333"><strong>False</strong></span>
<span style="color:#333333"><strong>else</strong>&l