python locust并发量大的时候时间戳过期_使用locust对设备ID的生成逻辑的并发测试初步实践...

项目背景：现阶段我们项目主要有两大场景，一是交易风控，二是账户风控，两大的场景的很多规则都和设备ID有关，比如设备黑名单，设备A在黑名单库并且相关规则开启，设备A请求交易时就会有预警事件发生，所以设备ID的生成逻辑至关重要，主要和A、B、C 三大因素有关，大概如下：

1、同A，不管后面的B,C是否不一致

首先根据传入消息的A ，到ES中查询 如果存在相同的则用原A的设备ID(DeviceID)

2、不同A

1)B一样，C不一样，则用原B的设备ID(DeviceID)

2)C一样,B不一样，则用原C的设备ID(DeviceID)

3)B不一样，C不一样，则生成新设备ID(DeviceID)，生成新的设备ID的逻是原来的最大的设备ID号，加1最为新的设备ID，例如：原数据中最大设备ID是D000000009，则新的设备ID是D000000010.

设备ID的生成在数据量大并发的情况下是非常有可能是同一时间过来的，所以，当开启脚本后，即使是A、B、C因子都不一样的时候，因为生成时间是相同的(精确到毫秒级)，所以生成的设备ID还是相同的(实际情况下是不可能生成相同设备的)，此问题的解决方案有很多，可以搜索，分布式ID的生成，我们开发采取的方案是使用redis的原子性，ID生成器解决的。主要的locust脚本如下：

#!/user/bin/env python

#coding=utf-8

author = 'zengweifang'

#风控系统的压测脚本

#第一步：模拟设备指纹的100万数据的生成

from locust import HttpLocust,TaskSet,task

import queue

import json

import requests

from random import Random

import random

import time

import string

import socket

import struct

def AutoGeneratedString(number):

'''随机生成字符串方法，主要用于输入框不能超过多少字符串的场景,此一次性产生的最大的字符串是62个'''

return ''.join(random.sample(string.ascii_letters + string.digits, number))

def getRandomBool():

'''随机获取是否是代理'''

isProxy = ["true","false"]

return random.sample(isProxy, 1)[0]

def get_random_ip():

'''随机生成合法的IP'''

RANDOM_IP_POOL=['192.168.10.222/0']

str_ip = RANDOM_IP_POOL[random.randint(0,len(RANDOM_IP_POOL) - 1)]

str_ip_addr = str_ip.split('/')[0]

str_ip_mask = str_ip.split('/')[1]

ip_addr = struct.unpack('>I',socket.inet_aton(str_ip_addr))[0]

mask = 0x0

for i in range(31, 31 - int(str_ip_mask), -1):

mask = mask | ( 1 << i)

ip_addr_min = ip_addr & (mask & 0xffffffff)

ip_addr_max = ip_addr | (~mask & 0xffffffff)

return socket.inet_ntoa(struct.pack('>I', random.randint(ip_addr_min, ip_addr_max)))

def getUnixTime():

'''获得13位的unix时间戳:1519800472673'''

return int(round(time.time() * 1000))

def getPCUserAgent():

'''获取PC端的UserAgent'''

pc_user_agent = [

'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_8; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

'Mozilla/5.0 (Windows; U; Windows NT 6.1; en-us) AppleWebKit/534.50 (KHTML, like Gecko) Version/5.1 Safari/534.50',

'Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Trident/5.0',

'Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.0',

'Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1',

'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)',

'Mozilla/5.0 (Windows NT 6.3; Trident/7.0; rv:11.0) like Gecko',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.6; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',

'Mozilla/5.0 (Windows NT 6.1; rv,2.0.1) Gecko/20100101 Firefox/4.0.1',

'Opera/9.80 (Macintosh; Intel Mac OS X 10.6.8; U; en) Presto/2.8.131 Version/11.11',

'Opera/9.80 (Windows NT 6.1; U; en) Presto/2.8.131 Version/11.11',

'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_0) AppleWebKit/535.11 (KHTML, like Gecko) Chrome/17.0.963.56 Safari/535.11',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Maxthon 2.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; TencentTraveler 4.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; The World)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Trident/4.0; SE 2.X MetaSr 1.0; SE 2.X MetaSr 1.0; .NET CLR 2.0.50727; SE 2.X MetaSr 1.0)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; 360SE)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; Avant Browser)',

'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)']

list(set(pc_user_agent))

return random.sample(pc_user_agent, 1)[0]

def messageDict(fingerId, sessionId):

topic = {"topic": "meta-fingerprint"}

tag = {"tag": ""}

key = {"key": ""}

appId = {"appId": 'A10007'}

clientType = {"clientType": "1"}

fingerId = {"fingerId": fingerId}

ip = {"ip": get_random_ip()}

proxyFlag = {"proxyFlag": getRandomBool()}

sessionId = {"sessionId": sessionId}

systemId = {"systemId": '7b6a99f3bce14915863cde5104bdf2c3'}

timestamp = {"timestamp": str(getUnixTime())}

user_agent = '{\"key\":\"user_agent\",\"value\":\"%s\"}' % getPCUserAgent()

language = '{\"key\":\"language\",\"value\":\"zh-cn\"}'

color_depth = '{\"key\":\"color_depth\",\"value\":32}'

device_memory = '{\"key\":\"device_memory\",\"value\":-1}'

pixel_ratio = '{\"key\":\"pixel_ratio\",\"value\":\"\"}'

hardware_concurrency = '{\"key\":\"hardware_concurrency\",\"value\":\"unknown\"}'

resolution = '{\"key\":\"resolution\",\"value\":[1920,1080]}'

available_resolution = '{\"key\":\"available_resolution\",\"value\":[1920,1040]}'

timezone_offset = '{\"key\":\"timezone_offset\",\"value\":-480}'

session_storage = '{\"key\":\"session_storage\",\"value\":1}'

local_storage = '{\"key\":\"local_storage\",\"value\":1}'

add_behavior = '{\"key\":\"add_behavior\",\"value\":1}'

cpu_class = '{\"key\":\"cpu_class\",\"value\":\"x86\"}'

navigator_platform = '{\"key\":\"navigator_platform\",\"value\":\"Win32\"}'

do_not_track = '{\"key\":\"do_not_track\",\"value\":\"unknown\"}'

ie_plugins = '{\"key\":\"ie_plugins\",\"value\":[\"AcroPDF.PDF\",null,null,null,\"MacromediaFlashPaper.MacromediaFlashPaper\",\"Msxml2.DOMDocument\",\"Msxml2.XMLHTTP\",null,null,null,null,null,null,\"Scripting.Dictionary\",null,\"Shell.UIHelper\",\"ShockwaveFlash.ShockwaveFlash\",null,\"TDCCtl.TDCCtl\",\"WMPlayer.OCX\",null,null]}'

adblock = '{\"key\":\"adblock\",\"value\":false}'

has_lied_languages = '{\"key\":\"has_lied_languages\",\"value\":false}'

has_lied_resolution = '{\"key\":\"has_lied_resolution\",\"value\":false}'

has_lied_os = '{\"key\":\"has_lied_os\",\"value\":false}'

has_lied_browser = '{\"key\":\"has_lied_browser\",\"value\":false}'

touch_support = '{\"key\":\"touch_support\",\"value\":[0,false,false]}'

js_fonts = '{\"key\":\"js_fonts\",\"value\":[\"Arial\",\"Arial Black\",\"Calibri\",\"Cambria\",\"Cambria Math\",\"Comic Sans MS\",\"Consolas\",\"Courier\",\"Courier New\",\"Georgia\",\"Helvetica\",\"Impact\",\"Lucida Console\",\"Lucida Sans Unicode\",\"Microsoft Sans Serif\",\"MS Gothic\",\"MS PGothic\",\"MS Sans Serif\",\"MS Serif\",\"Palatino Linotype\",\"Segoe Print\",\"Segoe Script\",\"Segoe UI\",\"Segoe UI Light\",\"Segoe UI Semibold\",\"Segoe UI Symbol\",\"Tahoma\",\"Times\",\"Times New Roman\",\"Trebuchet MS\",\"Verdana\",\"Wingdings\"]}'

fingerParametersList = [user_agent, language, color_depth, device_memory, pixel_ratio, hardware_concurrency,

resolution, available_resolution, timezone_offset,

session_storage, local_storage, add_behavior, cpu_class, navigator_platform,

do_not_track, ie_plugins, adblock, has_lied_languages,

has_lied_resolution, has_lied_os, has_lied_browser, touch_support, js_fonts]

fingerParameters = {"fingerParameters": str(fingerParametersList)}

{"messageBody":fingerParameters}

messageBodyDict = fingerParameters.copy()

messageBodyDict.update(clientType)

messageBodyDict.update(fingerId)

messageBodyDict.update(ip)

messageBodyDict.update(proxyFlag)

messageBodyDict.update(sessionId)

messageBodyDict.update(systemId)

messageBodyDict.update(timestamp)

messageBodyDict.update(appId)

messageBody = {"messageBody": str(messageBodyDict)}

# print(messageBody)

NewmessageBody = messageBody.copy()

NewmessageBody.update(topic)

NewmessageBody.update(tag)

NewmessageBody.update(key)

return NewmessageBody

def toJson(message):

return json.dumps(message, ensure_ascii=False)

def send(url, jsonbody, header):

print('发送的消息体为:%s' % jsonbody)

re = requests.post(url=url, data=jsonbody, headers=header).json()

return re

def getHeader():

'''获取请求头'''

header = {"Content-Type": "application/json"}

return header

class rcpTaskSet(TaskSet):

@task(1)

def fingerprint(self):

'''生产设备指纹'''

try:

sessionId_data = self.locust.sessionId_queue.get()

print(sessionId_data)

except queue.Empty:

print('sessionId data run out, test ended.')

exit(0)

try:

fingerId_data = self.locust.fingerId_queue.get()

print(fingerId_data)

except queue.Empty:

print('fingerId data run out, test ended.')

exit(0)

#self.messageDict(fingerId_data,sessionId_data)

json1 = toJson(messageDict(fingerId_data,sessionId_data))

send(RockMQ, json1, getHeader())

#保证并发测试数据唯一性，循环取数据

self.locust.sessionId_queue.put_nowait(sessionId_data)

self.locust.fingerId_queue.put_nowait(fingerId_data)

# payload = FG.messageDict( sessionId)

# json1 = FG.toJson(payload)

# self.send(self.RockMQ, json1, self.getHeader())

# @task

# def sendMQ(self):

# '''发送消息'''

# pass

class rcpLocust(HttpLocust):

task_set = rcpTaskSet

sessionId_queue = queue.Queue()

for a in range(1000000):

sessionId = AutoGeneratedString(32)

sessionId_queue.put_nowait(sessionId)

fingerId_queue = queue.Queue()

for b in range(1000000):

fingerId = AutoGeneratedString(32)

fingerId_queue.put_nowait(fingerId)

min_wait = 1000

max_wait = 3000

最后在web上设置用户数和每秒加载的用户，done