Sentinel Slot扩展实践-流控熔断预警实现

前言

前几天公司生产环境一个服务由于流量上升触发了 Sentinel 的流控机制，然后用户反馈访问慢，定位发现是 task 定时任务导致，后面 task 优化之后发布，流量恢复正常。

这是一个再正常不过的生产问题，可能大部分同学都经历过，经历过的大多数是解决问题之后就不了了之，导致事故还有再次发生的可能，最终对用户造成了不好的体验。所以我觉得所有的生产问题都需要进行复盘，当然复盘的目的不是为了追责，而是防止下次再发生同样的错误。那我们就简单分析一下这个问题，首先肯定是业务层面的疏漏导致 task 发出不合理的大量请求，其二我们的流控只是简单粗暴的流控，没有更好的预警措施，导致影响到用户之后我们才知晓（即流控或熔断已经触发）。

那我们的解决方案呢？首先肯定是业务层面的预防，但这不是本文要说的重点，这里不展开讨论了。其次就是预警，就是我们能否在快要触发流控之前知晓，然后报警到相关负责人提前介入处理，防止触发流控熔断。当然也不能完全避免，但是总比流控或熔断触发之后在报警要好得多。

由于之前流控用的阿里的 Sentinel，所以本文介绍的具体实现是用 Sentinel 的自定义 slot 功能，这个自定义 slot 卡槽在 Sentinel 官方文档里面就一句话带过，然后加上一个 demo 代码，我在使用的过程中也遇到过不少坑，所以分享一下结果给大家。

如果大家对 Sentinel 不是很了解，可以先去 github 先了解简单试用一下在阅读本文。github 地址：https://github.com/alibaba/Sentinel

如果想熟悉自定义 slot 功能建议了解一下 Sentinel 的工作原理：https://github.com/alibaba/Sentinel/wiki/Sentinel%E5%B7%A5%E4%BD%9C%E4%B8%BB%E6%B5%81%E7%A8%8B

还有源码中的 demo 对于自定义 slot 的写法：https://github.com/alibaba/Sentinel/tree/master/sentinel-demo/sentinel-demo-slot-chain-spi

具体实现

下面介绍下 Sentinel 预警功能的相关实现，使用的前提是你的系统已经在用 Sentinel 的流控或熔断等功能。

1. 自定义 CustomSlotChainBuilder 实现 SlotChainBuilder 接口，这里主要是把我们自定义的 Slot 加到 SlotChain 这个链中

import com.alibaba.csp.sentinel.slotchain.ProcessorSlotChain;
import com.alibaba.csp.sentinel.slotchain.SlotChainBuilder;
import com.alibaba.csp.sentinel.slots.DefaultSlotChainBuilder;
import com.qiaofang.tortoise.gateway.component.ApplicationContextUtil;
import com.qiaofang.tortoise.gateway.config.SentinelProperties;
import org.springframework.stereotype.Component;

import javax.annotation.Resource;

/**
 * 自定义slot
 *
 * @author chenhao
 */
public class CustomSlotChainBuilder implements SlotChainBuilder {


    @Override
    public ProcessorSlotChain build() {
        ProcessorSlotChain chain = new DefaultSlotChainBuilder().build();
        SentinelProperties sentinelProperties = (SentinelProperties) ApplicationContextUtil.getContext().getBean("sentinelProperties");
        chain.addLast(new FlowEarlyWarningSlot(sentinelProperties));
        chain.addLast(new DegradeEarlyWarningSlot(sentinelProperties));
        return chain;
    }
}

2.自定义 FlowEarlyWarningSlot、DegradeEarlyWarningSlot 流控熔断 2 个预警 slot

自定义 FlowEarlyWarningSlot

import com.alibaba.csp.sentinel.context.Context;
import com.alibaba.csp.sentinel.node.DefaultNode;
import com.alibaba.csp.sentinel.slotchain.AbstractLinkedProcessorSlot;
import com.alibaba.csp.sentinel.slotchain.ResourceWrapper;
import com.alibaba.csp.sentinel.slots.block.flow.FlowRule;
import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleChecker;
import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleManager;
import com.alibaba.csp.sentinel.slots.block.flow.FlowRuleUtil;
import com.alibaba.csp.sentinel.util.AssertUtil;
import com.google.common.collect.Lists;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.BeanUtils;
import org.springframework.util.CollectionUtils;

import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

/**
 * 流控预警slot
 *
 * @author chenhao
 */
public class FlowEarlyWarningSlot2 extends AbstractLinkedProcessorSlot<DefaultNode> {

    /**
     * log
     */
    private Logger logger = LoggerFactory.getLogger(this.getClass());

    private final FlowRuleChecker checker;

    public FlowEarlyWarningSlot2() {
        this(new FlowRuleChecker());
    }

    /**
     * Package-private for test.
     *
     * @param checker flow rule checker
     * @since 1.6.1
     */
    FlowEarlyWarningSlot2(FlowRuleChecker checker) {
        AssertUtil.notNull(checker, "flow checker should not be null");
        this.checker = checker;
    }


    private List<FlowRule> getRuleProvider(String resource) {
        // Flow rule map should not be null.
        List<FlowRule> rules = FlowRuleManager.getRules();
        List<FlowRule> earlyWarningRuleList = Lists.newArrayList();
        for (FlowRule rule : rules) {
            FlowRule earlyWarningRule = new FlowRule();
            BeanUtils.copyProperties(rule, earlyWarningRule);
            /**
             * 这里是相当于把规则阈值改成原来的80%，达到提前预警的效果，
             * 这里建议把0.8做成配置
             */
            earlyWarningRule.setCount(rule.getCount() * 0.8);
            earlyWarningRuleList.add(earlyWarningRule);
        }
        Map<String, List<FlowRule>> flowRules = FlowRuleUtil.buildFlowRuleMap(earlyWarningRuleList);
        return flowRules.get(resource);
    }

    /**
     * get origin rule
     *
     * @param resource
     * @return
     */
    private FlowRule getOriginRule(String resource) {
        List<FlowRule> originRule = FlowRuleManager.getRules().stream().filter(flowRule -> flowRule.getResource().equals(resource)).collect(Collectors.toList());
        if (CollectionUtils.isEmpty(originRule)) {
            return null;
        }
        return originRule.get(0);
    }

    @Override
    public void entry(Context context, ResourceWrapper resourceWrapper, DefaultNode node, int count, boolean prioritized, Object... args)
            throws Throwable {
        String resource = context.getCurEntry().getResourceWrapper().getName();
        List<FlowRule> rules = getRuleProvider(resource);
        if (rules != null) {
            for (FlowRule rule : rules) {
                //这里取到的规则都是配置阈值的80%,这里如果检查到阈值了，说明就是到了真实阈值的80%，既可以发报警给对应负责人了
                if (!checker.canPassCheck(rule, context, node, count, prioritized)) {
                    FlowRule originRule = getOriginRule(resource);
                    String originRuleCount = originRule == null ? "未知" : String.valueOf(originRule.getCount());
                    logger.info("FlowEarlyWarning:服务{}目前的流量指标已经超过{}，接近配置的流控阈值:{},", resource, rule.getCount(), originRuleCount);
                    //TODO 报警功能自行实现
                    break;
                }
            }
        }
        fireEntry(context, resourceWrapper, node, count, prioritized, args);
    }

    @Override
    public void exit(Context context, ResourceWrapper resourceWrapper, int count, Object... args) {
        fireExit(context, resourceWrapper, count, args);
    }
}

DegradeEarlyWarningSlotｈｔｔｐｓ：／／app.memrise.com/course/5854316/
ｈｔｔｐｓ：／／app.memrise.com/course/5854316/?9n2Ro
ｈｔｔｐｓ：／／www.memrise.com/course/5854316/nei-meng-gu-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854316/nei-meng-gu-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854316/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854337/
ｈｔｔｐｓ：／／app.memrise.com/course/5854337/?iec4o
ｈｔｔｐｓ：／／www.memrise.com/course/5854337/nei-meng-gu-min-zu-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854337/nei-meng-gu-min-zu-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854337/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854339/
ｈｔｔｐｓ：／／app.memrise.com/course/5854339/?5dBvN
ｈｔｔｐｓ：／／www.memrise.com/course/5854339/chi-feng-xue-yuan-bi-ye-zheng-yang-ben-wei-xin-796/
ｈｔｔｐｓ：／／app.memrise.com/course/5854339/chi-feng-xue-yuan-bi-ye-zheng-yang-ben-wei-xin-796/
ｈｔｔｐｓ：／／app.memrise.com/course/5854339/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854340/
ｈｔｔｐｓ：／／app.memrise.com/course/5854340/?qs6M0
ｈｔｔｐｓ：／／www.memrise.com/course/5854340/nei-meng-gu-cai-jing-da-xue-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854340/nei-meng-gu-cai-jing-da-xue-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854340/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854346/
ｈｔｔｐｓ：／／app.memrise.com/course/5854346/?Lh191
ｈｔｔｐｓ：／／www.memrise.com/course/5854346/hu-lun-bei-er-xue-yuan-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854346/hu-lun-bei-er-xue-yuan-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854346/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854355/
ｈｔｔｐｓ：／／app.memrise.com/course/5854355/?Is4Wi
ｈｔｔｐｓ：／／www.memrise.com/course/5854355/ji-zhu-shi-fan-xue-yuan-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854355/ji-zhu-shi-fan-xue-yuan-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854355/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854369/
ｈｔｔｐｓ：／／app.memrise.com/course/5854369/?c8L42
ｈｔｔｐｓ：／／www.memrise.com/course/5854369/hu-he-hao-te-min-zu-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854369/hu-he-hao-te-min-zu-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854369/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854371/
ｈｔｔｐｓ：／／app.memrise.com/course/5854371/?y0Z3A
ｈｔｔｐｓ：／／www.memrise.com/course/5854371/he-tao-xue-yuan-bi-ye-zheng-yang-ben-wei-xin-79691/
ｈｔｔｐｓ：／／app.memrise.com/course/5854371/he-tao-xue-yuan-bi-ye-zheng-yang-ben-wei-xin-79691/
ｈｔｔｐｓ：／／app.memrise.com/course/5854371/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854380/
ｈｔｔｐｓ：／／app.memrise.com/course/5854380/?P1TPE
ｈｔｔｐｓ：／／www.memrise.com/course/5854380/nei-meng-gu-shi-fan-da-xue-bi-ye-zheng-yang-ben-we/
ｈｔｔｐｓ：／／app.memrise.com/course/5854380/nei-meng-gu-shi-fan-da-xue-bi-ye-zheng-yang-ben-we/
ｈｔｔｐｓ：／／app.memrise.com/course/5854380/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854381/
ｈｔｔｐｓ：／／app.memrise.com/course/5854381/?5DK7y
ｈｔｔｐｓ：／／www.memrise.com/course/5854381/nei-meng-gu-da-xue-bi-ye-zheng-yang-ben-wei-xin-79/
ｈｔｔｐｓ：／／app.memrise.com/course/5854381/nei-meng-gu-da-xue-bi-ye-zheng-yang-ben-wei-xin-79/
ｈｔｔｐｓ：／／app.memrise.com/course/5854381/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854390/
ｈｔｔｐｓ：／／app.memrise.com/course/5854390/?Ty8B4
ｈｔｔｐｓ：／／www.memrise.com/course/5854390/nei-meng-gu-yi-zhu-xue-yuan-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854390/nei-meng-gu-yi-zhu-xue-yuan-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854390/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854395/
ｈｔｔｐｓ：／／app.memrise.com/course/5854395/?m616i
ｈｔｔｐｓ：／／www.memrise.com/course/5854395/e-er-duo-si-ying-yong-ji-zhu-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854395/e-er-duo-si-ying-yong-ji-zhu-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854395/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854402/
ｈｔｔｐｓ：／／app.memrise.com/course/5854402/?e4SGU
ｈｔｔｐｓ：／／www.memrise.com/course/5854402/nei-meng-gu-jian-zhu-zhi-ye-ji-zhu-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854402/nei-meng-gu-jian-zhu-zhi-ye-ji-zhu-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854402/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854409/
ｈｔｔｐｓ：／／app.memrise.com/course/5854409/?it4Nv
ｈｔｔｐｓ：／／www.memrise.com/course/5854409/nei-meng-gu-feng-zhou-zhi-ye-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854409/nei-meng-gu-feng-zhou-zhi-ye-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854409/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854416/
ｈｔｔｐｓ：／／app.memrise.com/course/5854416/?BG5zB
ｈｔｔｐｓ：／／www.memrise.com/course/5854416/bao-tou-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-be/
ｈｔｔｐｓ：／／app.memrise.com/course/5854416/bao-tou-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-be/
ｈｔｔｐｓ：／／app.memrise.com/course/5854416/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854421/
ｈｔｔｐｓ：／／app.memrise.com/course/5854421/?80NuK
ｈｔｔｐｓ：／／www.memrise.com/course/5854421/xing-an-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-be/
ｈｔｔｐｓ：／／app.memrise.com/course/5854421/xing-an-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-be/
ｈｔｔｐｓ：／／app.memrise.com/course/5854421/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854429/
ｈｔｔｐｓ：／／app.memrise.com/course/5854429/?Dt9l5
ｈｔｔｐｓ：／／www.memrise.com/course/5854429/hu-he-hao-te-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854429/hu-he-hao-te-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854429/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854432/
ｈｔｔｐｓ：／／app.memrise.com/course/5854432/?vbfD1
ｈｔｔｐｓ：／／www.memrise.com/course/5854432/bao-tou-qing-gong-zhi-ye-ji-zhu-xue-yuan-bi-ye-zhe/
ｈｔｔｐｓ：／／app.memrise.com/course/5854432/bao-tou-qing-gong-zhi-ye-ji-zhu-xue-yuan-bi-ye-zhe/
ｈｔｔｐｓ：／／app.memrise.com/course/5854432/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854449/
ｈｔｔｐｓ：／／app.memrise.com/course/5854449/?9I2xt
ｈｔｔｐｓ：／／www.memrise.com/course/5854449/nei-meng-gu-hua-gong-zhi-ye-xue-yuan-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854449/nei-meng-gu-hua-gong-zhi-ye-xue-yuan-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854449/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854458/
ｈｔｔｐｓ：／／app.memrise.com/course/5854458/?f70uG
ｈｔｔｐｓ：／／www.memrise.com/course/5854458/nei-meng-gu-ji-dian-zhi-ye-ji-zhu-xue-yuan-bi-ye-z/
ｈｔｔｐｓ：／／app.memrise.com/course/5854458/nei-meng-gu-ji-dian-zhi-ye-ji-zhu-xue-yuan-bi-ye-z/
ｈｔｔｐｓ：／／app.memrise.com/course/5854458/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854462/
ｈｔｔｐｓ：／／app.memrise.com/course/5854462/?2nox0
ｈｔｔｐｓ：／／www.memrise.com/course/5854462/nei-meng-gu-dian-zi-xin-xi-zhi-ye-ji-zhu-xue-yuan/
ｈｔｔｐｓ：／／app.memrise.com/course/5854462/nei-meng-gu-dian-zi-xin-xi-zhi-ye-ji-zhu-xue-yuan/
ｈｔｔｐｓ：／／app.memrise.com/course/5854462/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854476/
ｈｔｔｐｓ：／／app.memrise.com/course/5854476/?NX09N
ｈｔｔｐｓ：／／www.memrise.com/course/5854476/nei-meng-gu-jing-cha-zhi-ye-xue-yuan-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854476/nei-meng-gu-jing-cha-zhi-ye-xue-yuan-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854476/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854482/
ｈｔｔｐｓ：／／app.memrise.com/course/5854482/?Mcq1k
ｈｔｔｐｓ：／／www.memrise.com/course/5854482/xi-lin-guo-le-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854482/xi-lin-guo-le-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854482/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854488/
ｈｔｔｐｓ：／／app.memrise.com/course/5854488/?zUD62
ｈｔｔｐｓ：／／www.memrise.com/course/5854488/nei-meng-gu-ti-yu-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854488/nei-meng-gu-ti-yu-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854488/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854494/
ｈｔｔｐｓ：／／app.memrise.com/course/5854494/?9rsYT
ｈｔｔｐｓ：／／www.memrise.com/course/5854494/tong-liao-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854494/tong-liao-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854494/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854501/
ｈｔｔｐｓ：／／app.memrise.com/course/5854501/?WBgZh
ｈｔｔｐｓ：／／www.memrise.com/course/5854501/ke-er-qin-yi-zhu-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854501/ke-er-qin-yi-zhu-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854501/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854503/
ｈｔｔｐｓ：／／app.memrise.com/course/5854503/?WM5F1
ｈｔｔｐｓ：／／www.memrise.com/course/5854503/nei-meng-gu-jiao-tong-zhi-ye-ji-zhu-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854503/nei-meng-gu-jiao-tong-zhi-ye-ji-zhu-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854503/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854511/
ｈｔｔｐｓ：／／app.memrise.com/course/5854511/?Ar8c0
ｈｔｔｐｓ：／／www.memrise.com/course/5854511/wu-lan-cha-bu-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854511/wu-lan-cha-bu-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854511/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854517/
ｈｔｔｐｓ：／／app.memrise.com/course/5854517/?S4YY4
ｈｔｔｐｓ：／／www.memrise.com/course/5854517/bao-tou-gang-tie-zhi-ye-ji-zhu-xue-yuan-bi-ye-zhen/
ｈｔｔｐｓ：／／app.memrise.com/course/5854517/bao-tou-gang-tie-zhi-ye-ji-zhu-xue-yuan-bi-ye-zhen/
ｈｔｔｐｓ：／／app.memrise.com/course/5854517/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854520/
ｈｔｔｐｓ：／／app.memrise.com/course/5854520/?h3P5d
ｈｔｔｐｓ：／／www.memrise.com/course/5854520/wu-hai-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854520/wu-hai-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-ben/
ｈｔｔｐｓ：／／app.memrise.com/course/5854520/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854521/
ｈｔｔｐｓ：／／app.memrise.com/course/5854521/?gm68S
ｈｔｔｐｓ：／／www.memrise.com/course/5854521/nei-meng-gu-ke-ji-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854521/nei-meng-gu-ke-ji-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854521/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854529/
ｈｔｔｐｓ：／／app.memrise.com/course/5854529/?9v0x4
ｈｔｔｐｓ：／／www.memrise.com/course/5854529/chi-feng-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-b/
ｈｔｔｐｓ：／／app.memrise.com/course/5854529/chi-feng-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang-b/
ｈｔｔｐｓ：／／app.memrise.com/course/5854529/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854534/
ｈｔｔｐｓ：／／app.memrise.com/course/5854534/?8k47W
ｈｔｔｐｓ：／／www.memrise.com/course/5854534/nei-meng-gu-bei-fang-zhi-ye-ji-zhu-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854534/nei-meng-gu-bei-fang-zhi-ye-ji-zhu-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854534/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854543/
ｈｔｔｐｓ：／／app.memrise.com/course/5854543/?25jq5
ｈｔｔｐｓ：／／www.memrise.com/course/5854543/nei-meng-gu-jing-mao-wai-yu-zhi-ye-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854543/nei-meng-gu-jing-mao-wai-yu-zhi-ye-xue-yuan-bi-ye/
ｈｔｔｐｓ：／／app.memrise.com/course/5854543/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854550/
ｈｔｔｐｓ：／／app.memrise.com/course/5854550/?3jnfE
ｈｔｔｐｓ：／／www.memrise.com/course/5854550/wu-lan-cha-bu-yi-xue-gao-deng-zhuan-ke-xue-xiao-bi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854550/wu-lan-cha-bu-yi-xue-gao-deng-zhuan-ke-xue-xiao-bi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854550/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854552/
ｈｔｔｐｓ：／／app.memrise.com/course/5854552/?MdfWm
ｈｔｔｐｓ：／／www.memrise.com/course/5854552/bao-tou-tie-dao-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854552/bao-tou-tie-dao-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854552/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854559/
ｈｔｔｐｓ：／／app.memrise.com/course/5854559/?PhV6d
ｈｔｔｐｓ：／／www.memrise.com/course/5854559/e-er-duo-si-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854559/e-er-duo-si-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854559/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854564/
ｈｔｔｐｓ：／／app.memrise.com/course/5854564/?o7cWH
ｈｔｔｐｓ：／／www.memrise.com/course/5854564/hu-lun-bei-er-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854564/hu-lun-bei-er-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854564/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854569/
ｈｔｔｐｓ：／／app.memrise.com/course/5854569/?ifRfc
ｈｔｔｐｓ：／／www.memrise.com/course/5854569/nei-meng-gu-gong-ye-zhi-ye-xue-yuan-bi-ye-zheng-ya/
ｈｔｔｐｓ：／／app.memrise.com/course/5854569/nei-meng-gu-gong-ye-zhi-ye-xue-yuan-bi-ye-zheng-ya/
ｈｔｔｐｓ：／／app.memrise.com/course/5854569/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854570/
ｈｔｔｐｓ：／／app.memrise.com/course/5854570/?1zEGj
ｈｔｔｐｓ：／／www.memrise.com/course/5854570/man-zhou-li-e-yu-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854570/man-zhou-li-e-yu-zhi-ye-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854570/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854586/
ｈｔｔｐｓ：／／app.memrise.com/course/5854586/?1kfL1
ｈｔｔｐｓ：／／www.memrise.com/course/5854586/nei-meng-gu-neng-yuan-zhi-ye-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854586/nei-meng-gu-neng-yuan-zhi-ye-xue-yuan-bi-ye-zheng/
ｈｔｔｐｓ：／／app.memrise.com/course/5854586/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854588/
ｈｔｔｐｓ：／／app.memrise.com/course/5854588/?ZbE2p
ｈｔｔｐｓ：／／www.memrise.com/course/5854588/nei-meng-gu-mei-zhu-zhi-ye-xue-yuan-bi-ye-zheng-ya/
ｈｔｔｐｓ：／／app.memrise.com/course/5854588/nei-meng-gu-mei-zhu-zhi-ye-xue-yuan-bi-ye-zheng-ya/
ｈｔｔｐｓ：／／app.memrise.com/course/5854588/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854590/
ｈｔｔｐｓ：／／app.memrise.com/course/5854590/?h90Sf
ｈｔｔｐｓ：／／www.memrise.com/course/5854590/a-la-shan-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854590/a-la-shan-zhi-ye-ji-zhu-xue-yuan-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854590/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854606/
ｈｔｔｐｓ：／／app.memrise.com/course/5854606/?5by0q
ｈｔｔｐｓ：／／www.memrise.com/course/5854606/e-er-duo-si-sheng-tai-huan-jing-zhi-ye-xue-yuan-bi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854606/e-er-duo-si-sheng-tai-huan-jing-zhi-ye-xue-yuan-bi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854606/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854607/
ｈｔｔｐｓ：／／app.memrise.com/course/5854607/?n93oJ
ｈｔｔｐｓ：／／www.memrise.com/course/5854607/gao-deng-zhuan-ke-xue-xiao-bi-ye-zheng-yang-ben-we/
ｈｔｔｐｓ：／／app.memrise.com/course/5854607/gao-deng-zhuan-ke-xue-xiao-bi-ye-zheng-yang-ben-we/
ｈｔｔｐｓ：／／app.memrise.com/course/5854607/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854613/
ｈｔｔｐｓ：／／app.memrise.com/course/5854613/?Xwpl0
ｈｔｔｐｓ：／／www.memrise.com/course/5854613/da-lian-li-gong-da-xue-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854613/da-lian-li-gong-da-xue-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854613/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854614/
ｈｔｔｐｓ：／／app.memrise.com/course/5854614/?4B56v
ｈｔｔｐｓ：／／www.memrise.com/course/5854614/zha-lan-tun-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854614/zha-lan-tun-zhi-ye-xue-yuan-bi-ye-zheng-yang-ben-w/
ｈｔｔｐｓ：／／app.memrise.com/course/5854614/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854632/
ｈｔｔｐｓ：／／app.memrise.com/course/5854632/?0jaTE
ｈｔｔｐｓ：／／www.memrise.com/course/5854632/liao-zhu-da-xue-bi-ye-zheng-yang-ben-wei-xin-79691/
ｈｔｔｐｓ：／／app.memrise.com/course/5854632/liao-zhu-da-xue-bi-ye-zheng-yang-ben-wei-xin-79691/
ｈｔｔｐｓ：／／app.memrise.com/course/5854632/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854633/
ｈｔｔｐｓ：／／app.memrise.com/course/5854633/?Rdt23
ｈｔｔｐｓ：／／www.memrise.com/course/5854633/chen-yang-hang-kong-hang-tian-da-xue-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854633/chen-yang-hang-kong-hang-tian-da-xue-bi-ye-zheng-y/
ｈｔｔｐｓ：／／app.memrise.com/course/5854633/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854638/
ｈｔｔｐｓ：／／app.memrise.com/course/5854638/?qib7O
ｈｔｔｐｓ：／／www.memrise.com/course/5854638/chen-yang-gong-ye-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854638/chen-yang-gong-ye-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854638/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854646/
ｈｔｔｐｓ：／／app.memrise.com/course/5854646/?AKoLJ
ｈｔｔｐｓ：／／www.memrise.com/course/5854646/chen-yang-li-gong-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854646/chen-yang-li-gong-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854646/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854649/
ｈｔｔｐｓ：／／app.memrise.com/course/5854649/?T70Za
ｈｔｔｐｓ：／／www.memrise.com/course/5854649/dong-bei-da-xue-bi-ye-zheng-yang-ben-wei-xin-79691/
ｈｔｔｐｓ：／／app.memrise.com/course/5854649/dong-bei-da-xue-bi-ye-zheng-yang-ben-wei-xin-79691/
ｈｔｔｐｓ：／／app.memrise.com/course/5854649/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854655/
ｈｔｔｐｓ：／／app.memrise.com/course/5854655/?54mBI
ｈｔｔｐｓ：／／www.memrise.com/course/5854655/liao-zhu-ke-ji-da-xue-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854655/liao-zhu-ke-ji-da-xue-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854655/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854659/
ｈｔｔｐｓ：／／app.memrise.com/course/5854659/?mR3r2
ｈｔｔｐｓ：／／www.memrise.com/course/5854659/liao-zhu-gong-cheng-ji-zhu-da-xue-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854659/liao-zhu-gong-cheng-ji-zhu-da-xue-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854659/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854670/
ｈｔｔｐｓ：／／app.memrise.com/course/5854670/?pE56R
ｈｔｔｐｓ：／／www.memrise.com/course/5854670/liao-zhu-shi-you-hua-gong-da-xue-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854670/liao-zhu-shi-you-hua-gong-da-xue-bi-ye-zheng-yang/
ｈｔｔｐｓ：／／app.memrise.com/course/5854670/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854674/
ｈｔｔｐｓ：／／app.memrise.com/course/5854674/?5D76t
ｈｔｔｐｓ：／／www.memrise.com/course/5854674/chen-yang-hua-gong-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854674/chen-yang-hua-gong-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854674/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854677/
ｈｔｔｐｓ：／／app.memrise.com/course/5854677/?r53f9
ｈｔｔｐｓ：／／www.memrise.com/course/5854677/da-lian-jiao-tong-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854677/da-lian-jiao-tong-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854677/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854690/
ｈｔｔｐｓ：／／app.memrise.com/course/5854690/?WdHvd
ｈｔｔｐｓ：／／www.memrise.com/course/5854690/chen-yang-jian-zhu-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854690/chen-yang-jian-zhu-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854690/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854694/
ｈｔｔｐｓ：／／app.memrise.com/course/5854694/?YyQ7U
ｈｔｔｐｓ：／／www.memrise.com/course/5854694/da-lian-hai-shi-da-xue-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854694/da-lian-hai-shi-da-xue-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854694/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854702/
ｈｔｔｐｓ：／／app.memrise.com/course/5854702/?cxInf
ｈｔｔｐｓ：／／www.memrise.com/course/5854702/chen-yang-nong-ye-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854702/chen-yang-nong-ye-da-xue-bi-ye-zheng-yang-ben-wei/
ｈｔｔｐｓ：／／app.memrise.com/course/5854702/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854711/
ｈｔｔｐｓ：／／app.memrise.com/course/5854711/?q7qjT
ｈｔｔｐｓ：／／www.memrise.com/course/5854711/da-lian-hai-yang-da-xue-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854711/da-lian-hai-yang-da-xue-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854711/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854716/
ｈｔｔｐｓ：／／app.memrise.com/course/5854716/?sEeuE
ｈｔｔｐｓ：／／www.memrise.com/course/5854716/liao-zhu-gong-ye-da-xue-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854716/liao-zhu-gong-ye-da-xue-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854716/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854717/
ｈｔｔｐｓ：／／app.memrise.com/course/5854717/?esm7O
ｈｔｔｐｓ：／／www.memrise.com/course/5854717/zhong-guo-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854717/zhong-guo-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei-xi/
ｈｔｔｐｓ：／／app.memrise.com/course/5854717/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854740/
ｈｔｔｐｓ：／／app.memrise.com/course/5854740/?O0m12
ｈｔｔｐｓ：／／www.memrise.com/course/5854740/da-lian-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854740/da-lian-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854740/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854749/
ｈｔｔｐｓ：／／app.memrise.com/course/5854749/?J1PG2
ｈｔｔｐｓ：／／www.memrise.com/course/5854749/jin-zhou-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854749/jin-zhou-yi-ke-da-xue-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854749/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854754/
ｈｔｔｐｓ：／／app.memrise.com/course/5854754/?T2HD9
ｈｔｔｐｓ：／／www.memrise.com/course/5854754/chen-yang-yao-ke-da-xue-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854754/chen-yang-yao-ke-da-xue-bi-ye-zheng-yang-ben-wei-x/
ｈｔｔｐｓ：／／app.memrise.com/course/5854754/topic-activator/
ｈｔｔｐｓ：／／app.memrise.com/course/5854766/
ｈｔｔｐｓ：／／app.memrise.com/course/5854766/?S8haA
ｈｔｔｐｓ：／／www.memrise.com/course/5854766/chen-yang-yi-xue-yuan-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854766/chen-yang-yi-xue-yuan-bi-ye-zheng-yang-ben-wei-xin/
ｈｔｔｐｓ：／／app.memrise.com/course/5854766/topic-activator/
ｈｔｔｐｓ：／／