kubernetes+ prometheus自动伸缩的设计与实现(二)

上一篇介绍了kubernetes+ prometheus中k8s的hpa的部分，但是没有介绍转化的部分，下面接着介绍
如果每次hpa调用查询metrics的指标我就去调用prometheus,但指标在prometheus里面是否存在，我们先要本地缓存一下，免给prometheus带来很大压力，所以本地缓存还有应该有的，主要缓存prometheus里面存在的指标，定义如下：

metrics []provider.MetricInfo

上面metrics保存prometheus的指标。当然本地缓存指标数据还有定期更新。

func (l *cachingMetricsLister) Run() {
    l.RunUntil(wait.NeverStop)
}

func (l *cachingMetricsLister) RunUntil(stopChan <-chan struct{}) {
    go wait.Until(func() {
        if err := l.updateMetrics(); err != nil {
            utilruntime.HandleError(err)
        }
    }, l.updateInterval, stopChan)
}

通过updateMetrics更新本地数据，这个方法首先是查询数据

func (l *cachingMetricsLister) updateMetrics() error {
    startTime := pmodel.Now().Add(-1 * l.updateInterval)

    // container-specific metrics from cAdvsior have their own form, and need special handling
    containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))
    namespacedSel := prom.MatchSeries("", prom.LabelNeq("namespace", ""), prom.NameNotMatches("^container_.*"))

    series, err := l.promClient.Series(context.Background(), pmodel.Interval{startTime, 0}, containerSel, namespacedSel)
    if err != nil {
        return fmt.Errorf("unable to update list of all available metrics: %v", err)
    }

    glog.V(10).Infof("Set available metric list from Prometheus to: %v", series)
    if len(series)>0 {
        l.SetSeries(series)
    }
    return nil
}

先是通过Series获取指标序列，一个是容器的container，另一个是命名空间的namespace的。

func (h *queryClient) Series(ctx context.Context, interval model.Interval, selectors ...Selector) ([]Series, error) {
    vals := url.Values{}
    if interval.Start != 0 {
        vals.Set("start", interval.Start.String())
    }
    if interval.End != 0 {
        vals.Set("end", interval.End.String())
    }

    for _, selector := range selectors {
        vals.Add("match[]", string(selector))
    }

    res, err := h.api.Do(ctx, "GET", seriesURL, vals)
    if err != nil {
        return nil, err
    }

    var seriesRes []Series
    err = json.Unmarshal(res.Data, &seriesRes)
    return seriesRes, err
}

这里通过prometheus的series接口获取指标序列，从而更新本地数据

func (r *basicSeriesRegistry) SetSeries(newSeries []prom.Series) error {
    newInfo := make(map[provider.MetricInfo]seriesInfo)
    for _, series := range newSeries {
        if strings.HasPrefix(series.Name, "container_") {
            r.namer.processContainerSeries(series, newInfo)
        } else if namespaceLabel, hasNamespaceLabel := series.Labels["namespace"]; hasNamespaceLabel && namespaceLabel != "" {
            // we also handle namespaced metrics here as part of the resource-association logic
            if err := r.namer.processNamespacedSeries(series, newInfo); err != nil {
                glog.Errorf("Unable to process namespaced series %q: %v", series.Name, err)
                continue
            }
        } else {
            if err := r.namer.processRootScopedSeries(series, newInfo); err != nil {
                glog.Errorf("Unable to process root-scoped series %q: %v", series.Name, err)
                continue
            }
        }
    }

    newMetrics := make([]provider.MetricInfo, 0, len(newInfo))
    for info := range newInfo {
        newMetrics = append(newMetrics, info)
    }

    r.mu.Lock()
    defer r.mu.Unlock()

    r.info = newInfo
    r.metrics = newMetrics

    return nil
}

这里，如果指标是container_开头、包含namespace指标的和其它三种情况处理，先看processContainerSeries里面

func (n *metricNamer) processContainerSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) {

    originalName := series.Name

    var name string
    metricKind := GaugeSeries
    if override, hasOverride := n.overrides[series.Name]; hasOverride {
        name = override.metricName
        metricKind = override.kind
    } else {
        // 这里会将指标前缀 "container_" 去掉
        series.Name = series.Name[10:]
        name, metricKind = n.metricNameFromSeries(series)
    }

    info := provider.MetricInfo{
        GroupResource: schema.GroupResource{Resource: "pods"},
        Namespaced:    true,
        Metric:        name,
    }
    //infos 这个map的key类似pods/memory_swap(namespaced)这样
    infos[info] = seriesInfo{
        kind:        metricKind,
        baseSeries:  prom.Series{Name: originalName},
        isContainer: true,
    }
}

因为上面重写了String方法

func (i MetricInfo) String() string {
    if i.Namespaced {
        return fmt.Sprintf("%s/%s(namespaced)", i.GroupResource.String(), i.Metric)
    } else {
        return fmt.Sprintf("%s/%s", i.GroupResource.String(), i.Metric)
    }
}

所以key是pods/memory_swap(namespaced)。

如果是processNamespacedSeries或者processRootScopedSeries都是类似的，不过指标组不再是pod，譬如ingress_http_hits_total{pod=”foo”,service=”bar”,ingress=”baz”,namespace=”ns”}这样的一个指标
会返回三个GroupResources: “pods”, “services”, and “ingresses”.
这样指标就会变成services/ingress_http_hits_total。
补充一下，series 接口是prometheus查询数据序列的接口eg：

curl -g 'http://localhost:9090/api/v1/series?match[]=up&match[]=process_start_time_seconds{job="prometheus"}'
{
   "status" : "success",
   "data" : [
      {
         "__name__" : "up",
         "job" : "prometheus",
         "instance" : "localhost:9090"
      },
      {
         "__name__" : "up",
         "job" : "node",
         "instance" : "localhost:9091"
      },
      {
         "__name__" : "process_start_time_seconds",
         "job" : "prometheus",
         "instance" : "localhost:9090"
      }
   ]
}

这样就能获取到指标序列了并且指标的label。这样prometheus本地指标缓存功能有了。