Hadoop YARN的ResourceManager报错TableNotFoundException：prod.timelineservice.flowactivity解决

1. 问题

在resourcemanager.log中出现下面的异常：

java.io.UncheckedIOException: org.apache.hadoop.hbase.TableNotFoundException: prod.timelineservice.flowactivity
	at org.apache.hadoop.hbase.client.ResultScanner$1.hasNext(ResultScanner.java:55)
	at org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader.readEntities(TimelineEntityReader.java:283)
	at org.apache.hadoop.yarn.server.timelineservice.storage.HBaseStorageMonitor.healthCheck(HBaseStorageMonitor.java:77)
	at org.apache.hadoop.yarn.server.timelineservice.storage.TimelineStorageMonitor$MonitorThread.run(TimelineStorageMonitor.java:89)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)
	at java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)
...

2. 代码分析

分析查看源代码

1. 类org.apache.hadoop.yarn.server.timelineservice.storage.reader.TimelineEntityReader

  public Set<TimelineEntity> readEntities(Configuration hbaseConf,
      Connection conn) throws IOException {
    validateParams();
    augmentParams(hbaseConf, conn);

    Set<TimelineEntity> entities = new LinkedHashSet<>();
    FilterList filterList = createFilterList();
    if (filterList != null) {
      LOG.debug("FilterList created for scan is - {}", filterList);
    }
    // 目标--1[[
    ResultScanner results = getResults(hbaseConf, conn, filterList);
    //]]
    try {
      for (Result result : results) {
        TimelineEntity entity = parseEntity(result);
        if (entity == null) {
          continue;
        }
        entities.add(entity);
        if (entities.size() == filters.getLimit()) {
          break;
        }
      }
      return entities;
    } finally {
      results.close();
    }
  }

2. 类org.apache.hadoop.yarn.server.timelineservice.storage.reader.FlowActivityEntityReader extends TimelineEntityReader

// 目标--4[[
private static final FlowActivityTableRW FLOW_ACTIVITY_TABLE = new FlowActivityTableRW();
//]

// 目标--3[[
@Override
protected BaseTableRW<?> getTable() {
  return FLOW_ACTIVITY_TABLE;
}
// ]]

@Override
  protected ResultScanner getResults(Configuration hbaseConf,
      Connection conn, FilterList filterList) throws IOException {
    Scan scan = new Scan();
    String clusterId = getContext().getClusterId();
    if (getFilters().getFromId() == null
        && getFilters().getCreatedTimeBegin() == 0L
        && getFilters().getCreatedTimeEnd() == Long.MAX_VALUE) {
       // All records have to be chosen.
      scan.setRowPrefixFilter(new FlowActivityRowKeyPrefix(clusterId)
          .getRowKeyPrefix());
    } else if (getFilters().getFromId() != null) {
      FlowActivityRowKey key = null;
      try {
        key =
            FlowActivityRowKey.parseRowKeyFromString(getFilters().getFromId());
      } catch (IllegalArgumentException e) {
        throw new BadRequestException("Invalid filter fromid is provided.");
      }
      if (!clusterId.equals(key.getClusterId())) {
        throw new BadRequestException(
            "fromid doesn't belong to clusterId=" + clusterId);
      }
      scan.withStartRow(key.getRowKey());
      scan.withStopRow(
          new FlowActivityRowKeyPrefix(clusterId,
              (getFilters().getCreatedTimeBegin() <= 0 ? 0
                  : (getFilters().getCreatedTimeBegin() - 1)))
                      .getRowKeyPrefix());
    } else {
      scan.withStartRow(new FlowActivityRowKeyPrefix(clusterId, getFilters()
          .getCreatedTimeEnd()).getRowKeyPrefix());
      scan.withStopRow(new FlowActivityRowKeyPrefix(clusterId, (getFilters()
          .getCreatedTimeBegin() <= 0 ? 0
          : (getFilters().getCreatedTimeBegin() - 1))).getRowKeyPrefix());
    }
    // use the page filter to limit the result to the page size
    // the scanner may still return more than the limit; therefore we need to
    // read the right number as we iterate
    scan.setFilter(new PageFilter(getFilters().getLimit()));
    // 目标--2[[
    return getTable().getResultScanner(hbaseConf, conn, scan);
    //]]
  }

3. 类org.apache.hadoop.yarn.server.timelineservice.storage.flow.FlowActivityTableRW extends BaseTableRW

// 目标--7[[
private static final String PREFIX = YarnConfiguration.TIMELINE_SERVICE_PREFIX + "flowactivity";
/** config param name that specifies the flowactivity table name. */
public static final String TABLE_NAME_CONF_NAME = PREFIX + ".table.name";
//]]

/** default value for flowactivity table name. */
//目标--6[[
public static final String DEFAULT_TABLE_NAME = "timelineservice.flowactivity";
  
// 目标--5[[
public FlowActivityTableRW() {
    super(TABLE_NAME_CONF_NAME, DEFAULT_TABLE_NAME);
  }
//]]

4. 类org.apache.hadoop.yarn.server.timelineservice.storage.common.BaseTableRW

  public static TableName getTableName(Configuration conf, String tableNameInConf, String defaultTableName) {
    // 目标--8[[
    String tableName = conf.get(tableNameInConf, defaultTableName) ;
    //]
    return getTableName(conf, tableName);
  }

3. 解决

因为在笔者的hadoop配置yarn-site.xml中，指定了如下配置：

<property>
  <name>yarn.timeline-service.schema.prefix</name>
  <value>yarn:prod</value>
</property>

配置错误，正确的应该是

<property>
  <name>yarn.timeline-service.hbase-schema.prefix</name>
  <value>yarn:prod.</value>
</property>