OpenTSDB源码分析之TSDB表操作(新增)

OpenTSDB数据存储架构图如下，它分四个步骤：

1.     主机报告检测结果给本地的tcollection。

2.     tcollection发送检测结果给远程的tsd。

3.     tsd构造记录，并把数据写入HBase。

4.     HBase存储数据，并确认写入请求。

tcollector使用类Telnet的协议的发送监控信息的示例代码如下：

package test.telnet;

import java.io.PrintStream;

import org.apache.commons.net.telnet.TelnetClient;
 
public class TelnetUtil {
    private TelnetClient telnetClient = new TelnetClient();
    private PrintStream out;
 
    public TelnetUtil(String ip, Integer port) {
       try {
           telnetClient.connect(ip, port);
           out = new PrintStream(telnetClient.getOutputStream());
       } catch (Exception e) {
          e.printStackTrace();
       }
    }
 
    public static void main(String[] args) {
    	testGet("localhost", 4242);
    }
    
    public static void testGet(String url, Integer port) {
    	TelnetUtil telnetTest = new TelnetUtil(url, port);
    	String strTime = "" + System.currentTimeMillis();
    	System.out.println(strTime);
    	telnetTest.write("put t22 " + strTime + " 11 host=foo");
    	telnetTest.disconnect();
    }

    public void write(String command) {
       try {
           out.println(command);
           out.flush();
           System.out.println("本次执行的telnet命令:" + command);
       } catch (Exception e) {
           e.printStackTrace();
       }
    }
 
    public void disconnect() {
       try {
           Thread.sleep(10);
           telnetClient.disconnect();
       } catch (Exception e) {
           e.printStackTrace();
       }
    }
 
}

tsd服务主要用来接收监控信息、构造记录并写入HBase。OpenTSDB在TSDBMain.main()方法中启动netty服务器用来接收来自远程的Http或RPC请求，针对不同的请求封装成HttpRpc 或TelnetRpc来进行处理。

TSDBMain类方法：
public static void main(String[] args) throws IOException {

		// ./build/tsdb tsd --port=4242 --staticroot=build/staticroot
		// --cachedir=/tmp/tsdtmp –zkquorum=localhost

		args = new String[] { "--port=4242", "--staticroot=F:/workspace/OpenTsdb/staticroot",
				"--cachedir=D:/doc_hadoop/_00_hbase/opentsdb/opentsdb2_linux/build/tmp" };

		Logger log = LoggerFactory.getLogger(TSDMain.class);
		log.info("Starting.");
		log.info(BuildData.revisionString());
		log.info(BuildData.buildString());
		try {
			System.in.close(); // Release a FD we don't need.
		} catch (Exception e) {
			log.warn("Failed to close stdin", e);
		}

		final ArgP argp = new ArgP();
		CliOptions.addCommon(argp);
		argp.addOption("--port", "NUM", "TCP port to listen on.");
		argp.addOption("--bind", "ADDR", "Address to bind to (default: 0.0.0.0).");
		argp.addOption("--staticroot", "PATH", "Web root from which to serve static files (/s URLs).");
		argp.addOption("--cachedir", "PATH", "Directory under which to cache result of requests.");
		argp.addOption("--worker-threads", "NUM", "Number for async io workers (default: cpu * 2).");
		argp.addOption("--async-io", "true|false", "Use async NIO (default true) or traditional blocking io");
		argp.addOption("--backlog", "NUM", "Size of connection attempt queue (default: 3072 or kernel" + " somaxconn.");
		argp.addOption("--flush-interval", "MSEC", "Maximum time for which a new data point can be buffered" + " (default: " + DEFAULT_FLUSH_INTERVAL
				+ ").");
		CliOptions.addAutoMetricFlag(argp);
		args = CliOptions.parse(argp, args);
		args = null; // free().

		// get a config object
		Config config = CliOptions.getConfig(argp);

		// check for the required parameters
		try {
			if (config.getString("tsd.http.staticroot").isEmpty())
				usage(argp, "Missing static root directory", 1);
		} catch (NullPointerException npe) {
			usage(argp, "Missing static root directory", 1);
		}
		try {
			if (config.getString("tsd.http.cachedir").isEmpty())
				usage(argp, "Missing cache directory", 1);
		} catch (NullPointerException npe) {
			usage(argp, "Missing cache directory", 1);
		}
		try {
			if (!config.hasProperty("tsd.network.port"))
				usage(argp, "Missing network port", 1);
			config.getInt("tsd.network.port");
		} catch (NumberFormatException nfe) {
			usage(argp, "Invalid network port setting", 1);
		}

		// validate the cache and staticroot directories
		try {
			checkDirectory(config.getString("tsd.http.staticroot"), DONT_CREATE, !MUST_BE_WRITEABLE);
			checkDirectory(config.getString("tsd.http.cachedir"), CREATE_IF_NEEDED, MUST_BE_WRITEABLE);
		} catch (IllegalArgumentException e) {
			usage(argp, e.getMessage(), 3);
		}

		final ServerSocketChannelFactory factory;
		if (config.getBoolean("tsd.network.async_io")) {// 默认为true(在Config中设置)
			/**
			 * 默认为处理器的两倍
			 */
			int workers = Runtime.getRuntime().availableProcessors() * 2;
			if (config.hasProperty("tsd.network.worker_threads")) {
				try {
					workers = config.getInt("tsd.network.worker_threads");
				} catch (NumberFormatException nfe) {
					usage(argp, "Invalid worker thread count", 1);
				}
			}
			// System.out.println("workers=" + workers);
			factory = 
                   new NioServerSocketChannelFactory(Executors.newCachedThreadPool(), 
                   Executors.newCachedThreadPool(), workers);
		} else {
			factory = 
                    new OioServerSocketChannelFactory(Executors.newCachedThreadPool(), 
                    Executors.newCachedThreadPool());
		}

		TSDB tsdb = null;
		try {
			tsdb = new TSDB(config);
			tsdb.initializePlugins(true);

			// Make sure we don't even start if we can't find our tables.
			tsdb.checkNecessaryTablesExist().joinUninterruptibly();

			// 注册关闭程序
			registerShutdownHook(tsdb);

			final ServerBootstrap server = new ServerBootstrap(factory);

			/**
			 * 此方法是本类的重点
			 * PipelineFactory内部负责netty服务器的具体实现
			 */
			server.setPipelineFactory(new PipelineFactory(tsdb));
			
			
			if (config.hasProperty("tsd.network.backlog")) {
				server.setOption("backlog", config.getInt("tsd.network.backlog"));
			}
			server.setOption("child.tcpNoDelay", 
                      config.getBoolean("tsd.network.tcp_no_delay"));
			server.setOption("child.keepAlive", 
                       config.getBoolean("tsd.network.keep_alive"));
			server.setOption("reuseAddress", 
                       config.getBoolean("tsd.network.reuse_address"));

			// null is interpreted as the wildcard address.
			InetAddress bindAddress = null;
			if (config.hasProperty("tsd.network.bind")) {
				bindAddress = 
                         InetAddress.getByName(config.getString("tsd.network.bind"));
			}

			// we validated the network port config earlier
			final InetSocketAddress addr = new InetSocketAddress(bindAddress, 
                       config.getInt("tsd.network.port"));
			server.bind(addr);
			log.info("Ready to serve on " + addr);
		} catch (Throwable e) {
			factory.releaseExternalResources();
			try {
				if (tsdb != null)
					tsdb.shutdown().joinUninterruptibly();
			} catch (Exception e2) {
				log.error("Failed to shutdown HBase client", e2);
			}
			throw new RuntimeException("Initialization failed", e);
		}
		// The server is now running in separate threads, we can exit main.
	}

RpcHandler类方法
     /**
	 * TSDB服务可以监听telnet和http两种信号
	 */
	@Override
	public void messageReceived(final ChannelHandlerContext ctx, 
                 final MessageEvent msgevent) {
		try {
			final Object message = msgevent.getMessage();
			
			/**
			 * telnet信号 
			 */
			if (message instanceof String[]) {
				handleTelnetRpc(msgevent.getChannel(), (String[]) message);
				/**
				 * http信号
				 */
			} else if (message instanceof HttpRequest) {
				handleHttpQuery(tsdb, msgevent.getChannel(), (HttpRequest) message);
			} else {
				logError(msgevent.getChannel(), "Unexpected message type " + 
                               message.getClass() + ": " + message);
				exceptions_caught.incrementAndGet();
			}
		} catch (Exception e) {
			Object pretty_message = msgevent.getMessage();
			if (pretty_message instanceof String[]) {
				pretty_message = Arrays.toString((String[]) pretty_message);
			}
			logError(msgevent.getChannel(), "Unexpected exception caught" + 
                     " while serving " + pretty_message, e);
			exceptions_caught.incrementAndGet();
		}
	}

在telnet示例代码向tsd服务发送监控信息之后，tsd将接受的信息封装成PutDataPointRpc(implements TelnetRpc, HttpRpc)并实现其excute()方法，并调用TSDB类的addPointInternal()方法向HBase中添加数据。

public void execute(final TSDB tsdb, final HttpQuery query) throws IOException {
		requests.incrementAndGet();

		// only accept POST
		if (query.method() != HttpMethod.POST) {
			throw new BadRequestException(HttpResponseStatus.METHOD_NOT_ALLOWED, 
                       "Method not allowed", "The HTTP method [" + query.method().getName()
					+ "] is not permitted for this endpoint");
		}

		final List<IncomingDataPoint> dps = query.serializer().parsePutV1();
		if (dps.size() < 1) {
			throw new BadRequestException("No datapoints found in content");
		}

		final boolean show_details = query.hasQueryStringParam("details");
		final boolean show_summary = query.hasQueryStringParam("summary");
		final ArrayList<HashMap<String, Object>> details = show_details ? new ArrayList<HashMap<String, Object>>() : null;
		long success = 0;
		long total = 0;

		for (IncomingDataPoint dp : dps) {
			total++;
			try {
				if (dp.getMetric() == null || dp.getMetric().isEmpty()) {
					if (show_details) {
						details.add(this.getHttpDetails("Metric name was empty", dp));
					}
					LOG.warn("Metric name was empty: " + dp);
					continue;
				}
				if (dp.getTimestamp() <= 0) {
					if (show_details) {
						details.add(this.getHttpDetails("Invalid timestamp", dp));
					}
					LOG.warn("Invalid timestamp: " + dp);
					continue;
				}
				if (dp.getValue() == null || dp.getValue().isEmpty()) {
					if (show_details) {
						details.add(this.getHttpDetails("Empty value", dp));
					}
					LOG.warn("Empty value: " + dp);
					continue;
				}
				if (dp.getTags() == null || dp.getTags().size() < 1) {
					if (show_details) {
						details.add(this.getHttpDetails("Missing tags", dp));
					}
					LOG.warn("Missing tags: " + dp);
					continue;
				}
				if (Tags.looksLikeInteger(dp.getValue())) {
					tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), Tags.parseLong(dp.getValue()), dp.getTags());
				} else {
					tsdb.addPoint(dp.getMetric(), dp.getTimestamp(), Float.parseFloat(dp.getValue()), dp.getTags());
				}
				success++;
			} catch (NumberFormatException x) {
				if (show_details) {
					details.add(this.getHttpDetails("Unable to parse value to a number", dp));
				}
				LOG.warn("Unable to parse value to a number: " + dp);
				invalid_values.incrementAndGet();
			} catch (IllegalArgumentException iae) {
				if (show_details) {
					details.add(this.getHttpDetails(iae.getMessage(), dp));
				}
				LOG.warn(iae.getMessage() + ": " + dp);
				illegal_arguments.incrementAndGet();
			} catch (NoSuchUniqueName nsu) {
				if (show_details) {
					details.add(this.getHttpDetails("Unknown metric", dp));
				}
				LOG.warn("Unknown metric: " + dp);
				unknown_metrics.incrementAndGet();
			}
		}

		final long failures = total - success;
		if (!show_summary && !show_details) {
			if (failures > 0) {
				throw new BadRequestException(HttpResponseStatus.BAD_REQUEST, "One or more data points had errors",
						"Please see the TSD logs or append \"details\" to the put request");
			} else {
				query.sendReply(HttpResponseStatus.NO_CONTENT, "".getBytes());
			}
		} else {
			final HashMap<String, Object> summary = new HashMap<String, Object>();
			summary.put("success", success);
			summary.put("failed", failures);
			if (show_details) {
				summary.put("errors", details);
			}

			if (failures > 0) {
				query.sendReply(HttpResponseStatus.BAD_REQUEST, query.serializer().formatPutV1(summary));
			} else {
				query.sendReply(query.serializer().formatPutV1(summary));
			}
		}
	}




private Deferred<Object> addPointInternal(final String metric, final long timestamp, final byte[] value, final Map<String, String> tags,
			final short flags) {
		
		// we only accept unix epoch timestamps in seconds or milliseconds
		if ((timestamp & Const.SECOND_MASK) != 0 && (timestamp < 1000000000000L || timestamp > 9999999999999L)) {
			throw new IllegalArgumentException((timestamp < 0 ? "negative " : "bad") + " timestamp=" + timestamp + " when trying to add value="
					+ Arrays.toString(value) + '/' + flags + " to metric=" + metric + ", tags=" + tags);
		}

		/**
		 * 有效字符验证
		 */
		IncomingDataPoints.checkMetricAndTags(metric, tags);

		class AddPointCB implements Callback<Deferred<Object>, byte[]> {
			
			/**
			 * 此方法调用时会传入行键rowkey
			 */
			public Deferred<Object> call(final byte[] row) {
				final long base_time;
				
				/**
				 * 构造列修饰符
				 */
				final byte[] qualifier = Internal.buildQualifier(timestamp, flags);

				/**
				 * 完善rowKey 将部分时间戳放入rowKey
				 */
				if ((timestamp & Const.SECOND_MASK) != 0) {
					// drop the ms timestamp to seconds to calculate the base
					// timestamp
					base_time = ((timestamp / 1000) - ((timestamp / 1000) % Const.MAX_TIMESPAN));
				} else {
					base_time = (timestamp - (timestamp % Const.MAX_TIMESPAN));
				}

				Bytes.setInt(row, (int) base_time, metrics.width());
				scheduleForCompaction(row, (int) base_time);
				
				/**
				 * 此处执行插入操作
				 */
				final PutRequest point = new PutRequest(table, row, FAMILY, qualifier, value);

				// TODO(tsuna): Add a callback to time the latency of HBase and
				// store the
				// timing in a moving Histogram (once we have a class for this).
				Deferred<Object> result = client.put(point);
				if (!config.enable_realtime_ts() && !config.enable_tsuid_incrementing() && rt_publisher == null) {
					return result;
				}

				final byte[] tsuid = UniqueId.getTSUIDFromKey(row, METRICS_WIDTH, Const.TIMESTAMP_BYTES);
				if (config.enable_tsuid_incrementing() || config.enable_realtime_ts()) {
					TSMeta.incrementAndGetCounter(TSDB.this, tsuid);
				}

				if (rt_publisher != null) {

					/**
					 * Simply logs real time publisher errors when they're
					 * thrown. Without this, exceptions will just disappear
					 * (unless logged by the plugin) since we don't wait for a
					 * result.
					 */
					final class RTError implements Callback<Object, Exception> {
						@Override
						public Object call(final Exception e) throws Exception {
							LOG.error("Exception from Real Time Publisher", e);
							return null;
						}
					}

					rt_publisher.sinkDataPoint(metric, timestamp, value, tags, tsuid, flags).addErrback(new RTError());
				}
				return result;
			}
		}
		
		/**
		 * 先通过metric、tags生成rowKey的结构（byte[3+4+3+3]）
		 * 再通过回调函数完善rowKey，并生成PutRequest将数据保存到tsdb表中
		 */
		return IncomingDataPoints.rowKeyTemplate(this, metric, tags).addCallbackDeferring(new AddPointCB());
	}