Logstash之3 对于logstash加载管道文件的改进

1 为什么Logstash要引入管道文件的支持？

   因为Logstash对数据库，加密信息等进行配置时。需要将数据库密码，加密秘钥等信息配置到配置文件中。在某种程度上引入管道文件，能够对私密信息进行隐藏，以防止黑客等入侵时直接获取相关信息。并且管道文件在读取后，会自动销毁。是一种比较友好的加密信息隐藏方式。

2 管道文件是一种使用有名管道进行进程通信的媒介。

3 Logstash本身对于加载管道文件是不支持的。因此修改Logstash源代码使其能够加载管道文件成为必要。

4 对于配置文件的加载判断，参见logstash-core/lib/logstash/config/source/local.rb

# encoding: utf-8
require "logstash/config/source/base"
require "logstash/config/pipeline_config"
require "uri"

module LogStash module Config module Source
  # A locally defined configuration source
  #
  # Which can aggregate the following config options:
  #  - settings.config_string: "input { stdin {} }"
  #  - settings.config_path: /tmp/logstash/*.conf
  #  - settings.config_path: http://localhost/myconfig.conf
  #
  #  All theses option will create a unique pipeline, generated parts will be
  #  sorted alphabetically. Se `PipelineConfig` class for the sorting algorithm.
  #
  class Local < Base
    class ConfigStringLoader
      INPUT_BLOCK_RE = /input *{/
      OUTPUT_BLOCK_RE = /output *{/
      EMPTY_RE = /^\s*$/

      def self.read(config_string)
        config_parts = [org.logstash.common.SourceWithMetadata.new("string", "config_string", 0, 0, config_string)]

        # Make sure we have an input and at least 1 output
        # if its not the case we will add stdin and stdout
        # this is for backward compatibility reason
        if !INPUT_BLOCK_RE.match(config_string)
          config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", 0, 0, LogStash::Config::Defaults.input)

        end

        # include a default stdout output if no outputs given
        if !OUTPUT_BLOCK_RE.match(config_string)
          config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", 0, 0, LogStash::Config::Defaults.output)
        end

        config_parts
      end
    end

    class ConfigPathLoader
      include LogStash::Util::Loggable

      TEMPORARY_FILE_RE = /~$/
      LOCAL_FILE_URI = /^file:\/\//i

      def initialize(path)
        @path = normalize_path(path)
      end

      def read
        config_parts = []
        encoding_issue_files = []

        if logger.debug?
          logger.debug("Skipping the following files while reading config since they don't match the specified glob pattern", :files => get_unmatched_files)
        end

        get_matched_files.each do |file|
          next unless ::File.file?(file) # skip directory

          logger.debug("Reading config file", :config_file => file)

          if temporary_file?(file)
            logger.warn("NOT reading config file because it is a temp file", :config_file => file)
            next
          end

          config_string = ::File.read(file)
          config_string.force_encoding("UTF-8")

          if config_string.valid_encoding?
            part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
            config_parts << part
          else
            encoding_issue_files << file
          end
        end

        if encoding_issue_files.any?
          raise LogStash::ConfigLoadingError, "The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}"
        end

        if config_parts.empty?
          logger.info("No config files found in path", :path => path)
        end

        config_parts
      end

      def self.read(path)
        ConfigPathLoader.new(path).read
      end

      private
      def normalize_path(path)
        path.gsub!(LOCAL_FILE_URI, "")
        ::File.expand_path(path)
      end

      def get_matched_files
        Dir.glob(path).sort
      end

      def path
        if ::File.directory?(@path)
          ::File.join(@path, "*")
        else
          @path
        end
      end

      def get_unmatched_files
        # transform "/var/lib/*.conf" => /var/lib/*
        t = ::File.split(@path)
        all_files = Dir.glob(::File.join(t.first, "*")).sort
        all_files - get_matched_files
      end

      def temporary_file?(filepath)
        filepath.match(TEMPORARY_FILE_RE)
      end
    end

    class ConfigRemoteLoader
      def self.read(uri)
        uri = URI.parse(uri)

        Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == "https") do |http|
          request = Net::HTTP::Get.new(uri.path)
          response = http.request(request)

          # since we have fetching config we wont follow any redirection.
          case response.code.to_i
          when 200
            [org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, 0, 0, response.body)]
          when 302
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "We don't follow redirection for remote configuration")
          when 404
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "File not found")
          when 403
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "Permission denied")
          when 500
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "500 error on remote host")
          else
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "code: #{response.code}, message: #{response.class.to_s}")
          end
        end
      end
    end

    PIPELINE_ID = LogStash::SETTINGS.get("pipeline.id").to_sym
    HTTP_RE = /^http(s)?/

    def pipeline_configs
      if config_conflict?
        raise ConfigurationError, @conflict_messages.join(", ")
      end
      local_pipeline_configs
    end

    def match?
      # see basic settings predicates and getters defined in the base class
      (config_string? || config_path?) && !(modules_cli? || modules?) && !automatic_reload_with_config_string?
    end

    def config_conflict?
      @conflict_messages.clear

      # Check if configuration auto-reload is used that -f is specified
      if automatic_reload_with_config_string?
        @conflict_messages << I18n.t("logstash.runner.reload-with-config-string")
      end
      # Check if both -f and -e are present
      if config_string? && config_path?
        @conflict_messages << I18n.t("logstash.runner.config-string-path-exclusive")
      end

      @conflict_messages.any?
    end

    private

    def local_pipeline_configs
      config_parts = if config_string?
        ConfigStringLoader.read(config_string)
      elsif local_config?
        ConfigPathLoader.read(config_path)
      elsif remote_config?
        ConfigRemoteLoader.read(config_path)
      else
        []
      end

      return [] if config_parts.empty?

      [PipelineConfig.new(self.class, @settings.get("pipeline.id").to_sym, config_parts, @settings)]
    end

    def automatic_reload_with_config_string?
      config_reload_automatic? && !config_path? && config_string?
    end

    def local_config?
      return false unless config_path?

      begin
        uri = URI.parse(config_path)
        uri.scheme == "file" || uri.scheme.nil?
      rescue URI::InvalidURIError
        # fallback for windows.
        # if the parsing of the file failed we assume we can reach it locally.
        # some relative path on windows arent parsed correctly (.\logstash.conf)
        true
      end
    end

    def remote_config?
      return false unless config_path?

      begin
        uri = URI.parse(config_path)
        uri.scheme =~ HTTP_RE
      rescue URI::InvalidURIError
        false
      end
    end
  end
end end end

5 引入管道文件的支持后：

# encoding: utf-8
require "logstash/config/source/base"
require "logstash/config/pipeline_config"
require "uri"

module LogStash module Config module Source
  # A locally defined configuration source
  #
  # Which can aggregate the following config options:
  #  - settings.config_string: "input { stdin {} }"
  #  - settings.config_path: /tmp/logstash/*.conf
  #  - settings.config_path: http://localhost/myconfig.conf
  #
  #  All theses option will create a unique pipeline, generated parts will be
  #  sorted alphabetically. Se `PipelineConfig` class for the sorting algorithm.
  #
  class Local < Base
    class ConfigStringLoader
      INPUT_BLOCK_RE = /input *{/
      OUTPUT_BLOCK_RE = /output *{/
      EMPTY_RE = /^\s*$/

      def self.read(config_string)
        config_parts = [org.logstash.common.SourceWithMetadata.new("string", "config_string", 0, 0, config_string)]

        # Make sure we have an input and at least 1 output
        # if its not the case we will add stdin and stdout
        # this is for backward compatibility reason
        if !INPUT_BLOCK_RE.match(config_string)
          config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default input", 0, 0, LogStash::Config::Defaults.input)

        end

        # include a default stdout output if no outputs given
        if !OUTPUT_BLOCK_RE.match(config_string)
          config_parts << org.logstash.common.SourceWithMetadata.new(self.class.name, "default output", 0, 0, LogStash::Config::Defaults.output)
        end

        config_parts
      end
    end

    class ConfigPathLoader
      include LogStash::Util::Loggable

      TEMPORARY_FILE_RE = /~$/
      LOCAL_FILE_URI = /^file:\/\//i

      def initialize(path)
        @path = normalize_path(path)
      end

      def read
        config_parts = []
        encoding_issue_files = []

        if logger.debug?
          logger.debug("Skipping the following files while reading config since they don't match the specified glob pattern", :files => get_unmatched_files)
        end

        get_matched_files.each do |file|
          next unless ::File.file?(file) or ::File.pipe?(file) # skip directory

          logger.debug("Reading config file", :config_file => file)

          if temporary_file?(file)
            logger.warn("NOT reading config file because it is a temp file", :config_file => file)
            next
          end

          config_string = ::File.read(file)
          config_string.force_encoding("UTF-8")

          if config_string.valid_encoding?
            part = org.logstash.common.SourceWithMetadata.new("file", file, 0, 0, config_string)
            config_parts << part
          else
            encoding_issue_files << file
          end
        end

        if encoding_issue_files.any?
          raise LogStash::ConfigLoadingError, "The following config files contains non-ascii characters but are not UTF-8 encoded #{encoding_issue_files}"
        end

        if config_parts.empty?
          logger.info("No config files found in path", :path => path)
        end

        config_parts
      end

      def self.read(path)
        ConfigPathLoader.new(path).read
      end

      private
      def normalize_path(path)
        path.gsub!(LOCAL_FILE_URI, "")
        ::File.expand_path(path)
      end

      def get_matched_files
        Dir.glob(path).sort
      end

      def path
        if ::File.directory?(@path)
          ::File.join(@path, "*")
        else
          @path
        end
      end

      def get_unmatched_files
        # transform "/var/lib/*.conf" => /var/lib/*
        t = ::File.split(@path)
        all_files = Dir.glob(::File.join(t.first, "*")).sort
        all_files - get_matched_files
      end

      def temporary_file?(filepath)
        filepath.match(TEMPORARY_FILE_RE)
      end
    end

    class ConfigRemoteLoader
      def self.read(uri)
        uri = URI.parse(uri)

        Net::HTTP.start(uri.host, uri.port, :use_ssl => uri.scheme == "https") do |http|
          request = Net::HTTP::Get.new(uri.path)
          response = http.request(request)

          # since we have fetching config we wont follow any redirection.
          case response.code.to_i
          when 200
            [org.logstash.common.SourceWithMetadata.new(uri.scheme, uri.to_s, 0, 0, response.body)]
          when 302
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "We don't follow redirection for remote configuration")
          when 404
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "File not found")
          when 403
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "Permission denied")
          when 500
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "500 error on remote host")
          else
            raise LogStash::ConfigLoadingError, I18n.t("logstash.runner.configuration.fetch-failed", :path => uri.to_s, :message => "code: #{response.code}, message: #{response.class.to_s}")
          end
        end
      end
    end

    PIPELINE_ID = LogStash::SETTINGS.get("pipeline.id").to_sym
    HTTP_RE = /^http(s)?/

    def pipeline_configs
      if config_conflict?
        raise ConfigurationError, @conflict_messages.join(", ")
      end
      local_pipeline_configs
    end

    def match?
      # see basic settings predicates and getters defined in the base class
      (config_string? || config_path?) && !(modules_cli? || modules?) && !automatic_reload_with_config_string?
    end

    def config_conflict?
      @conflict_messages.clear

      # Check if configuration auto-reload is used that -f is specified
      if automatic_reload_with_config_string?
        @conflict_messages << I18n.t("logstash.runner.reload-with-config-string")
      end
      # Check if both -f and -e are present
      if config_string? && config_path?
        @conflict_messages << I18n.t("logstash.runner.config-string-path-exclusive")
      end

      @conflict_messages.any?
    end

    private

    def local_pipeline_configs
      config_parts = if config_string?
        ConfigStringLoader.read(config_string)
      elsif local_config?
        ConfigPathLoader.read(config_path)
      elsif remote_config?
        ConfigRemoteLoader.read(config_path)
      else
        []
      end

      return [] if config_parts.empty?

      [PipelineConfig.new(self.class, @settings.get("pipeline.id").to_sym, config_parts, @settings)]
    end

    def automatic_reload_with_config_string?
      config_reload_automatic? && !config_path? && config_string?
    end

    def local_config?
      return false unless config_path?

      begin
        uri = URI.parse(config_path)
        uri.scheme == "file" || uri.scheme.nil?
      rescue URI::InvalidURIError
        # fallback for windows.
        # if the parsing of the file failed we assume we can reach it locally.
        # some relative path on windows arent parsed correctly (.\logstash.conf)
        true
      end
    end

    def remote_config?
      return false unless config_path?

      begin
        uri = URI.parse(config_path)
        uri.scheme =~ HTTP_RE
      rescue URI::InvalidURIError
        false
      end
    end
  end
end end end

6 Github代码参见https://github.com/AndrewPanB/logstash。