class Fluent::Grok

Constants

PATTERN_RE

Much of the Grok implementation is based on Jordan Sissel's jls-grok See github.com/jordansissel/ruby-grok/blob/master/lib/grok-pure.rb

Attributes

multiline_start_regexp[R]
parsers[R]

Public Class Methods

new(plugin, conf) click to toggle source
# File lib/fluent/plugin/grok.rb, line 22
def initialize(plugin, conf)
  @pattern_map = {}
  @parsers = []
  @multiline_mode = false
  @conf = conf
  if plugin.respond_to?(:firstline?)
    @multiline_mode = true
  end
  if @conf['multiline_start_regexp']
    @multiline_start_regexp = Regexp.compile(@conf['multiline_start_regexp'][1..-2])
  end
end

Public Instance Methods

add_patterns_from_file(path) click to toggle source
# File lib/fluent/plugin/grok.rb, line 35
def add_patterns_from_file(path)
  File.new(path).each_line do |line|
    next if line[0] == '#' || /^$/ =~ line
    name, pat = line.chomp.split(/\s+/, 2)
    @pattern_map[name] = pat
  end
end
setup() click to toggle source
# File lib/fluent/plugin/grok.rb, line 43
def setup
  if @conf['grok_pattern']
    @parsers << expand_pattern_expression(@conf['grok_pattern'], @conf)
  else
    grok_confs = @conf.elements.select {|e| e.name == 'grok'}
    grok_confs.each do |grok_conf|
      @parsers << expand_pattern_expression(grok_conf['pattern'], grok_conf)
    end
  end
end

Private Instance Methods

expand_pattern(pattern) click to toggle source
# File lib/fluent/plugin/grok.rb, line 73
def expand_pattern(pattern)
  # It's okay to modify in place. no need to expand it more than once.
  type_map = {}
  while true
    m = PATTERN_RE.match(pattern)
    break unless m
    curr_pattern = @pattern_map[m["pattern"]]
    raise GrokPatternNotFoundError, "grok pattern not found: #{pattern}" unless curr_pattern
    if m["subname"]
      replacement_pattern = "(?<#{m["subname"]}>#{curr_pattern})"
      type_map[m["subname"]] = m["type"] || "string"
    else
      replacement_pattern = curr_pattern
    end
    pattern.sub!(m[0]) do |s| replacement_pattern end
  end

  [pattern, type_map]
end
expand_pattern_expression(grok_pattern, conf) click to toggle source
# File lib/fluent/plugin/grok.rb, line 56
def expand_pattern_expression(grok_pattern, conf)
  regexp, types = expand_pattern(grok_pattern)
  $log.info "Expanded the pattern #{conf['grok_pattern']} into #{regexp}"
  options = nil
  if @multiline_mode
    options = Regexp::MULTILINE
  end
  unless types.empty?
    conf["types"] = types.map{|subname,type| "#{subname}:#{type}" }.join(",")
  end
  TextParser::RegexpParser.new(Regexp.new(regexp, options), conf)
rescue GrokPatternNotFoundError => e
  raise e
rescue => e
  $log.error e.backtrace.join("\n")
end