Fluentd nginx error 로그의 패턴이 일정하지 않을 때 처리

2025. 2. 4. 20:38EFK

<source>
  @type tail
  path /var/log/nginx/error.log
  pos_file /var/log/fluentd/nginx-error.log.pos
  tag nginx.error
  read_from_head true

  <parse>
    @type regexp
    expression /^(?<time>\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}) \[(?<level>[^\]]+)\] (?<pid>\d+#\d+): (?<message>.*)$/
    time_key time
    time_format %Y/%m/%d %H:%M:%S
  </parse>
</source>

# Step 2: If Level is "error", extract structured fields
<filter nginx.error>
  @type record_transformer
  enable_ruby true

  <record>
    request_id ${record["level"] == "error" && record["message"] =~ /\*(\d+)/ ? $1 : nil}
    file_path  ${record["level"] == "error" && record["message"] =~ /open\(\) "([^"]+)"/ ? $1 : nil}
    error_code ${record["level"] == "error" && record["message"] =~ /failed \((\d+)/ ? $1 : nil}
    error_message ${record["level"] == "error" && record["message"] =~ /failed \(\d+: ([^)]+)/ ? $1 : nil}
    client_ip ${record["level"] == "error" && record["message"] =~ /client: ([\d\.]+)/ ? $1 : nil}
    server_name ${record["level"] == "error" && record["message"] =~ /server: ([^,]+)/ ? $1 : nil}
    http_request ${record["level"] == "error" && record["message"] =~ /request: "([^"]+)"/ ? $1 : nil}
    host ${record["level"] == "error" && record["message"] =~ /host: "([^"]+)"/ ? $1 : nil}
  </record>
</filter>