NGINX Logs

Nginx in docker:

log_format main escape=json '{'
    '"time_local":"$time_local",'
    '"remote_addr":"$remote_addr",'
     '"request":"$request",'
     '"status":"$status",'
     '"body_bytes_sent":"$body_bytes_sent",'
     '"http_referer":"$http_referer",'
     '"http_user_agent":"$http_user_agent",'
     '"http_x_forwarded_for":"$http_x_forwarded_for",'
     '"request_time":"$request_time"'
  '}';
  access_log /dev/stdout main;

Docker compose:

  myweb:
    image: 'my_nginx'
    labels:
      - "servicename=myweb"
  vector:
    image: 'timberio/vector:0.34.0-alpine'
    ports:
      - 9598:9598
    volumes:
      - ./vector034.yaml:/etc/vector/vector.yaml:ro
      - /var/run/docker.sock:/var/run/docker.sock
    command: ["-c", "/etc/vector/vector.yaml"]

vector034.yaml:

sources:
  nginx_logs:
    type: "docker_logs"
    include_labels:
      - "servicename=myweb"
transforms:
parse_logs:
type: "remap"
inputs: ["nginx_logs"]
source: |
if !exists(.message) {
abort
}
cleaned, err = strip_whitespace(.message)
if err != null {
# log("Strip whitespace error: " + err, level: "error")
cleaned = .message
}
  parsed, err = parse_json(cleaned)
  if err != null {
    # log("Parsing error: " + err, level: "error")
    abort
  }
  
  .time_local = parsed.time_local
  .remote_addr = parsed.remote_addr
  .request = parsed.request
  .status = parsed.status
  .body_bytes_sent = parsed.body_bytes_sent
  .http_referer = parsed.http_referer
  .http_user_agent = parsed.http_user_agent
  .http_x_forwarded_for = parsed.http_x_forwarded_for
  .request_time = parsed.request_time
  
  request_parts, err = parse_regex(.request, r'^(?P<method>\S+) (?P<uri>[^\s]+) (?P<protocol>[^"]+)$')
  if err != null {
    log("Request parsing error: " + err, level: "error")
    .request_method = "UNKNOWN"
    .request_uri = "/"
    .request_protocol = "HTTP/1.1"
  } else {
    .request_method = request_parts.method
    .request_uri = request_parts.uri
    .request_protocol = request_parts.protocol
  }
  
  .body_bytes_sent, err = if .body_bytes_sent == "" || .body_bytes_sent == null { 0 } else { to_int(.body_bytes_sent) }
  .request_method = if .request_method == "" || .request_method == null { "UNKNOWN" } else { .request_method }
  .status = if .status == "" || .status == null { "000" } else { .status }
  .request_uri = if .request_uri == "" || .request_uri == null { "/" } else { .request_uri }
  .request_time, err = if .request_time == "" || .request_time == null { 0.0 } else { to_float(.request_time) }
  
  # log("Parsed log: " + encode_json(.), level: "debug")
extract_metrics:
type: "log_to_metric"
inputs: ["parse_logs"]
metrics:
- type: "counter"
name: "nginx_http_response_count_total"
description: "Total HTTP requests"
field: "status"
tags:
method: "{{ request_method }}"
status: "{{ status }}"
path: "{{ request_uri }}"
- type: "counter"
name: "nginx_http_response_size_bytes"
description: "Total bytes sent"
field: "body_bytes_sent"
tags:
method: "{{ request_method }}"
status: "{{ status }}"
- type: "histogram"
name: "nginx_http_response_time_seconds"
description: "Request processing time in seconds"
field: "request_time"
tags:
method: "{{ request_method }}"
status: "{{ status }}"
sinks:
prometheus_exporter:
type: "prometheus_exporter"
inputs: ["extract_metrics"]
address: "0.0.0.0:9598"