Squid Logs Analyzer

Dashboard

Squid Logs Analyzer Dashboard by Omar Padrón Capote (www.sysadminsdecuba.com)
Last updated: a day ago

Downloads: 24

    This dashboard connected to elasticsearch shows the analysis of the squid logs filtered by logstash and stored in elasticsearch. enter image description here

    Configuration:

    Inputs logstash in /etc/logstash/conf.d:

    file filebeat-input.conf

    input {
      beats {
        port => 5443
        type => syslog
        ssl => true
        ssl_certificate => "/etc/logstash/logstash.crt"
        ssl_key => "/etc/logstash/logstash.key.pem"
      }
    }
    }
    

    filters in /etc/logstash/conf.d/

    file squid.conf

    filter {
    if [type] == "squid3" {
         grok {
           break_on_match => false
           # add/change pattern
           patterns_dir => [ "/etc/logstash/patterns/" ]
           match => [ "message" , "%{INT:timestamp:int}.%{INT:timestamp_ms}\s+%{INT:response_time} %{IPORHOST:src_ip} %{WORD:squid_request_s$
           # domain-name from url
           match => [ "message" , "%{INT}.%{INT}\s+%{INT} %{IPORHOST} %{WORD}/%{INT} %{INT} %{WORD} (%{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?($
        }
         date {
            # convert unixtime  to timestamp
            match => [ "timestamp", "UNIX" ]
            }
    
      if [server_ip] {
        geoip {
          source => "server_ip"
          target => "geoip"
          database => "/etc/logstash/geoip/GeoLite2-City.mmdb"
          fields => [ "country_code2", "country_name" ]
          fields => [ "latitude", "longitude" ]
          add_field => [ "[geoip][location]", "%{[geoip][longitude]}" ]
          add_field => [ "[geoip][location]", "%{[geoip][latitude]}" ]
          add_tag => "geoip"
        }
         mutate {
            convert => [ "[geoip][location]", "float"]
         }
      }
    }
    }
    

    Output logstash in /etc/logstash/conf.d/

    file output-elasticsearch.conf

    output {
    if [host] == "squidserver" {
       elasticsearch {
        hosts => ["elastic.domain.com:9200"]
        manage_template => false
        index => "squid-%{[@metadata][beat]}-%{+YYYY.MM.dd}"
        document_type => "%{[@metadata][type]}"
      }
    }
    }
    

    Note: squidserver is the hostname of squid server.

    Squid file pattern in /etc/logstash/patterns

    file squid

    URIPARAM \?[ÄÖÜöäüßA-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]*
    # add new pattern
    MYTEST  [A-Za-z0-9]*
    #########
    

    And geoip databases in /etc/logstash/geoip

    #wget -t0 -c http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz
    #tar -xf GeoLite2-City.tar.gz
    #mkdir  /etc/logstash/geoip
    #mv GeoLite2-City_(day)/GeoLite2-City.mmdb /etc/logstash/geoip
    

    Collector Configuration Details

    filebeat.prospectors:
    - input_type: log
      document_type: squid3
      paths:
        - /var/log/squid3/access.log
    
    
    output.logstash:
      # The Logstash hosts
      hosts: ["logstash.domain.com:5443"]
    
      # Optional SSL. By default is off.
      # List of root certificates for HTTPS server verifications
      bulk_max_size: 2048
      ssl.certificate_authorities: ["/etc/filebeat/logstash.crt"]
      template.name: "filebeat"
      template.path: "filebeat.template.json"
      template.overwrite: false
      # Certificate for SSL client authentication
      #ssl.certificate: "/etc/pki/client/cert.pem"
    
      # Client Certificate Key
      #ssl.key: "/etc/pki/client/cert.key"