Logstash cheat sheet: Difference between revisions

Line 126: Line 126:


==== Filebeat 設定 ====
==== Filebeat 設定 ====
'''filebeat.yml 例 (1.x -> 5.0 仮)'''
'''filebeat.yml 例 (6.0 仮)'''
: 6.0 は type が使えないので log_type という名前のカスタムフィールドに変更する。
: 6.0 は type が使えないので log_type という名前のカスタムフィールドに変更する。
::logstash 使うので filebeat modules は使わない。
:::modules 勉強しないといけない。
<syntaxhighlight lang="yaml" enclose="div">
<syntaxhighlight lang="yaml" enclose="div">
############################# Filebeat ######################################
######################## Filebeat Configuration ############################
filebeat:
  # List of prospectors to fetch data.
  prospectors:
    # Each - is a prospector. Below are the prospector specific configurations
    -
      paths:
        - /var/log/syslog
        - /var/log/mail.log
        - /var/log/auth.log
      input_type: log
      document_type: syslog
      fields_under_root: true


    -
#=========================== Filebeat prospectors =============================
      paths:
        - /var/log/apache2/access.log
      input_type: log
      document_type: apache
      fields_under_root: true


    -
filebeat.prospectors:
      paths:
        - /var/log/apache2/other_vhosts_access.log
      input_type: log
      document_type: apache-other-vhost
      fields_under_root: true


    -
- type: log
      paths:
  paths:
        - /var/log/apache2/error.log
    - /var/log/syslog
      input_type: log
    - /var/log/mail.log
      document_type: apache-error
    - /var/log/auth.log
      fields_under_root: true
  fields:
    log_type: syslog
  fields_under_root: true


    -
- type: log
      paths:
  paths:
        - /var/log/dpkg.log
    /var/log/apache2/access.log
      input_type: log
  fields:
      document_type: dpkg
    log_type: apache
      fields_under_root: true
  fields_under_root: true


   # General filebeat configuration options
- type: log
   #
   paths:
  # Event count spool threshold - forces network flush if exceeded
    /var/log/apache2/other_vhosts_access.log
   #spool_size: 2048
   fields:
    log_type: apache-other-vhost
   fields_under_root: true


   # Defines how often the spooler is flushed. After idle_timeout the spooler is
- type: log
   # Flush even though spool_size is not reached.
   paths:
   #idle_timeout: 5s
    /var/log/apache2/error.log
   fields:
    log_type: apache-error
   fields_under_root: true


   # Name of the registry file. Per default it is put in the current working
- type: log
   # directory. In case the working directory is changed after when running
   paths:
  # filebeat again, indexing starts from the beginning again.
    /var/log/varnish/varnishncsa.log
   registry_file: /var/lib/filebeat/registry
   fields:
    log_type: varnish
   fields_under_root: true


  # Full Path to directory with additional prospector configuration files. Each file must end with .yml
- type: log
   # These config files must have the full filebeat config part inside, but only
   paths:
  # the prospector part is processed. All global options like spool_size are ignored.
    /var/log/dpkg.log
   # The config_dir MUST point to a different directory then where the main filebeat config file is in.
   fields:
   #config_dir:
    log_type: dpkg
   fields_under_root: true


- type: log
  paths:
    /var/log/fail2ban.log
  fields:
    log_type: fail2ban
  fields_under_root: true


###############################################################################
#========================= Filebeat global options ============================
############################# Libbeat Config ##################################
# Base config file used by all other beats for using libbeat features


############################# Output ##########################################
filebeat.registry_file: /var/lib/filebeat/registry


# Configure what outputs to use when sending the data collected by the beat.
#================================ Outputs ======================================
# Multiple outputs may be used.
output:
  ### Logstash as output
  logstash:
    # The Logstash hosts
    hosts: ["localhost:5044"]


    # Number of workers per Logstash host.
#----------------------------- Logstash output ---------------------------------
    #worker: 1
output.logstash:
  # The Logstash hosts
  hosts: ["localhost:5044"]


    # Set gzip compression level.
  # Optional SSL. By default is off.
    #compression_level: 3
  # List of root certificates for HTTPS server verifications
  ssl.certificate_authorities: ["/etc/logstash/logstash-forwarder.crt"]


    # Optional load balance the events between the Logstash hosts
  # Certificate for TLS client authentication
    #loadbalance: true
  #ssl.certificate: "/etc/logstash/logstash-forwarder.crt"


    # Optional index name. The default index name depends on the each beat.
  # Client Certificate Key
    # For Packetbeat, the default is set to packetbeat, for Topbeat
  #ssl.key: "/etc/logstash/logstash-forwarder.key"
    # top topbeat and for Filebeat to filebeat.
    #index: filebeat


    # Optional TLS. By default is off.
  # Configure SSL verification mode. If `none` is configured, all server hosts
    ssl:
  # and certificates will be accepted. In this mode, SSL based connections are
      # List of root certificates for HTTPS server verifications
  # susceptible to man-in-the-middle attacks. Use only for testing. Default is
      certificate_authorities: ["/etc/logstash/logstash-forwarder.crt"]
  # `full`.
  #ssl.verification_mode: full
  ssl.verification_mode: none


      # Certificate for TLS client authentication
#================================ Logging ======================================
      #certificate: "/etc/logstash/logstash-forwarder.crt"
# There are three options for the log output: syslog, file, stderr.
# Under Windows systems, the log files are per default sent to the file output,
# under all other system per default to syslog.


      # Client Certificate Key
# Sets log level. The default log level is info.
      #key: "/etc/logstash/logstash-forwarder.key"
# Available log levels are: critical, error, warning, info, debug
#logging.level: info
logging.level: info


      #verification_mode: none
# Enable debug output for selected components. To enable all selectors use ["*"]
      verification_mode: none
# Other available selectors are "beat", "publish", "service"
# Multiple selectors can be chained.
#logging.selectors: [ ]


      # Configure cipher suites to be used for TLS connections
# Send all logging output to syslog. The default is false.
      #cipher_suites: []
#logging.to_syslog: true
 
logging.to_syslog: false
      # Configure curve types for ECDHE based cipher suites
      #curve_types: []
 
############################# Logging #########################################
 
# There are three options for the log ouput: syslog, file, stderr.
# Under Windos systems, the log files are per default sent to the file output,
# under all other system per default to syslog.
logging:


  # Send all logging output to syslog. On Windows default is false, otherwise
# If enabled, filebeat periodically logs its internal metrics that have changed
  # default is true.
# in the last period. For each metric that changed, the delta from the value at
  #to_syslog: true
# the beginning of the period is logged. Also, the total values for
  to_syslog: false
# all non-zero internal metrics are logged on shutdown. The default is true.
#logging.metrics.enabled: true


  # Write all logging output to files. Beats automatically rotate files if rotateeverybytes
# The period after which to log the internal metrics. The default is 30s.
  # limit is reached.
#logging.metrics.period: 30s
  #to_files: false
  to_files: true


  # To enable logging to files, to_files option has to be set to true
# Logging to rotating files files. Set logging.to_files to false to disable logging to
  files:
# files.
    # The directory where the log files will written to.
logging.to_files: true
    #path: /var/log/mybeat
logging.files:
    path: /var/log/filebeat
  # Configure the path where the logs are written. The default is the logs directory
  # under the home path (the binary location).
  #path: /var/log/filebeat
  path: /var/log/filebeat


    # The name of the files where the logs are written to.
  # The name of the files where the logs are written to.
    #name: mybeat
  #name: filebeat
    name: filebeat.log
  name: filebeat.log


    # Configure log file size limit. If limit is reached, log file will be
  # Configure log file size limit. If limit is reached, log file will be
    # automatically rotated
  # automatically rotated
    rotateeverybytes: 10485760 # = 10MB
  #rotateeverybytes: 10485760 # = 10MB
  rotateeverybytes: 10485760 # = 10MB


    # Number of rotated log files to keep. Oldest files will be deleted first.
  # Number of rotated log files to keep. Oldest files will be deleted first.
    #keepfiles: 7
  #keepfiles: 7
    keepfiles: 7
  keepfiles: 7


   # Enable debug output for selected components. To enable all selectors use ["*"]
   # The permissions mask to apply when rotating log files. The default value is 0600.
  # Other available selectors are beat, publish, service
   # Must be a valid Unix-style file permissions mask expressed in octal notation.
   # Multiple selectors can be chained.
   #permissions: 0600
   #selectors: [ ]


  # Sets log level. The default log level is error.
# Set to true to log messages in json format.
  # Available log levels are: critical, error, warning, info, debug
#logging.json: false
  #level: error
  level: info
</syntaxhighlight>
</syntaxhighlight>