- log_format log_json '{"@timestamp":"$time_iso8601",'
- '"server_addr":"$server_addr",'
- '"server_name":"$server_name",'
- '"server_port":"$server_port",'
- '"server_protocol":"$server_protocol",'
- '"client_ip":"$remote_addr",'
- '"client_user":"$remote_user",'
- '"status":"$status",'
- '"request_method": "$request_method",'
- '"request_length":"$request_length",'
- '"request_time":"$request_time",'
- '"request_url":"$request_uri",'
- '"request_line":"$request",'
- '"send_client_size":"$bytes_sent",'
- '"send_client_body_size":"$body_bytes_sent",'
- '"proxy_protocol_addr":"$proxy_protocol_addr",'
- '"proxy_add_x_forward":"$proxy_add_x_forwarded_for",'
- '"proxy_port":"$proxy_port",'
- '"proxy_host":"$proxy_host",'
- '"upstream_host":"$upstream_addr",'
- '"upstream_status":"$upstream_status",'
- '"upstream_cache_status":"$upstream_cache_status",'
- '"upstream_connect_time":"$upstream_connect_time",'
- '"upstream_response_time":"$upstream_response_time",'
- '"upstream_header_time":"$upstream_header_time",'
- '"upstream_cookie_name":"$upstream_cookie_name",'
- '"upstream_response_length":"$upstream_response_length",'
- '"upstream_bytes_received":"$upstream_bytes_received",'
- '"upstream_bytes_sent":"$upstream_bytes_sent",'
- '"http_host":"$host",'
- '"http_cookie":"$http_cooke",'
- '"http_user_agent":"$http_user_agent",'
- '"http_origin":"$http_origin",'
- '"http_upgrade":"$http_upgrade",'
- '"http_referer":"$http_referer",'
- '"http_x_forward":"$http_x_forwarded_for",'
- '"http_x_forwarded_proto":"$http_x_forwarded_proto",'
- '"https":"$https",'
- '"http_scheme":"$scheme",'
- '"invalid_referer":"$invalid_referer",'
- '"gzip_ratio":"$gzip_ratio",'
- '"realpath_root":"$realpath_root",'
- '"document_root":"$document_root",'
- '"is_args":"$is_args",'
- '"args":"$args",'
- '"connection_requests":"$connection_requests",'
- '"connection_number":"$connection",'
- '"ssl_protocol":"$ssl_protocol",'
- '"ssl_cipher":"$ssl_cipher"}';
-
- access_log logs/access_json.log log_json;
- vim /home/wwq/elk/filebeat-8.13.4/filebeat.yml
- ###################### Filebeat Configuration Example #########################
- ?
- # This file is an example configuration file highlighting only the most common
- # options. The filebeat.reference.yml file from the same directory contains all the
- # supported options with more comments. You can use it as a reference.
- #
- # You can find the full configuration reference here:
- # https://www.elastic.co/guide/en/beats/filebeat/index.html
- ?
- # For more available modules and options, please see the filebeat.reference.yml sample
- # configuration file.
- ?
- # ============================== Filebeat inputs ===============================
- ?
- filebeat.inputs:
- ?
- # Each - is an input. Most options can be set at the input level, so
- # you can use different inputs for various configurations.
- # Below are the input specific configurations.
- ?
- # filestream is an input for collecting log messages from files.
- - type: filestream
- ?
- # Unique ID among all inputs, an ID is required.
- id: my-filestream-id
- ?
- # Change to true to enable this input configuration.
- enabled: false
- ?
- # Paths that should be crawled and fetched. Glob based paths.
- paths:
- - /var/log/*.log
- #- c:\programdata\elasticsearch\logs\*
- ?
- # Exclude lines. A list of regular expressions to match. It drops the lines that are
- # matching any regular expression from the list.
- # Line filtering happens after the parsers pipeline. If you would like to filter lines
- # before parsers, use include_message parser.
- #exclude_lines: ['^DBG']
- ?
- # Include lines. A list of regular expressions to match. It exports the lines that are
- # matching any regular expression from the list.
- # Line filtering happens after the parsers pipeline. If you would like to filter lines
- # before parsers, use include_message parser.
- #include_lines: ['^ERR', '^WARN']
- ?
- # Exclude files. A list of regular expressions to match. Filebeat drops the files that
- # are matching any regular expression from the list. By default, no files are dropped.
- #prospector.scanner.exclude_files: ['.gz$']
- ?
- # Optional additional fields. These fields can be freely picked
- # to add additional information to the crawled log files for filtering
- #fields:
- # level: debug
- # review: 1
- ?
- # ============================== Filebeat modules ==============================
- ?
- filebeat.config.modules:
- # Glob pattern for configuration loading
- path: ${path.config}/modules.d/*.yml
- ?
- # Set to true to enable config reloading
- reload.enabled: false
- ?
- # Period on which files under path should be checked for changes
- #reload.period: 10s
- ?
- # ======================= Elasticsearch template setting =======================
- ?
- setup.template.settings:
- index.number_of_shards: 1
- #index.codec: best_compression
- #_source.enabled: false
- ?
- ?
- # ================================== General ===================================
- ?
- # The name of the shipper that publishes the network data. It can be used to group
- # all the transactions sent by a single shipper in the web interface.
- #name:
- ?
- # The tags of the shipper are included in their own field with each
- # transaction published.
- #tags: ["service-X", "web-tier"]
- ?
- # Optional fields that you can specify to add additional information to the
- # output.
- #fields:
- # env: staging
- ?
- # ================================= Dashboards =================================
- # These settings control loading the sample dashboards to the Kibana index. Loading
- # the dashboards is disabled by default and can be enabled either by setting the
- # options here or by using the `setup` command.
- #setup.dashboards.enabled: false
- ?
- # The URL from where to download the dashboards archive. By default this URL
- # has a value which is computed based on the Beat name and version. For released
- # versions, this URL points to the dashboard archive on the artifacts.elastic.co
- # website.
- #setup.dashboards.url:
- ?
- # =================================== Kibana ===================================
- ?
- # Starting with Beats version 6.0.0, the dashboards are loaded via the Kibana API.
- # This requires a Kibana endpoint configuration.
- setup.kibana:
- ?
- ?
- # ------------------------------ Logstash Output -------------------------------
- #output.logstash:
- # The Logstash hosts
- #hosts: ["localhost:5044"]
- ?
- # Optional SSL. By default is off.
- # List of root certificates for HTTPS server verifications
- #ssl.certificate_authorities: ["/etc/pki/root/ca.pem"]
- ?
- # Certificate for SSL client authentication
- #ssl.certificate: "/etc/pki/client/cert.pem"
- ?
- # Client Certificate Key
- #ssl.key: "/etc/pki/client/cert.key"
- filebeat.inputs:
- - type: log
- enabled: true
- paths:
- #nginx日志目录
- - /home/wwq/nginx-1.25.5/logs/access_json.log
- fields:
- logcategory: nginx
- json:
- keys_under_root: true
- overwrite_keys: true
- message_key: "message"
- add_error_key: true
- ?
- output.logstash:
- hosts: ["192.168.1.200:5044"]
- #
- # ================================= Processors =================================
- processors:
- - add_host_metadata:
- when.not.contains.tags: forwarded
- - add_cloud_metadata: ~
- - add_docker_metadata: ~
- - add_kubernetes_metadata: ~
- ?
- # ================================== Logging ===================================
- ?
- # Sets log level. The default log level is info.
- # Available log levels are: error, warning, info, debug
- #logging.level: debug
- ?
- # At debug level, you can selectively enable logging only for some components.
- # To enable all selectors use ["*"]. Examples of other selectors are "beat",
- # "publisher", "service".
- #logging.selectors: ["*"]
- ?
- # ============================= X-Pack Monitoring ==============================
- # Filebeat can export internal metrics to a central Elasticsearch monitoring
- # cluster. This requires xpack monitoring to be enabled in Elasticsearch. The
- # reporting is disabled by default.
- ?
- # Set to true to enable the monitoring reporter.
- #monitoring.enabled: false
- ?
- # Sets the UUID of the Elasticsearch cluster under which monitoring data for this
- # Filebeat instance will appear in the Stack Monitoring UI. If output.elasticsearch
- # is enabled, the UUID is derived from the Elasticsearch cluster referenced by output.elasticsearch.
- #monitoring.cluster_uuid:
- ?
- # Uncomment to send the metrics to Elasticsearch. Most settings from the
- # Elasticsearch output are accepted here as well.
- # Note that the settings should point to your Elasticsearch *monitoring* cluster.
- # Any setting that is not set is automatically inherited from the Elasticsearch
- # output configuration, so if you have the Elasticsearch output configured such
- # that it is pointing to your Elasticsearch monitoring cluster, you can simply
- # uncomment the following line.
- #monitoring.elasticsearch:
- ?
- # ============================== Instrumentation ===============================
- ?
- # Instrumentation support for the filebeat.
- #instrumentation:
- # Set to true to enable instrumentation of filebeat.
- #enabled: false
- ?
- # Environment in which filebeat is running on (eg: staging, production, etc.)
- #environment: ""
- ?
- # APM Server hosts to report instrumentation results to.
- #hosts:
- # - http://localhost:8200
- ?
- # API Key for the APM Server(s).
- # If api_key is set then secret_token will be ignored.
- #api_key:
- ?
- # Secret token for the APM Server(s).
- #secret_token:
- ?
- ?
- # ================================= Migration ==================================
- ?
- # This allows to enable 6.7 migration aliases
- #migration.6_to_7.enabled: true
- filebeat -e -c filebeat.yml
- vim /lib/systemd/system/filebeat.service
- [Unit]
- Description=filebeat
- Wants=network-online.target
- After=network-online.target
- ?
- [Service]
- User=tomcat
- ExecStart=/home/tomcat/elk/filebeat-8.13.4/filebeat -e -c /home/tomcat/elk/filebeat-8.13.4/filebeat.yml
- Restart=always
- ?
- [Install]
- WantedBy=multi-user.target
- systemctl enable filebeat #开机自启
- systemctl start filebeat #启动
- systemctl stop filebeat #停止
- systemctl restart filebeat #重启