77 lines
4.9 KiB
Plaintext
Executable File
77 lines
4.9 KiB
Plaintext
Executable File
# The logstash configuration below takes Titanium Cloud syslog input and outputs the custom Titanium Cloud log data to elasticsearch.
|
|
# Extending the openstack log format, Titanium Cloud syslog messages required the use of grok to parse log data into something structured and queryable.
|
|
# - Inconsistent formating of log level, pid and program
|
|
# - custom Titanium Cloud syslog fields and naming
|
|
|
|
input {
|
|
# Do not use syslog input plugin (or type)
|
|
#TCP_PARAMS
|
|
#UDP_PARAMS
|
|
}
|
|
|
|
filter {
|
|
# "Grok is currently the best way in logstash to parse log data into something structured and queryable."
|
|
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-grok.html
|
|
# The input plugins above convert the
|
|
grok {
|
|
match => {
|
|
"message" => [
|
|
# The default break_on_match is used so first successful match by grok will result in the filter being finished.
|
|
# Use Titanium Cloud term node instead of host.
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}\: %{TIMESTAMP_ISO8601:syslog_trash} %{POSINT:pid} %{LOGLEVEL:level} %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program} %{TIMESTAMP_ISO8601:syslog_trash} %{POSINT:pid} %{LOGLEVEL:level} %{DATA:program} %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}(?:\[%{POSINT:pid}\])?: %{LOGLEVEL:level}?: %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program} \[%{POSINT:pid}\] %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}: %{LOGLEVEL:level} %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}: %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}\[%{POSINT:pid}\]?: %{LOGLEVEL:level} %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}\[%{POSINT:pid}\]?: %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program}(?:\[%{POSINT:pid}\])?: %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{NOTSPACE:system_name} %{NOTSPACE:filename} %{SYSLOGHOST:node} %{DATA:program} %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{TIMESTAMP_ISO8601:syslog_timestamp} %{POSINT:pid} %{LOGLEVEL:level} %{DATA:program} %{GREEDYDATA:message}",
|
|
"<%{POSINT:syslog_pri}>%{DATESTAMP:syslog_timestamp} %{POSINT:pid} %{LOGLEVEL:level} %{DATA:program} %{GREEDYDATA:message}"
|
|
]
|
|
}
|
|
overwrite => [ "message" ]
|
|
remove_field => [ "syslog_trash" ]
|
|
add_field => { "host_timestamp" => "%{syslog_timestamp}" }
|
|
}
|
|
|
|
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-syslog_pri.html
|
|
syslog_pri {
|
|
facility_labels => ["kernel", "user-level", "mail", "daemon", "security/authorization", "syslogd", "line printer", "network news", "UUCP", "clock", "security/authorization", "FTP", "NTP", "log audit", "log alert", "clock", "postgres", "platform", "openstack", "sm", "local4", "mtce", "sysinv", "horizon"]
|
|
|
|
severity_labels => [
|
|
"emergency", "alert", "crit", "error", "warn", "notice", "info", "debug"
|
|
]
|
|
# syslog_pri has served its purpose, and syslog_facility_code isn't useful
|
|
remove_field => [ "syslog_pri", "syslog_facility_code", "syslog_severity_code", "severity_label", "syslog_severity" ]
|
|
}
|
|
|
|
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-date.html
|
|
date {
|
|
# set <at> timestamp from the grok'd syslog_timestamp and remove the field
|
|
match => [ "syslog_timestamp", "MMM d HH:mm:ss.SSS", "MMM dd HH:mm:ss.SSS" , "yyyy-MM-dd HH:mm:ss.SSS"]
|
|
remove_field => [ "syslog_timestamp" ]
|
|
timezone => [ "UTC" ]
|
|
}
|
|
|
|
# Rename and remove unwanted syslog fields
|
|
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-mutate.html
|
|
mutate {
|
|
rename => [
|
|
"host", "system_address"
|
|
]
|
|
# https://www.elastic.co/guide/en/logstash/current/plugins-filters-mutate.html#plugins-filters-mutate-remove_field
|
|
remove_field => [ "type", "syslog_facility" ]
|
|
}
|
|
}
|
|
|
|
output {
|
|
if "_grokparsefailure" in [tags] {
|
|
file { path => "/var/log/logstash/wrs-grokparsefailure-%{+YYYY-MM-dd}" }
|
|
} else {
|
|
elasticsearch { hosts => ["127.0.0.1:9200"] }
|
|
}
|
|
}
|