ELK - Logstash exemples de patterns grok

#Gestion des logs access apache dans syslog
grok {
	match => { "message" => "%{SYSLOG5424PRI}%{SYSLOGBASE2} %{COMBINEDAPACHELOG}" }
  add_tag => [ "apache", "accesslogs" ]
}

grok {
	patterns_dir => ["/etc/logstash/patterns"]
	match => { "message" => "%{IPTABLES}"}
	add_tag => [ "iptables", "iptables-drop"]
}

DHCPD ((%{SYSLOGTIMESTAMP:timestamp})\s*(%{HOSTNAME:hostname})\s*dhcpd\S+\s*(%{WORD:dhcp_action})?.*[for|on] (%{IPV4:dhcp_client_ip})?.*[from|to] (%{COMMONMAC:dhcp_client_mac})?.*via (%{USERNAME:interface}))
#IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s*(%{HOSTNAME:nf_host})\s*kernel\S+\s*(%{WORD:nf_action})?.*IN=(%{USERNAME:nf_in_interface})?.*OUT=(%{USERNAME:nf_out_interface})?.*MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*))
#Gestion de IPv4 et v6
IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s*(%{HOSTNAME:nf_host})\s*kernel\S+\s*(%{WORD:nf_action})?.*IN=(%{USERNAME:nf_in_interface})?.*OUT=(%{USERNAME:nf_out_interface})?.*MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.*SRC=(%{IP:nf_src_ip}).*DST=(%{IP:nf_dst_ip}).*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*))
DNS ((%{MONTHDAY:day})-(%{MONTH:month})-(%{YEAR:year}) (%{TIME:timestamp}) client (%{IPV4:dns_client_ip})#(%{NONNEGINT:dns_uuid})?.*query: (%{HOSTNAME:dns_dest}) (%{WORD:dns_type}) (%{WORD:dns_record})?.*(%{IPV4:dns_server}))
PGSQL ((%{SYSLOGTIMESTAMP:pgsql_timestamp}) (%{HOSTNAME:pgsql_hostname})?.*SAST >(%{WORD:pgsql_severity}):  (%{GREEDYDATA:pgsql_message}))
SQUID ((%{DATA:squid_timestamp_unix}) (%{INT:squid_time_elapsed_ms}) (%{IPV4:squid_client_ip}) (%{DATA:squid_action})/(%{INT:squid_reply_code}) (%{INT:squid_request_size}) (%{WORD:squid_method}) (%{URIPROTO:squid_request_protocol})://(%{DATA:squid_request_hostname}) - (%{WORD:squid_heirarchy})/(%{IPV4:squid_dest_ip}) (%{WORD:squid_content_type})/(%{WORD:squid_content_object}))
RADIUSINFO ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_messsage}))
RADIUSERROR ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{DATA:radius_module}): (%{GREEDYDATA:radius_messsage}))
RADIUSAUTHOKMAC ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}) (%{WORD}) (%{GREEDYDATA:radius_mac})\))
RADIUSAUTHOK ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))
RADIUSAUTHFAIL ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}): \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))

grok {
	match => { "message" => "(%{SYSLOGTIMESTAMP:f2b_timestamp})\s*(%{HOSTNAME:f2b_host})\s*fail2ban.action\S+\s*(%{WORD:f2b_level})\s*\[(%{GREEDYDATA:f2b_jail})\s*\]\s*(%{WORD:f2b_action})\s*(%{IPORHOST:src_ip})" }
	add_tag => [ "fail2ban" ]
}

grok {
	keep_empty_captures => true
	match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3})\s+(?<first_timestamp_dd>\d{1,2})\s+(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2})\s+(?<device>\S+)\s+(?<program>\S+)\s+(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2})\s+(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2})\.(?<timestamp_SSSSS>\d+)\s+(?<timestamp_ZZ>[^,]*),%{IP:local_ip},(?<hostname>[^,]+),%{IP:remote_ip},(?<x_forwarded_for>(\"([^\"])*\")|([^,]*)),(?<via>[^,]*),(?<user>[^,]*),(?<user_agent>(\"+[^\"]+\"+|[^,]*)),(?<SSL>\d),(?<SSL_version>[^,]*),(?<SSL_client_DN>[^,]*),(?<SSL_certificate_start>[^,]*),(?<SSL_certificate_end>[^,]*),(?<HTTP_method>[^,]+),(?<URL>[^,]+),(?<URL_options>.*),(?<HTTP_version>HTTP[^,]+),%{NUMBER:HTTP_response_code:int},%{NUMBER:HTTP_response_time:int},%{NUMBER:bytes_received:int},%{NUMBER:bytes_sent:int},(?<referrer>[^,]*),(?<cache>[^,]*),(?<gzip_ratio>[^,]*),(?<applicationID>[^,]+),(?<source_port>\d+),(?<application>[^,]+),(?<unique_id>[^,]+),(?<cookie>[^,]*)(,?)(?<posted_data>.*)"}
	# Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
	add_field => [ "timezone", "+0100" ]
	add_field => [ "log_type", "traffic" ]
	add_tag => [ "rweb_logs", "rweb_traffic_logs" ]
}

grok {
	keep_empty_captures => true
	match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3})\s+(?<first_timestamp_dd>\d{1,2})\s+(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2})\s+(?<device>\S+)\s+\S+\s+(?<log_id>\d+)\s+(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2})\s+(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2})\.(?<timestamp_SSSSS>\d+)\s+%{IP:source_ip}\s+%{IP:remote_ip}\s+\S+\s+(?<alert_id>\d+\.\d+\.\d+\.\d+)\s+(?<applicationID>\S+)\s+(?<unique_id>\S+)\s+(?<sec_bl_sl_id>(([0-9]{5}|.{0})-[0-9]{0,3}\s)+)\s*(?<sec_attack_id>([0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{12}|-))\s+\'(?<description>[^']*)\'\s+\'(?<attack_family>[^']+)\'" }

	# Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
	add_field => [ "timezone", "+0100" ]
	add_field => [ "log_type", "alert" ]
	# Duplicate field for get Application Name after
	add_field => [ "application", "%{applicationID}" ]
	add_tag => [ "rweb_logs" , "rweb_alert_logs"]
}

mutate {
	#  remove_field => [ "message" ]
	} alter {
	condrewrite => [
	"user", "", "-",
	"bytes_received", "", "-",
	"referrer", "", "-",
	"application", "56b7df6c-e9e6-454d-9ab5-45d357aaf5e7", "APP_NAME",
	"timestamp_MM", "01", "Jan",
	"timestamp_MM", "02", "Feb",
	"timestamp_MM", "03", "Mar",
	"timestamp_MM", "04", "Apr",
	"timestamp_MM", "05", "May",
	"timestamp_MM", "06", "Jun",
	"timestamp_MM", "07", "Jul",
	"timestamp_MM", "08", "Aug",
	"timestamp_MM", "09", "Sep",
	"timestamp_MM", "10", "Oct",
	"timestamp_MM", "11", "Nov",
	"timestamp_MM", "12", "Dec" ]
	}
}