Patterns :

Access log Apache

1 #Gestion des logs access apache dans syslog
2 grok {
3   match => { "message" => "%{SYSLOG5424PRI}%{SYSLOGBASE2} %{COMBINEDAPACHELOG}" }
4   add_tag => [ "apache", "accesslogs" ]
5 }

iptables

1 grok {
2   patterns_dir => ["/etc/logstash/patterns"]
3   match => { "message" => "%{IPTABLES}"}
4   add_tag => [ "iptables", "iptables-drop"]
5 }

Fichier de pattern :

 1 DHCPD ((%{SYSLOGTIMESTAMP:timestamp})\s*(%{HOSTNAME:hostname})\s*dhcpd\S+\s*(%{WORD:dhcp_action})?.*[for|on] (%{IPV4:dhcp_client_ip})?.*[from|to] (%{COMMONMAC:dhcp_client_mac})?.*via (%{USERNAME:interface}))
 2 #IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s*(%{HOSTNAME:nf_host})\s*kernel\S+\s*(%{WORD:nf_action})?.*IN=(%{USERNAME:nf_in_interface})?.*OUT=(%{USERNAME:nf_out_interface})?.*MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.*SRC=(%{IPV4:nf_src_ip}).*DST=(%{IPV4:nf_dst_ip}).*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*))
 3 #Gestion de IPv4 et v6
 4 IPTABLES ((%{SYSLOGTIMESTAMP:nf_timestamp})\s*(%{HOSTNAME:nf_host})\s*kernel\S+\s*(%{WORD:nf_action})?.*IN=(%{USERNAME:nf_in_interface})?.*OUT=(%{USERNAME:nf_out_interface})?.*MAC=(%{COMMONMAC:nf_dst_mac}):(%{COMMONMAC:nf_src_mac})?.*SRC=(%{IP:nf_src_ip}).*DST=(%{IP:nf_dst_ip}).*PROTO=(%{WORD:nf_protocol}).?*SPT=(%{INT:nf_src_port}?.*DPT=%{INT:nf_dst_port}?.*))
 5 DNS ((%{MONTHDAY:day})-(%{MONTH:month})-(%{YEAR:year}) (%{TIME:timestamp}) client (%{IPV4:dns_client_ip})#(%{NONNEGINT:dns_uuid})?.*query: (%{HOSTNAME:dns_dest}) (%{WORD:dns_type}) (%{WORD:dns_record})?.*(%{IPV4:dns_server}))
 6 PGSQL ((%{SYSLOGTIMESTAMP:pgsql_timestamp}) (%{HOSTNAME:pgsql_hostname})?.*SAST >(%{WORD:pgsql_severity}):  (%{GREEDYDATA:pgsql_message}))
 7 SQUID ((%{DATA:squid_timestamp_unix}) (%{INT:squid_time_elapsed_ms}) (%{IPV4:squid_client_ip}) (%{DATA:squid_action})/(%{INT:squid_reply_code}) (%{INT:squid_request_size}) (%{WORD:squid_method}) (%{URIPROTO:squid_request_protocol})://(%{DATA:squid_request_hostname}) - (%{WORD:squid_heirarchy})/(%{IPV4:squid_dest_ip}) (%{WORD:squid_content_type})/(%{WORD:squid_content_object}))
 8 RADIUSINFO ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_messsage}))
 9 RADIUSERROR ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{DATA:radius_module}): (%{GREEDYDATA:radius_messsage}))
10 RADIUSAUTHOKMAC ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}) (%{WORD}) (%{GREEDYDATA:radius_mac})\))
11 RADIUSAUTHOK ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}) \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))
12 RADIUSAUTHFAIL ((%{SYSLOGTIMESTAMP:radius_timestamp}) (%{YEAR}) : (%{WORD:radius_severity}): (%{GREEDYDATA:radius_login_status}): \[(%{USERNAME:radius_user})/(%{USERNAME:radius_password})\] \(from client (%{GREEDYDATA:radius_from_client}) port (%{INT:radius_nas_port}))

fail2ban

1 grok {
2   match => { "message" => "(%{SYSLOGTIMESTAMP:f2b_timestamp})\s*(%{HOSTNAME:f2b_host})\s*fail2ban.action\S+\s*(%{WORD:f2b_level})\s*\[(%{GREEDYDATA:f2b_jail})\s*\]\s*(%{WORD:f2b_action})\s*(%{IPORHOST:src_ip})" }
3   add_tag => [ "fail2ban" ]
4 }

DenyALL rWeb

Trafic logs

1 grok {
2   keep_empty_captures => true
3   match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3})\s+(?<first_timestamp_dd>\d{1,2})\s+(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2})\s+(?<device>\S+)\s+(?<program>\S+)\s+(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2})\s+(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2})\.(?<timestamp_SSSSS>\d+)\s+(?<timestamp_ZZ>[^,]*),%{IP:local_ip},(?<hostname>[^,]+),%{IP:remote_ip},(?<x_forwarded_for>(\"([^\"])*\")|([^,]*)),(?<via>[^,]*),(?<user>[^,]*),(?<user_agent>(\"+[^\"]+\"+|[^,]*)),(?<SSL>\d),(?<SSL_version>[^,]*),(?<SSL_client_DN>[^,]*),(?<SSL_certificate_start>[^,]*),(?<SSL_certificate_end>[^,]*),(?<HTTP_method>[^,]+),(?<URL>[^,]+),(?<URL_options>.*),(?<HTTP_version>HTTP[^,]+),%{NUMBER:HTTP_response_code:int},%{NUMBER:HTTP_response_time:int},%{NUMBER:bytes_received:int},%{NUMBER:bytes_sent:int},(?<referrer>[^,]*),(?<cache>[^,]*),(?<gzip_ratio>[^,]*),(?<applicationID>[^,]+),(?<source_port>\d+),(?<application>[^,]+),(?<unique_id>[^,]+),(?<cookie>[^,]*)(,?)(?<posted_data>.*)"}
4   # Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
5   add_field => [ "timezone", "+0100" ]
6   add_field => [ "log_type", "traffic" ]
7   add_tag => [ "rweb_logs", "rweb_traffic_logs" ]
8 }

Alert logs

 1 grok {
 2  keep_empty_captures => true
 3  match => { "message" => "<(?<id>[^>]+)>(?<first_timestamp_MMM>\w{3})\s+(?<first_timestamp_dd>\d{1,2})\s+(?<first_timestamp_hh>\d{2}):(?<first_timestamp_mm>\d{2}):(?<first_timestamp_ss>\d{2})\s+(?<device>\S+)\s+\S+\s+(?<log_id>\d+)\s+(?<timestamp_yyyy>\d{4})-(?<timestamp_MM>\d{2})-(?<timestamp_dd>\d{2})\s+(?<timestamp_hh>\d{2}):(?<timestamp_mm>\d{2}):(?<timestamp_ss>\d{2})\.(?<timestamp_SSSSS>\d+)\s+%{IP:source_ip}\s+%{IP:remote_ip}\s+\S+\s+(?<alert_id>\d+\.\d+\.\d+\.\d+)\s+(?<applicationID>\S+)\s+(?<unique_id>\S+)\s+(?<sec_bl_sl_id>(([0-9]{5}|.{0})-[0-9]{0,3}\s)+)\s*(?<sec_attack_id>([0-9a-zA-Z]{8}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{4}-[0-9a-zA-Z]{12}|-))\s+\'(?<description>[^']*)\'\s+\'(?<attack_family>[^']+)\'" }
 4 
 5  # Specify in this custom field the +/- hours of the timezone (+1h for Europe/Paris)
 6  add_field => [ "timezone", "+0100" ]
 7  add_field => [ "log_type", "alert" ]
 8  # Duplicate field for get Application Name after
 9  add_field => [ "application", "%{applicationID}" ]
10  add_tag => [ "rweb_logs" , "rweb_alert_logs"]
11 }

Du fait que les Alert logs ne fournissent pas le nom d'application mais que l'UID, on duplique l'UID dans un nouveau champ (application) et ensuite on modifie de manière statique (ou scripté) via un mutate.

 1 mutate {
 2  #  remove_field => [ "message" ]
 3  } alter {
 4  condrewrite => [
 5  "user", "", "-",
 6  "bytes_received", "", "-",
 7  "referrer", "", "-",
 8  "application", "56b7df6c-e9e6-454d-9ab5-45d357aaf5e7", "APP_NAME",
 9  "timestamp_MM", "01", "Jan",
10  "timestamp_MM", "02", "Feb",
11  "timestamp_MM", "03", "Mar",
12  "timestamp_MM", "04", "Apr",
13  "timestamp_MM", "05", "May",
14  "timestamp_MM", "06", "Jun",
15  "timestamp_MM", "07", "Jul",
16  "timestamp_MM", "08", "Aug",
17  "timestamp_MM", "09", "Sep",
18  "timestamp_MM", "10", "Oct",
19  "timestamp_MM", "11", "Nov",
20  "timestamp_MM", "12", "Dec" ]
21  }
22 }

Aide au debug

Aide/debug de pattern grok