Initial commit

This commit is contained in:
root
2023-08-09 14:01:28 +02:00
commit f4efbc7a63
199 changed files with 13338 additions and 0 deletions

View File

@@ -0,0 +1,19 @@
filter: "evt.Line.Labels.type == 'containerd'"
onsuccess: next_stage
name: crowdsecurity/cri-logs
description: CRI logging format parser
nodes:
- grok:
pattern: "^%{TIMESTAMP_ISO8601:cri_timestamp} %{WORD:stream} %{WORD:logtag} %{GREEDYDATA:message}"
apply_on: Line.Raw
statics:
- parsed: "logsource"
value: "cri"
- target: evt.StrTime
expression: evt.Parsed.cri_timestamp
- parsed: program
expression: evt.Line.Labels.program
- meta: datasource_path
expression: evt.Line.Src
- meta: datasource_type
expression: evt.Line.Module

View File

@@ -0,0 +1,12 @@
#If it's docker, we are going to extract log line from it
filter: "evt.Line.Labels.type == 'docker'"
onsuccess: next_stage
name: crowdsecurity/docker-logs
description: docker json logs parser
statics:
- target: evt.StrTime
expression: JsonExtract(evt.Line.Raw, "time")
- parsed: message
expression: JsonExtractUnescape(evt.Line.Raw, "log")
- parsed: program
expression: evt.Line.Labels.program

View File

@@ -0,0 +1,48 @@
#If it's syslog, we are going to extract progname from it
filter: "evt.Line.Labels.type == 'syslog'"
onsuccess: next_stage
pattern_syntax:
RAW_SYSLOG_PREFIX: '^<%{NUMBER:stuff1}>%{NUMBER:stuff2} %{SYSLOGBASE2} %{DATA:program} %{NUMBER:pid}'
RAW_SYSLOG_META: '\[meta sequenceId="%{NOTDQUOTE:seq_id}"\]'
name: crowdsecurity/syslog-logs
nodes:
- grok:
#this is a named regular expression. grok patterns can be kept into separate files for readability
pattern: "^%{SYSLOGLINE}"
#This is the field of the `Event` to which the regexp should be applied
apply_on: Line.Raw
- grok:
#a second pattern for unparsed syslog lines, as saw in opnsense
pattern: '%{RAW_SYSLOG_PREFIX} - %{RAW_SYSLOG_META} %{GREEDYDATA:message}'
apply_on: Line.Raw
#if the node was successfull, statics will be applied.
statics:
- meta: machine
expression: evt.Parsed.logsource
- parsed: "logsource"
value: "syslog"
# syslog date can be in two different fields (one of hte assignment will fail)
- target: evt.StrTime
expression: evt.Parsed.timestamp
- target: evt.StrTime
expression: evt.Parsed.timestamp8601
- meta: datasource_path
expression: evt.Line.Src
- meta: datasource_type
expression: evt.Line.Module
---
#if it's not syslog, the type is the progname
filter: "evt.Line.Labels.type != 'syslog'"
onsuccess: next_stage
name: crowdsecurity/non-syslog
#debug: true
statics:
- parsed: message
expression: evt.Line.Raw
- parsed: program
expression: evt.Line.Labels.type
- meta: datasource_path
expression: evt.Line.Src
- meta: datasource_type
expression: evt.Line.Module

View File

@@ -0,0 +1,26 @@
#contribution by @ltsich
onsuccess: next_stage
debug: false
filter: "evt.Parsed.program == 'dovecot'"
name: crowdsecurity/dovecot-logs
description: "Parse dovecot logs"
nodes:
- grok:
pattern: "%{WORD:protocol}-login: %{DATA:dovecot_login_message}: user=<%{DATA:dovecot_user}>.*, rip=%{IP:dovecot_remote_ip}, lip=%{IP:dovecot_local_ip}"
apply_on: message
- grok:
pattern: "auth-worker\\(%{INT}\\): %{WORD:dovecot_user_backend}\\(%{DATA:dovecot_user},%{IP:dovecot_remote_ip},?%{DATA}\\): (%{DATA}: )?%{DATA:dovecot_login_message}$"
apply_on: message
- grok:
pattern: "auth-worker\\(%{INT}\\): (Info: )?conn unix:auth-worker \\(pid=%{INT},uid=%{INT}\\): auth-worker<%{INT}>: %{WORD:dovecot_user_backend}\\(%{DATA:dovecot_user},%{IP:dovecot_remote_ip},?%{DATA}\\): (%{DATA}: )?%{DATA:dovecot_login_message}$"
apply_on: message
- grok:
pattern: "auth: passwd-file\\(%{DATA:dovecot_user},%{IP:dovecot_remote_ip}\\): (%{DATA}: )?%{DATA:dovecot_login_message}$"
apply_on: message
statics:
- meta: log_type
value: dovecot_logs
- meta: source_ip
expression: "evt.Parsed.dovecot_remote_ip"
- meta: dovecot_login_result
expression: "any(['Authentication failure', 'Password mismatch', 'password mismatch', 'auth failed', 'unknown user'], {evt.Parsed.dovecot_login_message contains #}) ? 'auth_failed' : ''"

View File

@@ -0,0 +1,70 @@
filter: "evt.Parsed.program startsWith 'nginx'"
onsuccess: next_stage
name: crowdsecurity/nginx-logs
description: "Parse nginx access and error logs"
pattern_syntax:
NGCUSTOMUSER: '[a-zA-Z0-9\.\@\-\+_%]+'
nodes:
- grok:
pattern: '(%{IPORHOST:target_fqdn} )?%{IPORHOST:remote_addr} - %{NGCUSTOMUSER:remote_user}? \[%{HTTPDATE:time_local}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER:status} %{NUMBER:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"( %{NUMBER:request_length} %{NUMBER:request_time} \[%{DATA:proxy_upstream_name}\] \[%{DATA:proxy_alternative_upstream_name}\])?'
apply_on: message
statics:
- meta: log_type
value: http_access-log
- target: evt.StrTime
expression: evt.Parsed.time_local
- grok:
# and this one the error log
pattern: '(%{IPORHOST:target_fqdn} )?%{NGINXERRTIME:time} \[%{LOGLEVEL:loglevel}\] %{NONNEGINT:pid}#%{NONNEGINT:tid}: (\*%{NONNEGINT:cid} )?%{GREEDYDATA:message}, client: %{IPORHOST:remote_addr}, server: %{DATA:target_fqdn}, request: "%{WORD:verb} ([^/]+)?%{URIPATHPARAM:request}( HTTP/%{NUMBER:http_version})?", host: "%{IPORHOST}(:%{NONNEGINT})?"'
apply_on: message
statics:
- meta: log_type
value: http_error-log
- target: evt.StrTime
expression: evt.Parsed.time
pattern_syntax:
NO_DOUBLE_QUOTE: '[^"]+'
onsuccess: next_stage
nodes:
- filter: "evt.Parsed.message contains 'was not found in'"
pattern_syntax:
USER_NOT_FOUND: 'user "%{NO_DOUBLE_QUOTE:username}" was not found in "%{NO_DOUBLE_QUOTE}"'
grok:
pattern: '%{USER_NOT_FOUND}'
apply_on: message
statics:
- meta: sub_type
value: "auth_fail"
- meta: username
expression: evt.Parsed.username
- filter: "evt.Parsed.message contains 'password mismatch'"
pattern_syntax:
PASSWORD_MISMATCH: 'user "%{NO_DOUBLE_QUOTE:username}": password mismatch'
grok:
pattern: '%{PASSWORD_MISMATCH}'
apply_on: message
statics:
- meta: sub_type
value: "auth_fail"
- meta: username
expression: evt.Parsed.username
- filter: "evt.Parsed.message contains 'limiting requests, excess'"
statics:
- meta: sub_type
value: "req_limit_exceeded"
# these ones apply for both grok patterns
statics:
- meta: service
value: http
- meta: source_ip
expression: "evt.Parsed.remote_addr"
- meta: http_status
expression: "evt.Parsed.status"
- meta: http_path
expression: "evt.Parsed.request"
- meta: http_verb
expression: "evt.Parsed.verb"
- meta: http_user_agent
expression: "evt.Parsed.http_user_agent"
- meta: target_fqdn
expression: "evt.Parsed.target_fqdn"

View File

@@ -0,0 +1,61 @@
# Copyright (c) 2014, 2015, Rudy Gevaert
# Copyright (c) 2020 Crowdsec
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
# The above copyright notice and this permission notice shall be
# included in all copies or substantial portions of the Software.
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
# Some of the groks used here are from https://github.com/rgevaert/grok-patterns/blob/master/grok.d/postfix_patterns
onsuccess: next_stage
filter: "evt.Parsed.program in ['postfix/smtpd','postfix/smtps/smtpd','postfix/submission/smtpd', 'postfix/smtps-haproxy/smtpd', 'postfix/submission-haproxy/smtpd']"
name: crowdsecurity/postfix-logs
pattern_syntax:
POSTFIX_HOSTNAME: '(%{HOSTNAME}|unknown)'
POSTFIX_COMMAND: '(AUTH|STARTTLS|CONNECT|EHLO|HELO|RCPT)'
POSTFIX_ACTION: 'discard|dunno|filter|hold|ignore|info|prepend|redirect|replace|reject|warn'
RELAY: '(?:%{HOSTNAME:remote_host}(?:\[%{IP:remote_addr}\](?::[0-9]+(.[0-9]+)?)?)?)'
description: "Parse postfix logs"
nodes:
- grok:
apply_on: message
pattern: 'lost connection after %{DATA:smtp_response} from %{RELAY}'
statics:
- meta: log_type_enh
value: spam-attempt
- grok:
apply_on: message
pattern: 'warning: %{POSTFIX_HOSTNAME:remote_host}\[%{IP:remote_addr}\]: SASL ((?i)LOGIN|PLAIN|(?:CRAM|DIGEST)-MD5) authentication failed:%{GREEDYDATA:message_failure}'
statics:
- meta: log_type_enh
value: spam-attempt
- grok:
apply_on: message
pattern: 'NOQUEUE: %{POSTFIX_ACTION:action}: %{DATA:command} from %{RELAY}: %{GREEDYDATA:reason}'
statics:
- meta: action
expression: "evt.Parsed.action"
statics:
- meta: service
value: postfix
- meta: source_ip
expression: "evt.Parsed.remote_addr"
- meta: source_hostname
expression: "evt.Parsed.remote_host"
- meta: log_type
value: postfix

View File

@@ -0,0 +1,20 @@
onsuccess: next_stage
filter: "evt.Parsed.program in ['postfix/postscreen', 'haproxy/postscreen']"
name: crowdsecurity/postscreen-logs
pattern_syntax:
POSTSCREEN_PREGREET: 'PREGREET'
POSTSCREEN_PREGREET_TIME_ATTEMPT: '\d+.\d+'
description: "Parse postscreen logs"
nodes:
- grok:
apply_on: message
pattern: '%{POSTSCREEN_PREGREET:pregreet} %{INT:count} after %{POSTSCREEN_PREGREET_TIME_ATTEMPT:time_attempt} from \[%{IP:remote_addr}\]:%{INT:port}: %{GREEDYDATA:message_attempt}'
statics:
- meta: service
value: postscreen
- meta: source_ip
expression: "evt.Parsed.remote_addr"
- meta: pregreet
expression: "evt.Parsed.pregreet"

View File

@@ -0,0 +1,100 @@
onsuccess: next_stage
#debug: true
filter: "evt.Parsed.program == 'sshd'"
name: crowdsecurity/sshd-logs
description: "Parse openSSH logs"
pattern_syntax:
# The IP grok pattern that ships with crowdsec is buggy and does not capture the last digit of an IP if it is the last thing it matches, and the last octet starts with a 2
# https://github.com/crowdsecurity/crowdsec/issues/938
IPv4_WORKAROUND: (?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)
IP_WORKAROUND: (?:%{IPV6}|%{IPv4_WORKAROUND})
SSHD_AUTH_FAIL: 'pam_%{DATA:pam_type}\(sshd:auth\): authentication failure; logname= uid=%{NUMBER:uid}? euid=%{NUMBER:euid}? tty=ssh ruser= rhost=%{IP_WORKAROUND:sshd_client_ip}( %{SPACE}user=%{USERNAME:sshd_invalid_user})?'
SSHD_MAGIC_VALUE_FAILED: 'Magic value check failed \(\d+\) on obfuscated handshake from %{IP_WORKAROUND:sshd_client_ip} port \d+'
SSHD_INVALID_USER: 'Invalid user\s*%{USERNAME:sshd_invalid_user}? from %{IP_WORKAROUND:sshd_client_ip}( port \d+)?'
SSHD_INVALID_BANNER: 'banner exchange: Connection from %{IP_WORKAROUND:sshd_client_ip} port \d+: invalid format'
SSHD_PREAUTH_AUTHENTICATING_USER: 'Connection closed by (authenticating|invalid) user %{USERNAME:sshd_invalid_user} %{IP_WORKAROUND:sshd_client_ip} port \d+ \[preauth\]'
#following: https://github.com/crowdsecurity/crowdsec/issues/1201 - some scanners behave differently and trigger this one
SSHD_PREAUTH_AUTHENTICATING_USER_ALT: 'Disconnected from (authenticating|invalid) user %{USERNAME:sshd_invalid_user} %{IP_WORKAROUND:sshd_client_ip} port \d+ \[preauth\]'
SSHD_BAD_KEY_NEGOTIATION: 'Unable to negotiate with %{IP_WORKAROUND:sshd_client_ip} port \d+: no matching (host key type|key exchange method|MAC) found.'
nodes:
- grok:
name: "SSHD_FAIL"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_PREAUTH_AUTHENTICATING_USER_ALT"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_PREAUTH_AUTHENTICATING_USER"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_DISC_PREAUTH"
apply_on: message
- grok:
name: "SSHD_BAD_VERSION"
apply_on: message
- grok:
name: "SSHD_INVALID_USER"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_INVALID_BANNER"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: extra_log_type
value: ssh_bad_banner
- grok:
name: "SSHD_USER_FAIL"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_AUTH_FAIL"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_MAGIC_VALUE_FAILED"
apply_on: message
statics:
- meta: log_type
value: ssh_failed-auth
- meta: target_user
expression: "evt.Parsed.sshd_invalid_user"
- grok:
name: "SSHD_BAD_KEY_NEGOTIATION"
apply_on: message
statics:
- meta: log_type
value: ssh_bad_keyexchange
statics:
- meta: service
value: ssh
- meta: source_ip
expression: "evt.Parsed.sshd_client_ip"

View File

@@ -0,0 +1,69 @@
# co-authored with gmelodie (https://github.com/gmelodie)
name: crowdsecurity/traefik-logs
description: "Parse Traefik access logs"
filter: "evt.Parsed.program startsWith 'traefik'"
#debug: true
onsuccess: next_stage
pattern_syntax:
TRAEFIK_ROUTER: '(%{USER}@%{URIHOST}|\-)'
TRAEFIK_SERVER_URL: '(%{URI}|\-)'
NUMBER_MINUS: '[0-9-]+'
NGCUSTOMUSER: '[a-zA-Z0-9\.\@\-\+_%]+'
NGINXACCESS2: '%{IPORHOST:remote_addr} - %{NGCUSTOMUSER:remote_user} \[%{HTTPDATE:time_local}\] "%{WORD:verb} %{DATA:request} HTTP/%{NUMBER:http_version}" %{NUMBER_MINUS:status} %{NUMBER_MINUS:body_bytes_sent} "%{NOTDQUOTE:http_referer}" "%{NOTDQUOTE:http_user_agent}"'
nodes:
- grok: # CLF parser
pattern: '%{NGINXACCESS2} %{NUMBER:number_of_requests_received_since_traefik_started} "%{TRAEFIK_ROUTER:traefik_router_name}" "%{TRAEFIK_SERVER_URL:traefik_server_url}" %{NUMBER:request_duration_in_ms}ms'
apply_on: message
- filter: UnmarshalJSON(evt.Line.Raw, evt.Unmarshaled, "traefik") in ["", nil]
statics:
- parsed: remote_addr
expression: evt.Unmarshaled.traefik.ClientHost
- parsed: dest_addr
## Split dest_addr to get IP only as this is original functionality
expression: Split(evt.Unmarshaled.traefik.ClientAddr, ':')[0]
- parsed: request_addr
expression: evt.Unmarshaled.traefik.RequestAddr
- parsed: service_addr
## Split service_addr to get IP only as this is original functionality
expression: "evt.Unmarshaled.traefik.ServiceAddr != nil ? Split(evt.Unmarshaled.traefik.ServiceAddr, ':')[0] : nil"
- parsed: http_user_agent
expression: evt.Unmarshaled.traefik["request_User-Agent"] ## We have to access via [] as the key contains a dash
- parsed: body_bytes_sent
## We have to check if DownstreamContentSize is nil, as it will cause EXPR error if it is
expression: "evt.Unmarshaled.traefik.DownstreamContentSize != nil ? int(evt.Unmarshaled.traefik.DownstreamContentSize) : nil"
- parsed: request_duration_in_ms
expression: int(evt.Unmarshaled.traefik.Duration)
- parsed: traefik_router_name
expression: evt.Unmarshaled.traefik.RouterName
- parsed: time_local
expression: evt.Unmarshaled.traefik.time
- parsed: verb
expression: evt.Unmarshaled.traefik.RequestMethod
- parsed: request
expression: evt.Unmarshaled.traefik.RequestPath
- parsed: http_version
## Split http_version to get version only as this is original functionality
expression: Split(evt.Unmarshaled.traefik.RequestProtocol, '/')[1]
- parsed: status
expression: int(evt.Unmarshaled.traefik.DownstreamStatus)
statics:
- meta: service
value: http
- meta: http_status
expression: "evt.Parsed.status"
- meta: http_path
expression: "evt.Parsed.request"
- meta: user
expression: "evt.Parsed.remote_user"
- meta: source_ip
expression: "evt.Parsed.remote_addr"
- meta: http_user_agent
expression: "evt.Parsed.http_user_agent"
- meta: log_type
value: http_access-log
- target: evt.StrTime
expression: "evt.Parsed.time_local"
- meta: traefik_router_name
expression: "evt.Parsed.traefik_router_name"
- meta: http_verb
expression: "evt.Parsed.verb"

View File

@@ -0,0 +1,11 @@
filter: "evt.StrTime != ''"
name: crowdsecurity/dateparse-enrich
#debug: true
#it's a hack lol
statics:
- method: ParseDate
expression: evt.StrTime
- target: MarshaledTime
expression: evt.Enriched.MarshaledTime
- meta: timestamp
expression: evt.Enriched.MarshaledTime

View File

@@ -0,0 +1,27 @@
filter: "'source_ip' in evt.Meta"
name: crowdsecurity/geoip-enrich
description: "Populate event with geoloc info : as, country, coords, source range."
data:
- source_url: https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/GeoLite2-City.mmdb
dest_file: GeoLite2-City.mmdb
- source_url: https://crowdsec-statics-assets.s3-eu-west-1.amazonaws.com/GeoLite2-ASN.mmdb
dest_file: GeoLite2-ASN.mmdb
statics:
- method: GeoIpCity
expression: evt.Meta.source_ip
- meta: IsoCode
expression: evt.Enriched.IsoCode
- meta: IsInEU
expression: evt.Enriched.IsInEU
- meta: GeoCoords
expression: evt.Enriched.GeoCoords
- method: GeoIpASN
expression: evt.Meta.source_ip
- meta: ASNNumber
expression: evt.Enriched.ASNNumber
- meta: ASNOrg
expression: evt.Enriched.ASNOrg
- method: IpToRange
expression: evt.Meta.source_ip
- meta: SourceRange
expression: evt.Enriched.SourceRange

View File

@@ -0,0 +1,33 @@
filter: "evt.Meta.service == 'http' && evt.Meta.log_type in ['http_access-log', 'http_error-log']"
description: "Parse more Specifically HTTP logs, such as HTTP Code, HTTP path, HTTP args and if its a static ressource"
name: crowdsecurity/http-logs
pattern_syntax:
DIR: "^.*/"
FILE: "[^/].*?"
EXT: "\\.[^.]*$|$"
nodes:
- statics:
- parsed: "impact_completion"
# the value of a field can as well be determined as the result of an expression
expression: "evt.Meta.http_status in ['404', '403', '502'] ? 'false' : 'true'"
- target: evt.Parsed.static_ressource
value: 'false'
# let's split the path?query if possible
- grok:
pattern: "^%{GREEDYDATA:request}\\?%{GREEDYDATA:http_args}$"
apply_on: request
# this is another node, with its own pattern_syntax
- #debug: true
grok:
pattern: "%{DIR:file_dir}(%{FILE:file_frag}%{EXT:file_ext})?"
apply_on: request
statics:
- meta: http_path
expression: "evt.Parsed.http_path"
# meta af
- meta: http_args_len
expression: "len(evt.Parsed.http_args)"
- parsed: file_name
expression: evt.Parsed.file_frag + evt.Parsed.file_ext
- parsed: static_ressource
expression: "Upper(evt.Parsed.file_ext) in ['.JPG', '.CSS', '.JS', '.JPEG', '.PNG', '.SVG', '.MAP', '.ICO', '.OTF', '.GIF', '.MP3', '.MP4', '.WOFF', '.WOFF2', '.TTF', '.OTF', '.EOT', '.WEBP', '.WAV', '.GZ', '.BROTLI', '.BVR', '.TS', '.BMP'] ? 'true' : 'false'"

View File

@@ -0,0 +1,14 @@
name: crowdsecurity/whitelists
description: "Whitelist events from private ipv4 addresses"
whitelist:
reason: "private ipv4/ipv6 ip/ranges"
ip:
- "127.0.0.1"
- "::1"
cidr:
- "192.168.0.0/16"
- "10.0.0.0/8"
- "172.16.0.0/12"
# expression:
# - "'foo.com' in evt.Meta.source_ip.reverse"