3KR : d Z ddlZddlZddlZddlZddlZddlZddlZddl m
Z
m Z ddl m
Z
ddlmZ ddlmZmZ ddlmZ ddlmZ dd lmZmZ dd
lmZ dd lmZm Z dd l m Z m Z dd
l m!Z! dZ"d\ Z#Z$Z%d\ Z&Z'Z(Z)Z*Z+dZ,dde*ddddde#dddd d e+e,e$fe"z Z-d x\ Z.Z/Z0 ej1 d ej2 Z3 ee4 Z5 e e e5j6 Z7 G d! d"e8 Z9 G d# d$e8 Z: G d% d&e8 Z; G d' d(e< Z= G d) d*e
+ Z> G d, d-e> Z? G d. d/e> Z@ G d0 d1e> ZA G d2 d3e> ZB G d4 d5e> ZC G d6 d7eC ZD G d8 d9e> ZE G d: d;e> ZF G d< d=e
+ ZG G d> d?eG ZH G d@ dAeG ZI G dB dCeI ZJ G dD dEeI ZK G dF dGeK ZL G dH dIeK ZM G dJ dKeL ZN G dL dMeG ZO G dN dOeH ZP G dP dQeG ZQ G dR dSeH ZR G dT dU ZSdedVZTdeUdWeVfdXZWdYeUdZeUdWeeeU eXf fd[ZYdWeeU fd\ZZd] Z[d^eVfd_Z\d` Z]daeeU dWeeU fdbZ^dceUdWe_fddZ`dS )fz
SecAuditLog parser
N)ABCMetaabstractmethod)suppress)copy) CookieError SimpleCookie)product) getLogger)OptionalTuple)parse_qs)DAY
rate_limit)USER_IDENTITY_FIELD
user_identity) ModsecSensor)inbound_anomality_scoreoutbound_anomality_score)severityadvancedheaders)uri http_methodformquery attackers_ip status_code engine_modez
User-AgentHosttransaction_idrulemsgmessage
access_deniedvertagProducermodsec_versionvendor)cookie
authorizationz((Inbound|Outbound) Anomaly Score.*?(\d+))periodc e Zd ZdZdS )
ParseErrorE
log as logger.exception(*e.args) to avoid sentry duplicates
N__name__
__module__ __qualname____doc__ R/opt/imunify360/venv/lib/python3.11/site-packages/im360/subsys/modsec_audit_log.pyr. r. D Dr6 r. c e Zd ZdZdS )_AmbiguousSeverityz
to log as warning
Nr0 r5 r6 r7 r: r: L r8 r6 r: c e Zd ZdZdS )MalformedFileErrorr/ Nr0 r5 r6 r7 r< r< T r8 r6 r< c e Zd ZdS )_MiscDataNotInterestingN)r1 r2 r3 r5 r6 r7 r> r> \ s Dr6 r> c N e Zd ZdZed Zeed ZdS )_SerialLogSectionParserNc \ | j s
J d | j | d uS )Nzregex should be implemented)_IS_APPLICABLE_SINCE_REGEXmatchclslines r7 is_applicable_sincez+_SerialLogSectionParser.is_applicable_sincec s6 -LL/LLL--33D99EEr6 c dS )z_
:return: name, value tokens
:raise ValueError: if cannot parse string
Nr5 rD s r7 parse_name_value_tokensz/_SerialLogSectionParser.parse_name_value_tokensh s
r6 )r1 r2 r3 rB classmethodrG r rI r5 r6 r7 r@ r@ ` s[ !% F F [F
^ [
r6 r@ ) metaclassc d e Zd Z ej d Z ej d Zed ZdS )_SectionAParserz^-{2,3}\w+-{1,3}A--$z ^\[.*?\] (?P\S+) (?P\S+)c # K | j | }|s6t dt | t t
j | d }nJ# t $ r= t d| d t w xY wdt | fV d| d fV d S )Nz)--section-A--: cannot parse this line: %sipz5--section-A--: cannot use %s for IPv4 or IPv6 address r r id) _PARSE_IP_REGEXrC loggerwarningreprr> ipaddress
ip_addressgroup
ValueErrorstr)rE rF rC ipv4_or_ipv6s r7 rI z'_SectionAParser.parse_name_value_tokensv s #))$// , NN;T$ZZ
*++ + ,$/ D0A0ABBLL , , , NNG A
*++ + , c, / ///// D 1 1111111s
'A> >ACN) r1 r2 r3 recompilerB rR rJ rI r5 r6 r7 rM rM r sR !+,C!D!D bj!DEEO2 2 [2 2 2r6 rM c d e Zd Z ej d Z ej d Zed ZdS )_SectionBParserz^-{2,3}\w+-{1,3}B--$z^(\S*): ?(.*)$c # K | j | }|rR| d | d pd }}t || \ }}|rt ||gfV d S d S | ^}}} | rt j | rgt |fV t j
| }
t |
j fV t j r#t" t% |
j fV d S d S d S d S # t( $ r t+ w xY w)NrP )_PARSE_NAME_VALUErC rX #obfuscate_if_sensitive_and_requiredHEADERSsplitisupperospathisabs HTTP_METHODurllibparseurlparseURIr SEND_ADDITIONAL_DATA QUERY_PARAMS obfuscater rY r> ) rE rF rC namevaluemaybe_obfuscated_value is_requiredmethodr _parseds r7 rI z'_SectionBParser.parse_name_value_tokens s %++D11 0 ++a..%++a..*@D%D 4D%@@
&
> &< =======
>
>
0"&**,,a>>## D
c(:(: D%v---- #\ 2 23 7 7F v{****#8 D *Ifl,C,C CCCCCC
D D D D
D D
0
0
0-///
0s
2B-D' 'E N) r1 r2 r3 r\ r] rB rb rJ rI r5 r6 r7 r_ r_ sS !+,C!D!D "
#4550 0 [0 0 0r6 r_ c D e Zd Z ej d Zed ZdS )_SectionCParserz^-{2,3}\w+-{1,3}C--$c # V K t j rt t | fV d S d S N)r ro FORM_PARAMSrq rD s r7 rI z'_SectionCParser.parse_name_value_tokens s8 , / y. . . . . . / /r6 Nr1 r2 r3 r\ r] rB rJ rI r5 r6 r7 rz rz s@ !+,C!D!D / / [/ / /r6 rz c d e Zd Z ej d Z ej d Zed ZdS )_SectionFParserz^-{2,3}\w+-{1,3}F--$z^HTTP/\d\.\d (\d+).*$c # K | j | }|st t | d fV d S )NrP )_STATUS_CODE_REGEXrC r> STATUS_CODErX )rE rF rC s r7 rI z'_SectionFParser.parse_name_value_tokens sQ &,,T22 .)++ + u{{1~~- - - - - -r6 N) r1 r2 r3 r\ r] rB r rJ rI r5 r6 r7 r r sS !+,C!D!D #$<==. . [. . .r6 r c & e Zd Z ej d Z ej d Z ej d Z ej d Z ej d Z ej d Z
e d Z e d Z
e d Ze d
Zd S ) _BaseSectionHParserz^-{2,3}\w+-{1,3}H--$z ([^=]*) (ModSecurity):\s+([^=]*)z(\[\w+ ".*?"\])zAccess denied with code \d+z\[(\w+) "(.*?)"\]z
(\[\w+ ".*?$)c | | \ }}|dv r|g| || R S |dk r| || S |dk r t }| d }||fS )N)Message ModSecurityz Apache-Errorz Engine-Mode")_parse_name_value_parse_message_parse_apache_err ENGINE_MODEstrip)rE rF rr rs s r7 rI z+_BaseSectionHParser.parse_name_value_tokens s ++D11 e - - -9#,,UD9999 9 > ! !((55 5 = DKK$$EU{r6 c | d \ }}|d dk rt d |dd }||fS )z;
:raise ValueError: if cannot parse string
rP maxsplit:z Expecting sort of "Name: value"N)re rY )rE srr rs s r7 r z%_BaseSectionHParser._parse_name_value sP
ggqg)) e 8s?? >?? ? 9DU{r6 c | j | }|r2| \ }}}|g| |d R S t )N )_PARSE_MESSAGE_APACHE_ERRORrC groupsr r> )rE msgvalrw rC rr modsecs r7 r z%_BaseSectionHParser._parse_apache_err sb /55f== , #llnnOAtV8#,,VR8888 8 *++ +r6 c | j | }d | }|}|D ]*}| |d }+| j | }|D ]*}| |d }+t
| }| j | } | rd|d< | j
| D ]}| \ }
} | |
} | b|
dk r)t d| t t! | t" s| gx} ||
< | | n0|
d k r| g||
< n#|
d
k r| |d < | t' d | n| ||
< |
d
k r$t) | }
| |
|g|R S )aB
Parse string with format [key "val"] [key "val"] ... [key "val"]
to dictionary {"key": "val", "key": "val", ...}
In case when there is uncompleted value part
e.g. '[key "val'
with lost '"]' in the end
it also returns incomplete part as string '[key "val'
r )r# Tr$ Nr zAmbiguous severity: %rr& rQ r! z&rule field is None, modsec message: %sr" )_PARSE_MESSAGE_REGEXfindalljoinreplacer _INCOMPLETE_MESSAGE_REGEXdict_PARSE_ACCESS_DENIED_CODEsearch_PARSE_MESSAGE_TOKENS_REGEXfinditerr getrS rT r:
isinstancelistappendthrottled_log_exceptionget_anomality_score_itemsupdate)rE r audit_log_line tokens_listtokensmsgtxttokenincomplete_partresultrC rr textexistinganomality_score_itemss r7 r z"_BaseSectionHParser._parse_message s .66v>> +&& 7 7E^^E2..4466FF 7??GG$ 7 7E^^E2..4466FFf%%%-44V<< +&*F? #4==fEE 5 5E JD$ zz$''H#:%%NN#;^LLL,...!(D11 9/7j8Hvd|%%%% $vt !%v<+@& $t u}}(A$(G(G%
3444''''r6 N)r1 r2 r3 r\ r] rB r r r r r rJ rI r r r r5 r6 r7 r r s !+,C!D!D ","*+# # &2:&899 *
+I J J ","*-A"B"B *
+; < < [ [ , , [, 2( 2( [2( 2( 2(r6 r c J e Zd ZdZd e dddZdZd Z fdZ xZ S )_SectionHParserz3
_BaseSectionHParser + DEF-2617 workaround
r r )rr rs r count c F d| _ t | j | _ d S )NF)_open_matched_stringr _INCOMPLETE_RESULT_incomplete_resultselfs r7 __init__z_SectionHParser.__init__5 s# $)!"&t'>"?"? r6 c # K g } t | }| j d rL| j d }| j d }|dxx | j d z
cc< | ||f t | dk r|d |d |d }}}|d
k rEd |v rA|dxx |z
cc< | ||f t | j | _ ni|| j d< || j d< || j d< d| j d< n@|d |d }}| ||f t | j | _ |d
k od |v | _ nO# t $ rA | j d } | r | | j k r| j s t | | j d |z | }|d d t | dk rq| j d
|d | j d }| j d }| ||f t | j | _ nK| j d
|d |d | j d< | j dxx dz
cc< Y nw xY w|D ] \ }}||fV d S # |D ] \ }}||fV w xY w)
a
In case with correct line e.g.
Name: Message [key "val"] [key "val"] [key "val"]
if there is no data in self._incomplete_result:
returns result of _BaseSectionHParser.parse_name_value_tokens
else
returns name, value of self._incomplete_result
and name, value of current line
it is possible since this function is generator.
In case with incomplete line e.g.
Name: Message [key "val"] [key "val"] [key "val
if there is no data in self._incomplete_result:
collect name, value and incomplete part values
that was got from _BaseSectionHParser.parse_name_value_tokens
to self._incomplete_result and wait next line.
returns empty list
else:
does the same but
returns name, value of self._incomplete_result
In case with non 'name: value' format:
if there is no data in self._incomplete_result:
raises
else:
Concatenate incomplete result of previous
iteration with current line and parse it
by _BaseSectionHParser._parse_message.
if there is still incomplete line:
it updates current self._incomplete_result and
returns empty list
else:
returns name, value from collected data
r rr rs r# r r rP ra r z[MatchedStringN)superrI r r lenr r r rY _MAX_INCOMPLETE_MESSAGEr> r popr ) r rF r _res_name_valuerr rs
incomplete_count __class__s r7 rI z'_SectionHParser.parse_name_value_tokens9 sk B 3 "7722488D2 &w/
/ /7 09y!!!T%<=N%OO!!!
ufo...4yyA~~*.q'47DGZe9$$)9T)A)A)$$$
2$$$MM4-000.243J.K.KD++6:D+F3778D+G44 "1gtAwe
tUm,,,*.t/F*G*G' !>&6$&>
% %Y 6 6 6,W5F
4Vd&B B B0 41333&&'(9:TA4 D
GKK " " "4yyA~~'077Q@@@.v6 /8
tUm,,,*.t/F*G*G'''077Q@@@=A!W'(9:'000A5000+ 6` &
"
" eEk!!!!
"
"v
"
" eEk!!!!
"s* !E1 E K 1E J=:K <J==K K )
r1 r2 r3 r4 r r r r rI
__classcell__)r s @r7 r r ( s
@ @ @U" U" U" U" U" U" U" U" U"r6 r c D e Zd Z ej d Zed ZdS )_SectionZParserz^-{2,3}\w+-{1,3}Z--$c t d )Nz&Expecting this method never be called.NotImplementedErrorrD s r7 rI z'_SectionZParser.parse_name_value_tokens s !"JKKKr6 Nr~ r5 r6 r7 r r sE !+,C!D!D L L [L L Lr6 r c D e Zd Z ej d Zed ZdS ) _SectionNotInterestedInParserz^-{2,3}\w+-{1,3}[^ABHZ]--$c t r| )r> rD s r7 rI z5_SectionNotInterestedInParser.parse_name_value_tokens s %'''r6 Nr~ r5 r6 r7 r r s@ !+,I!J!J ( ( [( ( (r6 r c e Zd Zd d edd D Zed Zd Ze d Z
e d Z d S )
Parserr c # Z K | ]&}t | d k t | V 'dS )%N)chr).0cs r7 zParser. sD
q66S==
A === r6 c dS )z
:param bytes_: audit log line
:return dict: for complete result and None for incomplete.
See unit tests for result dict fields.
:raise ParseError:
Nr5 r bytes_s r7 feedz Parser.feed s
r6 c d S r| r5 r s r7 flushz Parser.flush s r6 c | | S # t $ r) t j || j cY S w xY w)z6
apply fallback on UnicodeDecodeError
)safe) decode_bytesUnicodeDecodeErrorrk rl quote_from_bytes_SAFE_UNQUOTED)rE r s r7 _adaptive_decode_bytesz Parser._adaptive_decode_bytes sg
##F++ +! <00S / 1
s 0A
A
c * | S )zD
because cannot mock 'read-only' bstr.decode() attr
)decode)r s r7 r zParser.decode_bytes s
}} r6 N)
r1 r2 r3 r ranger r r r rJ r staticmethodr r5 r6 r7 r r s WW
N
^
[ \ r6 r c 6 e Zd ZdZg ZddZd Zd Zd Zd Z dS ) _RevolverParser Nc T || _ | | j | _ dS zn
:param str audit_logdir_path: audit log dir path for ModSecurity
concurrent mode
N)_audit_logdir_path_revolve_SKIP_FIRST_NUM_ERRORS_skip_first_num_errorsr audit_logdir_paths r7 r z_RevolverParser.__init__ s*
#4
&*&A###r6 c | | }| rd| _ n-# t $ r | j dk r| xj dz c_ Y d S w xY w|S )Nr rP )
_feed_implr r r. )r r r s r7 r z_RevolverParser.feed s
0__V,,F ||~~
0 /0+ *Q..++q0++tt
s 3 %A A c 4 | j S r| )_current_parserr r s r7 r z_RevolverParser.flush s #))+++r6 c 8 t dt | j dz D ]s} | j | c S # t
$ rH}|t | j k r| nt d| |Y d }~ld }~ww xY wd S )NrP z:Neither of available parsers is capable to parse this: %r )r r _available_parsersr r r. r )r r attemptes r7 r z_RevolverParser._feed_impl s QD$; < BBc | t | j dk s
J d t d| j d j | j d j | j dd | j dd z | _ | j d }t }| j
| j |d< |di || _ dS ) zExchange parser to the nextra z4Count of available_parsers should not be less than 2z Swap %s<->%sr rP Nr r5 )r r rS infor1 r r r )r rE
parser_kwargss r7 r z_RevolverParser._revolve s
' ( (A - - - A
. - - #A & / #A & /
#ABB '$*A"1"*E E
%a(
" .151HM- . "s 3 3] 3 3 r6 r| )
r1 r2 r3 r r r r r r r r5 r6 r7 r r sv B B B B $, , ,
4 4 4 4 4r6 r c 0 e Zd Zd ZdedefdZd Zd ZdS )_JsonLogParserBasec " i | _ g | _ d S r| )_accumulating_result_accumulating_linesr rw s r7 r z_JsonLogParserBase.__init__ s $&!#% r6 datareturnc t r| r )r r s r7 parse_json_dataz"_JsonLogParserBase.parse_json_data s !###r6 c v | | }|r | j | | t j | | _ | S # t $ r Y d S t $ r}t d|d| |d }~ww xY wd S )N#Error occurs while parse json line
, reason: ) r rstripr
r r jsonloadsr r r> Exceptionr. r r rF r s r7 r z_JsonLogParserBase.feed! s **6==??;;
(//555,0,@,@Jt$$- -) zz||#*
tt
jttQQ
s AB
B6 B6B11B6c | t | j | j i | _ g | _ S # i | _ g | _ w xY wN) debug_ctx)mk_modsec_indicent_listr r
r s r7 r z_JsonLogParserBase.flush2 sX **)T5M )+D %')D $ $ )+D %')D $ ) ) ) )s + ;N)r1 r2 r3 r r r r r r5 r6 r7 r
r
sc & & &$D $T $ $ $ $ "* * * * *r6 r
c " e Zd ZdZdedefdZdS )_JsonLogParserv2z&Parse ModSecurity v2 json log entries.r r c h d}| | st d i }|d }t j } t t
j |d |t < n=# t $ r0 t d|d t w xY w|
dd |d< g |t < |d
d i D ]<\ }}t! || \ }} | r"|t ||g =|d
d
r|d d
^}
} } |
rut( j | rV|
|t. < t0 j | }
|
j |t6 < |r t9 |
j |t< < |d
d r8|r6t9 d |d d |t@ < |d
d r|d
d |tB < |d
d rqtE |d d t
r$tG |d d \ }} ||d< n,|d d ^}}tG | \ }} ||d< ||d< |d
d r|d d |tH < |d
d rd |d d D |d< |S )a
The contents of the log entry can be seen here
https://github.com/SpiderLabs/ModSecurity/blob/v2/master/apache2/msc_logging.c#L644
Optional fields may be omitted due to SecAuditLogParts setting.
Expected data (some unused keys are omitted):
{'transaction': {
'time': str,
'transaction_id': str,
'remote_address': str,
'remote_port': int,
'local_address': str,
'local_port': int
},
'request': {
'request_line': str, # optional field
'body': [], # optional field
'headers': {}, # optional field
},
'response': {
'protocol': str, # optional field
'body': str, # optional field
'headers': {}, # optional field
'status': int, # optional field
},
'audit_data': {
'producer': str or [], # optional field
'messages': [], # optional field
'error_messages': [], # optional field
'engine_mode': str, # optional field
}
}
> requestresponse
audit_data transactionz)Not expected json data for ModSecurity v2r% remote_address&Cannot use %s for IPv4 or IPv6 addressr -r" r request_linebodyr r# statusr$ producerr( r) r messagesc P g | ]#}t |d d $S )r r )r r )r r" s r7
z4_JsonLogParserv2.parse_json_data.. s= % % % ..sB77:% % %r6 MessageList)%issubsetkeysr. r ro rZ rV rW ATTACKERS_IPrY rS rT r> r rd itemsrc r re rf rg rh ri rj rk rl rm rn rq r rp r r} r r get_producer_datar )r r expected_data_keysr r% additional_data_is_requiredheaderrs rt ru rv r rw rx versionmodsec_full_namevendorss r7 r z _JsonLogParserv2.parse_json_data? s B
"**499;;77 J HII I=) &2&G# ,#&$[1A%BCC$ $F< , , , NN8 ,-
*++ + , $/??3CS#I#I w !)_ 0 0B ? ? E E G G I IMFE 4FEBB
&
Iw&&0F'GHHH ? ~ . . C "9on = C C E EOFC!~~
CBGMM#$6$6
C&,{# ..s33 $ks . C+4V\+B+BF<( ? v & & N+F N"+BGGDOF4K,L,L"M"MF;
) ) ="&z"28">
3 .tL/A*/M N N
+2'((-1,-?
-K* 7 ./? @ @
#*x +2'( ! !- 0 0 D"&|"4]"CF; ! !* - - % % -j9% % %F= !
s /B :B<N)r1 r2 r3 r4 r r r5 r6 r7 r r < sD 00aD aT a a a a a ar6 r c , e Zd ZdZi Zd ZdedefdZdS )_KeyMappedJsonLogParserv3z&Parse ModSecurity v3 json log entries.c t |t r fd| D S t |t r fd|D S |S )z9
Assume that *obj* is json serializeble.
c i | ]G\ }}| | d k r | n|HS )r )lower_to_lower_keys)r kvr s r7
z<_KeyMappedJsonLogParserv3._to_lower_keys.. s_ Aq ./ggii9.D.DD''***! r6 c : g | ]} | S r5 )rA )r itemr s r7 r/ z<_KeyMappedJsonLogParserv3._to_lower_keys.. s' >>>$D''-->>>r6 )r r r4 r )r objs ` r7 rA z(_KeyMappedJsonLogParserv3._to_lower_keys s c4 IIKK
T
"
" >>>>#>>> >Jr6 r r c j | j st d ddi}||k r(t d t dh}| |k rt
d | | }i }|d }t j } t t j |d |t < n=# t $ r0 t d|d t w xY w| | j d d
}|t
d t | |d < g |t < |d d
i D ]<\ }} t% || \ }
} | r"|t ||
g =|d d |t( < t* j |d d } | j |t2 < |r t5 | j |t8 < |d d r%|r#t5 |d d |t: < |d | j d |t< < | d rb|d d |d< t? |d d \ }
}|
|d< |d | j d |t@ < | d rg |d< |d D ]}|d }t |! d |d < d |v r|d n| d |d < |d d
}| d p| d! }||d"< |d d#k |d$< | d r(|" tG |d |d | |S )%a
Parse log entry data for JSON SecAuditLogFormat.
https://github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual-(v2.x)#SecAuditLogFormat
The contents of the log entry can be seen here
https://github.com/SpiderLabs/ModSecurity/blob/v3/master/src/transaction.cc#L1622
Optional fields may be omitted due to SecAuditLogParts setting.
Expected data:
{'transaction': {
'client_ip': str,
'time_stamp': str,
'server_id': str,
'client_port': int,
'host_ip': str,
'host_port': int,
'unique_id': str,
'request': {
'http_version': str,
'method': str,
'uri': str,
'body': str, # optional field
'headers': {}, # optional field
},
'response': {
'body': str, # optional field
'headers': {}, # optional field
'http_code': int,
},
# optional fields below
'producer': {
'components': [],
'connector': str,
'modsecurity': str,
'secrules_engine': str,
},
'messages': [{'details': {'accuracy': str,
'data': str,
'file': str
'lineNumber': str,
'match': str,
'maturity': str,
'reference': str,
'rev': '1',
'ruleId': str,
'severity': str,
'tags': [],
'ver': str},
'message': str}]}}
z7_KeyMappedJsonLogParserv3 misses KEYS_MAPPER definitionerrorz/ModSecurity was not compiled with JSON support.zRSecAuditLogFormat set to JSON, but ModSecurity was not compiled with JSON support.r% z/Not expected json data for Coraza/ModSecurityV3 client_ipr' r Nz=Missing unique id field in json data for Coraza/ModSecurityV3r" r rv r r* r# r r,
componentsr) modsecurityr( secrules_enginer- r0 detailsruleidr! r" r# r hosthostnamei r$ )$ KEYS_MAPPERr rS rT r> r2 r. rA r ro rZ rV rW r3 rY r rd r4 rc r rj rk rl rm rh rn rq r rp r} r r5 r r r r )r r
error_datar6 r r% r7 r r8 rs rt ru rx r9 rw r" r# r rQ s r7 r z)_KeyMappedJsonLogParserv3.parse_json_data s b %I
F
: NNB
*++ + +_ 99;;, , , NOO O ""4((=) &2&G# ,#&$[%=>>$ $F< , , , NN8K(
*++ + , % - .
! O
$'~#6#6 w ( 3 7 7 2 F F L L N N I IMFE 4FEBB
&
Iw&&0F'GHHH )) 4X >{ &&{9'=e'DEE ks & ;#,V\#:#:F< y ! % %f - - L2M L"+K ,B6,J"K"KF; )* 5 9 9 ] +
{ ??: & & *: 6| DF8 *J'
6 JGQ (/F# $"-j"9"="= !23# #F; ??: & & 6$&F= !":.
6
6 i."%gkk(&;&;"<"<"'3,,CJJCGGI4F4F &i0;";;v..E'++f2E2E&.
#+1-+@C+G(;;u%% NNN# r r s r7 r z_JsonPrettyLogParser.feed_ s` **6==??;; s{{+ /1D,$B / $E
$ + +D 1 1 1s{{ 040D0D
2774+C#D#DEE1 1D- ::<<'. 44 $* 44$ / {s AC
D* D3D DN)r1 r2 r3 r r5 r6 r7 r^ r^ ^ s# r6 r^ c H e Zd ZdZeeeeee e
fZ d Z d Z
d Zd Zd Zd ZdS ) _NativeLogParsera
audit log parser state machine
Parse log entry ModSecurity 2 Data for Native audit log format.
https://github.com/SpiderLabs/ModSecurity/wiki/ModSecurity-2-Data-Formats#Audit_Log
More about ModSecurity log message format
https://gerrit.cloudlinux.com/plugins/gitiles/defence360/+/refs/changes/73/111973/1/opt/DEF-21076-multiline-modsec-header-field-alert/README.txt
Expected data:
--2172df61-A--
[02/Feb/2018:11:49:22 +0000] WnRQQvAURie7wq9bOfH25AAAAAE ::1 57480 ::1 80
--2172df61-B--
POST /1/request?x=yxz&z=xy HTTP/1.1
User-Agent: Nessus
Host: localhost
Accept: */*
cOOkIe: ABC=abc;SESSIONID=hash256
Content-Length: 9
Content-Type: application/x-www-form-urlencoded
--2172df61-C--
bc=d123&cd=e&bc=a0
--2172df61-F--
HTTP/1.1 404 Not Found
Accept-Ranges: bytes
Transfer-Encoding: chunked
Content-Type: text/html
--2172df61-H--
Message: Warning. Matched phrase "nessus" at REQUEST_HEADERS:User-Agent. [file "/etc/apache2/conf.d/modsec_vendor_configs/imunify360_full_apache/103_Global_Agents.conf"] [line "17"] [id "210801"] [rev "2"] [msg "COMODO WAF: Request Indicates a Security Scanner Scanned the Site||localhost|F|2"] [data "nessus"] [severity "CRITICAL"] [tag "CWAF"] [tag "Agents"]
Message: Warning. Pattern match "(?i:(?:^(?:microsoft url|user-Agent|www\.weblogs\.com|(?:jakart|vi)a|(google|i{0,1}explorer{0,1}\.exe|(ms){0,1}ie( [0-9.]{1,}){0,1} {0,1}(compatible( browser){0,1}){0,1})$)|\bdatacha0s\b|; widows|\\r|a(?: href=|d(?:sarobot|vanced email extractor ..." at REQUEST_HEADERS:User-Agent. [file "/etc/apache2/conf.d/modsec_vendor_configs/imunify360_full_apache/103_Global_Agents.conf"] [line "29"] [id "210831"] [rev "2"] [msg "COMODO WAF: Rogue web site crawler||localhost|F|4"] [data "Nessus"] [severity "WARNING"] [tag "CWAF"] [tag "Agents"]
Message: Warning. Operator GE matched 5 at TX:incoming_points. [file "/etc/apache2/conf.d/modsec_vendor_configs/imunify360_full_apache/122_Outgoing_FiltersEnd.conf"] [line "35"] [id "214930"] [rev "1"] [msg "COMODO WAF: Inbound Points Exceeded|Total Incoming Points: 8|localhost|F|2"] [severity "CRITICAL"] [tag "CWAF"] [tag "FiltersEnd"]
Apache-Handler: default-handler
Stopwatch: 1517572162346260 98908 (- - -)
Stopwatch2: 1517572162346260 98908; combined=33129, p1=737, p2=32228, p3=0, p4=0, p5=163, sr=0, sw=1, l=0, gc=0
Producer: ModSecurity for Apache/2.9.2 (http://www.modsecurity.org/); CWAF_Apache.
Server: Apache
Engine-Mode: "DETECTION_ONLY"
--2172df61-Z--
c 0 d | _ i | _ g | _ d S r| )
_state_parserr r
r s r7 r z_NativeLogParser.__init__ s !$&!#% r6 c ` | j | | | S r| )r
r r r s r7 r z_NativeLogParser.feed s+ ''///v&&&r6 c 6 | | sd S t fd| j D }|t u r| S | | _ d S # t $ r | j t d Y nw xY w| }|D ]\ }}|dk r/| j
dg | :|dk r8d| j
vr/| j
dg | x|t k r4| j
t g | |dk r| | || j
|< d S )Nc 3 F K | ]}| |V d S r| )rG )r parserrF s r7 r z._NativeLogParser._feed_impl.. sK --d33 r6 zNo parser for line %rr r0 r r' )r r next_AVAILABLE_SUBPARSERSr r re
StopIterationr. rI r
setdefaultr rd _parse_producer_filed)r r next_parsername_value_tokensrr rs rF s @r7 r z_NativeLogParser._feed_impl s **6==??;; 4 " 8 K o--zz||# . "- D 4
@ @ @!) !8$??? *) @ ! 8 8 > >, 8 8KD%y )44]BGGNN
%%!)BBB)44]BGGNN )44WbAAHHOOOO##**5111127)$//ts r: )r rF r s r7 rI z(_NativeLogParser.parse_name_value_tokens s| *BB4HHII I , , , SVV$$! +');< III s &)
A) AA)(A)c t | j | j d | _ i | _ g | _ S # d | _ i | _ g | _ w xY wr )r r r
re r s r7 r z_NativeLogParser.flush sh **)T5M "&D (*D %')D $ $ "&D (*D %')D $ ) ) ) )s 2 A N)r1 r2 r3 r4 rM r_ rz r r r r rk r r r rn rI r r5 r6 r7 rc rc s ) )X % & & &
' ' '- - -^: : : "* * * * *r6 rc c 6 e Zd ZdZeeeeegZ e
d Z dS )SerialLogParserr c | }t |d 5 }t | D ]T\ }} | | }||c cddd S .# t $ r}t d||dz |d}~ww xY w ddd n # 1 swxY w Y t
d||j j | S )zdo-it-all stylerbNz"Error in audit log file %r line %drP z Incomplete audit log file %r: %r)
open enumerater r. r< rS rI r r
r )rE filepathri
filestreamlinenor r r s r7
parse_filezSerialLogParser.parse_file s]
(D
!
! Z"+J"7"7
#[[ 0 0F ) %
* " ,<
" . " 6
||~~ s: B
A B
B
A=#A88A==B
BBN) r1 r2 r3 r r rU rc r[ r^ r rJ r~ r5 r6 r7 rv rv sL [ r6 rv c & e Zd Zd Zd Zd Zd ZdS )ConcurrentLogParserc || _ dS r r r s r7 r z ConcurrentLogParser.__init__8 s
#4 r6 c | j |z S )K
:param str: token is expected to start with posixpath.sep
r r r s r7 _normconcatz ConcurrentLogParser._normconcat? s &..r6 c | | j t j z | t j d z |z S )r rP )r posixpathsepre r s r7 _directadmin_concatz'ConcurrentLogParser._directadmin_concatE s>
#m
kk)-((+
,
r6 c | }t d | D }t t j j | }t || j | j f D ]%\ }} d } || }t j
| r|}n!t j
| r|}t d
||ft | t t 5 |rKt j | dk r(t$ | cd d d c S d d d n # 1 swxY w Y # t( t* f$ r Y #w xY wn# t, $ r Y nw xY wt/ d| )Nc 8 g | ]}| d S )z[]r )r ts r7 r/ z,ConcurrentLogParser.feed..X s" 'L'L'L!
'L'L'Lr6 zos.path.isfile({!r}) = {!r}r zNo audit log found in %r)r reversedre filterrg rh ri r r r isfilerS debugformatboolr FileNotFoundErrorgetsizerv r~ UnicodeEncodeErrorrY r r. ) r r str_reversed_tokensfiltered_tokensr
concat_funlog_path
token_paths r7 r zConcurrentLogParser.feedP sX H==??D ''L'Ltzz||'L'L'L M MO $RW]O D DO%, $"2D4L!M& &
H
H!zH #H!+E!2!2Jw~~j11 )#- .. )#( LL5<< ' /h
""344 H H# H(A(AA(E(E#2#=#=h#G#GH H H H H H H H H H H H H H H H H H H H H H H H H +J7 D
H "
D @ 3V<<>F F F F $
F10F1N)r1 r2 r3 r r r r r5 r6 r7 r r 7 sP 4 4 4/ / /
#= #= #= #= #=r6 r c e Zd ZeegZdS )RevolverParserN)r1 r2 r3 rv r r r5 r6 r7 r r v s )+>?r6 r c e Zd Z G d de Zed Zed Zed Zed Z ed Z
ed Z ed Z d
S ) _IncidentFixupListc e Zd ZdZdS )"_IncidentFixupList.InvalidIncidentzh
For incidents we cannot use data from, e.g.
"Message: Rule processing failed."
Nr0 r5 r6 r7 InvalidIncidentr { s
r6 r c # ` K |D ]}| | } | | | | | | | | |V q# t
j $ r& d|d v r| | |V Y w xY wd S )Nz[msg r# )_fixup_camel_case_fixup_msg_inplace_fixup_host_tag_inplace_fixup_severity_inplace_fixup_useragent_inplacer r _fixup_unparsed_message)rE incidentsr incidents r7 applyz_IncidentFixupList.apply s ! # #H,,X66H
#&&x000++H555++H555,,X666 %5
#
#
#hy 111//999"NNN
# # #s AA662B+*B+c d dg|d< d|d< |d d d |d< d S )Nnoshowr& r r# z[msgra ) partitionrE r s r7 r z*_IncidentFixupList._fixup_unparsed_message s? #* &y 1 ; ;F C CA Fr6 c > d | D S )zH
make "Host" be in the same case as "msg", "rule", etc.
c > i | ]\ }}| |S r5 )r@ r rB rC s r7 rD z8_IncidentFixupList._fixup_camel_case.. s& :::A 1:::r6 )r4 r s r7 r z$_IncidentFixupList._fixup_camel_case s"
;: )9)9::::r6 c | d | | d }||d< | dd }|d |d< d S )Nr" r# z||rP r r rr )r r r re )rE r r" partss r7 r z%_IncidentFixupList._fixup_msg_inplace sl << &%%'' ' ,,u%%C"%HY IIdQI//E $QxHV r6 c B d|v r| d |d< dS dS )zR
fixup 'user-agent' to 'useragent' to match agent sqlitedb naming
z
user-agent
user_agentN)r r s r7 r z+_IncidentFixupList._fixup_useragent_inplace s3
8 # #%-\\,%?%?H\ " " " $ #r6 c d|v rgt d | t i t g D d rfd|d D |d< dS dS dS )zB
remove "Host: %hostname%" field duplicate in tag
r& c 3 , K | ]\ }}|d k |V dS )r Nr5 )r r8 rs s r7 r z=_IncidentFixupList._fixup_host_tag_inplace.. sA % '' (''' r6 Nc & g | ]
}|d z k |S )zHost: %sr5 )r r& rP s r7 r/ z>_IncidentFixupList._fixup_host_tag_inplace.. s/ # # # cZ$=N6N6NC6N6N6Nr6 )rj r ADVANCEDrd )rE r rP s @r7 r z*_IncidentFixupList._fixup_host_tag_inplace s
H )1 h)C)C)G)G * * D
# # # ##+E?# # #
r6 c R ddddddddd }| d
}|d S t | |d
< d S # t $ r Y nw xY w || |d
< d S # t $ rD t d t
| t
t j Y d S w xY w)
a
map modsec severity string <-> ossec modsec severity int,
incident table expects int for severity and
also consistent severity level is good for ML.
github.com/SpiderLabs/ModSecurity/wiki/Reference-Manual#severity
r rP ra r r r ) EMERGENCYALERTCRITICALERRORWARNINGNOTICEINFODEBUGr Nz9Cannot measure severity level for %s literal in %s plugin) r intrY KeyErrorrS rI rU r PLUGIN_ID)rE r map_r s r7 r z*_IncidentFixupList._fixup_severity_inplace s #
& <<
++ F #&x==HZ F D #'>HZ LLKX\+,,
s" :
AA A A
B&%B&N)
r1 r2 r3 RuntimeErrorr rJ r r r r r r r r5 r6 r7 r r z s
,
# # [ # G G [G
; ; [; ( ( [ ( @ @ [@ [, + + [+ + +r6 r c g }d v r6t t d t t g } d D ]ފdt
j d}t fdt D }| | t t g i|t < t t fD ]-} | r | |t |< .| t |i |
| g t || }|S )z
unroll modsec audit log into incident list
:param top_level_tokens: see how unit test describe this data structure
:param debug_ctx: to be shown in sentry incident
:return list:
r0 r INCIDENT)rv plugin_idc 3 K | ]8}|v s|v
| | p | fV 9d S r| )r )r fieldmessage_enclosing_tokenstop_level_tokenss r7 r z*mk_modsec_indicent_list.. sw 444 ,,,
,0077 3'++E22 -,,, r6 )r r r3 r rd r r PICK_SECAUDITLOG_FIELDSr r rn rj r r r r ) r r
raw_incidentsuser_idr update_dictr r r s ` @r7 r r s M((( r 2 2 !%%gr22 3 3
)9 (G + + $$ ) 3 H 5
K
OOK ( ( ( #*+;+?+?+L+L!MHX k)
H
H#''.. H0@0GHX&u- OO 0' : ; ; ; * * * *
B!''
yAA
BF Mr6 r c i }t | D ]7\ }}d| z }|t v s
J d|z |||< 8|S )Nz%s_anomality_scorez'invalid anomality score key %s detected)_PARSE_SCORE_REGEXr r@ ANOMALITY_SCORE_FIELDS)r" r
score_typers keys r7 r r * ss
F / 7 7 < <
E"Z%5%5%7%77,,,, 5 ; -,, s Mr6 rr rs c N t | | }||k p t j }||fS )zc
Return value/obfuscated value if header is (not) sensitive
and whether it is required
)obfuscate_item_if_sensitiver ro )rr rs result_valueru s r7 rc rc 5 s0 /tU;;L %'L<+LK + &&r6 c | } | t k rt | n| t v rt | n|S )z|
If header name is 'authorization' or 'cookie', then obfuscate it
so as not to disclosure sensitive client info
)r@ _COOKIEobfuscate_cookie_SENSITIVE_HEADERSobfuscate_item)rr rs s r7 r r A sR
::<.V s+ MMM$!Q!^AG,,-MMMr6 )r sortedr4 r rZ )r* scrs r s r7 r r P sz N
&
!
! NM&:L:LMMM A Ls =
A AA r c i }| D ]@\ }}|D ]8}| |g t | 9A|S r| )r4 rm r r )r r rB r4 rF s r7 _obfuscate_itemsr Z so
FJJLL B B5 B BD a $ $ + +N4,@,@ A A A A B Mr6 c : t t | S r| )r r
)r s r7 rq rq b s HUOO , ,,r6 rF c z | s| S g }d}dddddd}t | } |t | k r| | }t | | v r/| |t | |dz
}nt | r|}|t | k rB| | }t | sn|dz
}|t | k B|}||z
dk r| d n| d
||z
z $t | r|}|t | k rB| | }t | sn|dz
}|t | k B|}||z
dk r| d n| d ||z
z | | |dz
}|t | k d
| S )Nr z[space]z[tab]z[LF]z[CR]z[NULL])r
r rP z[chr]z [chr]{%d}z[digit]z [digit]{%d}r ) r r ordr2 r rZ isalpha isnumericr )rF obf_buffpos special_dictx start_posend_poss r7 r r f sP H
C L ::D
D // I q66\&&(( ( ( OOLQ 0 1 1 1 1HCC1vv~~
CIIoo S A q66>>++ 1HC CIIoo
i'A-- OOG,,,, OOK7Y3F$GHHH1vv !!
CIIoo S A q66++-- 1HC CIIoo
i'A-- OOI.... OOMWy5H$IJJJ OOA 1HCC D //F 778 r6 rF c d }d }d| v r6| d ^}}t t d | }n| }t j d| }|r| }||fS )N;c , | d S )Nz.| r )r s r7 z#get_producer_data.. s QWWU^^ r6 z
\d\.\d\.\d)re r mapr\ r rX )rF r; rr r9 rC s r7 r5 r5 s GJ
d{{ JJsOO's33W==>> ImW - -E #[[]]
w r6 r| )ar4 r rV rg os.pathr r\ urllib.parserk abcr r
contextlibr r http.cookiesr r itertoolsr loggingr
typingr r r
defence360agent.utils.commonr r defence360agent.utilsr r im360.contracts.configr r SEVERITYr rd rn rj r} rp r3 r r r r rw r r]
IGNORECASEr r1 rS exceptionr r r. r: r< r r> r@ rM r_ rz r r r r r r r r
r r= rU r[ r^ rc rv r r r r rZ r r r rc r r r rq r tupler5 r5 r6 r7 r s ' ' ' ' ' ' ' ' 2 2 2 2 2 2 2 2 " " " " " " " " ! ! ! ! ! ! 8 8 8 8 8 8 8 8 D D D D D D D D / / / / / / @ (GJ F[+|\;
#$ % & #> =
RZ/
8 0**C0001ABB i
$2 2 2 2 2- 2 2 22 0 0 0 0 0- 0 0 0>/ / / / /- / / /
.
.
.
.
.-
.
.
.e( e( e( e( e(1 e( e( e(Pf" f" f" f" f") f" f" f"RL L L L L- L L L( ( ( ( ($; ( ( (* * * * * w * * * * ZH4 H4 H4 H4 H4f H4 H4 H4V * * * * * * * *Fd d d d d) d d dNh h h h h 2 h h hV 0 4 + BN* N* N* N* N*v N* N* N*b$ $ $ $ $ o $ $ $ N<= <= <= <= <=& <= <= <=~@ @ @ @ @_ @ @ @| | | | | | | |~. . . .b3 4 '
' '
8C=$ ' ' ' '
4 - - -4 # 4 8C= 4 4 4 4 n C E r6