7a92cda8556d904cc4b3d986b773de809a159e71
[public/netxms.git] / src / libnxlp / parser.cpp
1 /*
2 ** NetXMS - Network Management System
3 ** Log Parsing Library
4 ** Copyright (C) 2008-2010 Victor Kirhenshtein
5 **
6 ** This program is free software; you can redistribute it and/or modify
7 ** it under the terms of the GNU General Public License as published by
8 ** the Free Software Foundation; either version 2 of the License, or
9 ** (at your option) any later version.
10 **
11 ** This program is distributed in the hope that it will be useful,
12 ** but WITHOUT ANY WARRANTY; without even the implied warranty of
13 ** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 ** GNU General Public License for more details.
15 **
16 ** You should have received a copy of the GNU General Public License
17 ** along with this program; if not, write to the Free Software
18 ** Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19 **
20 ** File: parser.cpp
21 **
22 **/
23
24 #include "libnxlp.h"
25 #include <expat.h>
26
27
28 //
29 // Context state texts
30 //
31
32 static const TCHAR *s_states[] = { _T("MANUAL"), _T("AUTO"), _T("INACTIVE") };
33
34
35 //
36 // XML parser state for creating LogParser object from XML
37 //
38
39 #define XML_STATE_INIT -1
40 #define XML_STATE_END -2
41 #define XML_STATE_ERROR -255
42 #define XML_STATE_PARSER 0
43 #define XML_STATE_RULES 1
44 #define XML_STATE_RULE 2
45 #define XML_STATE_MATCH 3
46 #define XML_STATE_EVENT 4
47 #define XML_STATE_FILE 5
48 #define XML_STATE_ID 6
49 #define XML_STATE_LEVEL 7
50 #define XML_STATE_SOURCE 8
51 #define XML_STATE_CONTEXT 9
52 #define XML_STATE_MACROS 10
53 #define XML_STATE_MACRO 11
54 #define XML_STATE_DESCRIPTION 12
55
56
57 typedef struct
58 {
59 LogParser *parser;
60 int state;
61 String regexp;
62 String event;
63 String file;
64 String id;
65 String level;
66 String source;
67 String context;
68 String description;
69 int contextAction;
70 String ruleContext;
71 int numEventParams;
72 String errorText;
73 String macroName;
74 String macro;
75 bool invertedRule;
76 bool breakFlag;
77 } XML_PARSER_STATE;
78
79
80 //
81 // Parser default constructor
82 //
83
84 LogParser::LogParser()
85 {
86 m_numRules = 0;
87 m_rules = NULL;
88 m_cb = NULL;
89 m_userArg = NULL;
90 m_fileName = NULL;
91 m_eventNameList = NULL;
92 m_eventResolver = NULL;
93 m_thread = INVALID_THREAD_HANDLE;
94 m_recordsProcessed = 0;
95 m_recordsMatched = 0;
96 m_processAllRules = FALSE;
97 m_traceLevel = 0;
98 m_traceCallback = NULL;
99 }
100
101
102 //
103 // Destructor
104 //
105
106 LogParser::~LogParser()
107 {
108 int i;
109
110 for(i = 0; i < m_numRules; i++)
111 delete m_rules[i];
112 safe_free(m_rules);
113 safe_free(m_fileName);
114 }
115
116
117 //
118 // Trace
119 //
120
121 void LogParser::trace(int level, const TCHAR *format, ...)
122 {
123 va_list args;
124
125 if ((m_traceCallback == NULL) || (level > m_traceLevel))
126 return;
127
128 va_start(args, format);
129 m_traceCallback(format, args);
130 va_end(args);
131 }
132
133
134 //
135 // Add rule
136 //
137
138 bool LogParser::addRule(LogParserRule *rule)
139 {
140 bool isOK;
141
142 isOK = rule->isValid();
143 if (isOK)
144 {
145 m_rules = (LogParserRule **)realloc(m_rules, sizeof(LogParserRule *) * (m_numRules + 1));
146 m_rules[m_numRules++] = rule;
147 }
148 else
149 {
150 delete rule;
151 }
152 return isOK;
153 }
154
155 bool LogParser::addRule(const char *regexp, DWORD event, int numParams)
156 {
157 return addRule(new LogParserRule(this, regexp, event, numParams));
158 }
159
160
161 //
162 // Check context
163 //
164
165 const TCHAR *LogParser::checkContext(LogParserRule *rule)
166 {
167 const TCHAR *state;
168
169 if (rule->getContext() == NULL)
170 {
171 trace(5, _T(" rule has no context"));
172 return s_states[CONTEXT_SET_MANUAL];
173 }
174
175 state = m_contexts.get(rule->getContext());
176 if (state == NULL)
177 {
178 trace(5, _T(" context '%s' inactive, rule should be skipped"), rule->getContext());
179 return NULL; // Context inactive, don't use this rule
180 }
181
182 if (!_tcscmp(state, s_states[CONTEXT_CLEAR]))
183 {
184 trace(5, _T(" context '%s' inactive, rule should be skipped"), rule->getContext());
185 return NULL;
186 }
187 else
188 {
189 trace(5, _T(" context '%s' active (mode=%s)"), rule->getContext(), state);
190 return state;
191 }
192 }
193
194
195 //
196 // Match log record
197 //
198
199 bool LogParser::matchLogRecord(bool hasAttributes, const char *source, DWORD eventId,
200 DWORD level, const char *line, DWORD objectId)
201 {
202 int i;
203 const TCHAR *state;
204 bool matched = false;
205
206 if (hasAttributes)
207 trace(2, _T("Match event: source=\"%s\" id=%u level=%d text=\"%s\""), source, eventId, level, line);
208 else
209 trace(2, _T("Match line: \"%s\""), line);
210
211 m_recordsProcessed++;
212 for(i = 0; i < m_numRules; i++)
213 {
214 trace(4, _T("checking rule %d \"%s\""), i + 1, m_rules[i]->getDescription());
215 if ((state = checkContext(m_rules[i])) != NULL)
216 {
217 bool ruleMatched = hasAttributes ?
218 m_rules[i]->matchEx(source, eventId, level, line, m_cb, objectId, m_userArg) :
219 m_rules[i]->match(line, m_cb, objectId, m_userArg);
220 if (ruleMatched)
221 {
222 trace(1, _T("rule %d \"%s\" matched"), i + 1, m_rules[i]->getDescription());
223 if (!matched)
224 m_recordsMatched++;
225
226 // Update context
227 if (m_rules[i]->getContextToChange() != NULL)
228 {
229 m_contexts.set(m_rules[i]->getContextToChange(), s_states[m_rules[i]->getContextAction()]);
230 trace(2, _T("rule %d \"%s\": context %s set to %s"), i + 1, m_rules[i]->getDescription(), m_rules[i]->getContextToChange(), s_states[m_rules[i]->getContextAction()]);
231 }
232
233 // Set context of this rule to inactive if rule context mode is "automatic reset"
234 if (!_tcscmp(state, s_states[CONTEXT_SET_AUTOMATIC]))
235 {
236 m_contexts.set(m_rules[i]->getContext(), s_states[CONTEXT_CLEAR]);
237 trace(2, _T("rule %d \"%s\": context %s cleared because it was set to automatic reset mode"),
238 i + 1, m_rules[i]->getDescription(), m_rules[i]->getContext());
239 }
240 matched = true;
241 if (!m_processAllRules || m_rules[i]->getBreakFlag())
242 break;
243 }
244 }
245 }
246 if (i < m_numRules)
247 trace(2, _T("processing stopped at rule %d \"%s\"; result = %s"), i + 1,
248 m_rules[i]->getDescription(), matched ? _T("true") : _T("false"));
249 else
250 trace(2, _T("Processing stopped at end of rules list; result = %s"), matched ? _T("true") : _T("false"));
251 return matched;
252 }
253
254
255 //
256 // Match simple log line
257 //
258
259 bool LogParser::matchLine(const char *line, DWORD objectId)
260 {
261 return matchLogRecord(false, NULL, 0, 0, line, objectId);
262 }
263
264
265 //
266 // Match log event (text with additional attributes - source, severity, event id)
267 //
268
269 bool LogParser::matchEvent(const char *source, DWORD eventId, DWORD level, const char *line, DWORD objectId)
270 {
271 return matchLogRecord(true, source, eventId, level, line, objectId);
272 }
273
274
275 //
276 // Set associated file name
277 //
278
279 void LogParser::setFileName(const TCHAR *name)
280 {
281 safe_free(m_fileName);
282 m_fileName = (name != NULL) ? _tcsdup(name) : NULL;
283 }
284
285
286 //
287 // Add macro
288 //
289
290 void LogParser::addMacro(const TCHAR *name, const TCHAR *value)
291 {
292 m_macros.set(name, value);
293 }
294
295
296 //
297 // Get macro
298 //
299
300 const TCHAR *LogParser::getMacro(const TCHAR *name)
301 {
302 const TCHAR *value;
303
304 value = m_macros.get(name);
305 return CHECK_NULL_EX(value);
306 }
307
308
309 //
310 // Create parser configuration from XML
311 //
312
313 static void StartElement(void *userData, const char *name, const char **attrs)
314 {
315 XML_PARSER_STATE *ps = (XML_PARSER_STATE *)userData;
316
317 if (!strcmp(name, "parser"))
318 {
319 ps->state = XML_STATE_PARSER;
320 ps->parser->setProcessAllFlag(XMLGetAttrBoolean(attrs, "processAll", false));
321 ps->parser->setTraceLevel(XMLGetAttrInt(attrs, "trace", 0));
322 }
323 else if (!strcmp(name, "file"))
324 {
325 ps->state = XML_STATE_FILE;
326 }
327 else if (!strcmp(name, "macros"))
328 {
329 ps->state = XML_STATE_MACROS;
330 }
331 else if (!strcmp(name, "macro"))
332 {
333 const char *name;
334
335 ps->state = XML_STATE_MACRO;
336 name = XMLGetAttr(attrs, "name");
337 ps->macroName = CHECK_NULL_A(name);
338 ps->macro = NULL;
339 }
340 else if (!strcmp(name, "rules"))
341 {
342 ps->state = XML_STATE_RULES;
343 }
344 else if (!strcmp(name, "rule"))
345 {
346 ps->regexp = NULL;
347 ps->invertedRule = false;
348 ps->event = NULL;
349 ps->context = NULL;
350 ps->contextAction = CONTEXT_SET_AUTOMATIC;
351 ps->description = NULL;
352 ps->id = NULL;
353 ps->source = NULL;
354 ps->level = NULL;
355 ps->ruleContext = XMLGetAttr(attrs, "context");
356 ps->breakFlag = XMLGetAttrBoolean(attrs, "break", false);
357 ps->state = XML_STATE_RULE;
358 }
359 else if (!strcmp(name, "match"))
360 {
361 ps->state = XML_STATE_MATCH;
362 ps->invertedRule = XMLGetAttrBoolean(attrs, "invert", false);
363 }
364 else if (!strcmp(name, "id") || !strcmp(name, "facility"))
365 {
366 ps->state = XML_STATE_ID;
367 }
368 else if (!strcmp(name, "level") || !strcmp(name, "severity"))
369 {
370 ps->state = XML_STATE_LEVEL;
371 }
372 else if (!strcmp(name, "source") || !strcmp(name, "tag"))
373 {
374 ps->state = XML_STATE_SOURCE;
375 }
376 else if (!strcmp(name, "event"))
377 {
378 ps->numEventParams = XMLGetAttrDWORD(attrs, "params", 0);
379 ps->state = XML_STATE_EVENT;
380 }
381 else if (!strcmp(name, "context"))
382 {
383 const char *action;
384
385 ps->state = XML_STATE_CONTEXT;
386
387 action = XMLGetAttr(attrs, "action");
388 if (action == NULL)
389 action = "set";
390
391 if (!strcmp(action, "set"))
392 {
393 const char *mode;
394
395 mode = XMLGetAttr(attrs, "reset");
396 if (mode == NULL)
397 mode = "auto";
398
399 if (!strcmp(mode, "auto"))
400 {
401 ps->contextAction = CONTEXT_SET_AUTOMATIC;
402 }
403 else if (!strcmp(mode, "manual"))
404 {
405 ps->contextAction = CONTEXT_SET_MANUAL;
406 }
407 else
408 {
409 ps->errorText = _T("Invalid context reset mode");
410 ps->state = XML_STATE_ERROR;
411 }
412 }
413 else if (!strcmp(action, "clear"))
414 {
415 ps->contextAction = CONTEXT_CLEAR;
416 }
417 else
418 {
419 ps->errorText = _T("Invalid context action");
420 ps->state = XML_STATE_ERROR;
421 }
422 }
423 else if (!strcmp(name, "description"))
424 {
425 ps->state = XML_STATE_DESCRIPTION;
426 }
427 else
428 {
429 ps->state = XML_STATE_ERROR;
430 }
431 }
432
433 static void EndElement(void *userData, const char *name)
434 {
435 XML_PARSER_STATE *ps = (XML_PARSER_STATE *)userData;
436
437 if (!strcmp(name, "parser"))
438 {
439 ps->state = XML_STATE_END;
440 }
441 else if (!strcmp(name, "file"))
442 {
443 ps->parser->setFileName(ps->file);
444 ps->state = XML_STATE_PARSER;
445 }
446 else if (!strcmp(name, "macros"))
447 {
448 ps->state = XML_STATE_PARSER;
449 }
450 else if (!strcmp(name, "macro"))
451 {
452 ps->parser->addMacro(ps->macroName, ps->macro);
453 ps->state = XML_STATE_MACROS;
454 }
455 else if (!strcmp(name, "rules"))
456 {
457 ps->state = XML_STATE_PARSER;
458 }
459 else if (!strcmp(name, "rule"))
460 {
461 DWORD event;
462 char *eptr;
463 LogParserRule *rule;
464
465 ps->event.trim();
466 event = strtoul(ps->event, &eptr, 0);
467 if (*eptr != 0)
468 event = ps->parser->resolveEventName(ps->event);
469 if (ps->regexp.isEmpty())
470 ps->regexp = _T(".*");
471 rule = new LogParserRule(ps->parser, (const char *)ps->regexp, event, ps->numEventParams);
472 if (!ps->ruleContext.isEmpty())
473 rule->setContext(ps->ruleContext);
474 if (!ps->context.isEmpty())
475 {
476 rule->setContextToChange(ps->context);
477 rule->setContextAction(ps->contextAction);
478 }
479
480 if (!ps->description.isEmpty())
481 rule->setDescription(ps->description);
482
483 if (!ps->source.isEmpty())
484 rule->setSource(ps->source);
485
486 if (!ps->level.isEmpty())
487 rule->setLevel(_tcstoul(ps->level, NULL, 0));
488
489 if (!ps->id.isEmpty())
490 {
491 DWORD start, end;
492 TCHAR *eptr;
493
494 start = strtoul(ps->id, &eptr, 0);
495 if (*eptr == 0)
496 {
497 end = start;
498 }
499 else /* TODO: add better error handling */
500 {
501 while(!_istdigit(*eptr))
502 eptr++;
503 end = strtoul(eptr, NULL, 0);
504 }
505 rule->setIdRange(start, end);
506 }
507
508 rule->setInverted(ps->invertedRule);
509 rule->setBreakFlag(ps->breakFlag);
510
511 ps->parser->addRule(rule);
512 ps->state = XML_STATE_RULES;
513 }
514 else if (!strcmp(name, "match"))
515 {
516 ps->state = XML_STATE_RULE;
517 }
518 else if (!strcmp(name, "id") || !strcmp(name, "facility"))
519 {
520 ps->state = XML_STATE_RULE;
521 }
522 else if (!strcmp(name, "level") || !strcmp(name, "severity"))
523 {
524 ps->state = XML_STATE_RULE;
525 }
526 else if (!strcmp(name, "source") || !strcmp(name, "tag"))
527 {
528 ps->state = XML_STATE_RULE;
529 }
530 else if (!strcmp(name, "event"))
531 {
532 ps->state = XML_STATE_RULE;
533 }
534 else if (!strcmp(name, "context"))
535 {
536 ps->state = XML_STATE_RULE;
537 }
538 else if (!strcmp(name, "description"))
539 {
540 ps->state = XML_STATE_RULE;
541 }
542 }
543
544 static void CharData(void *userData, const XML_Char *s, int len)
545 {
546 XML_PARSER_STATE *ps = (XML_PARSER_STATE *)userData;
547
548 switch(ps->state)
549 {
550 case XML_STATE_MATCH:
551 ps->regexp.addString(s, len);
552 break;
553 case XML_STATE_ID:
554 ps->id.addString(s, len);
555 break;
556 case XML_STATE_LEVEL:
557 ps->level.addString(s, len);
558 break;
559 case XML_STATE_SOURCE:
560 ps->source.addString(s, len);
561 break;
562 case XML_STATE_EVENT:
563 ps->event.addString(s, len);
564 break;
565 case XML_STATE_FILE:
566 ps->file.addString(s, len);
567 break;
568 case XML_STATE_CONTEXT:
569 ps->context.addString(s, len);
570 break;
571 case XML_STATE_DESCRIPTION:
572 ps->description.addString(s, len);
573 break;
574 case XML_STATE_MACRO:
575 ps->macro.addString(s, len);
576 break;
577 default:
578 break;
579 }
580 }
581
582 bool LogParser::createFromXml(const char *xml, int xmlLen, char *errorText, int errBufSize)
583 {
584 XML_Parser parser = XML_ParserCreate(NULL);
585 XML_PARSER_STATE state;
586 bool success;
587
588 // Parse XML
589 state.parser = this;
590 state.state = -1;
591 XML_SetUserData(parser, &state);
592 XML_SetElementHandler(parser, StartElement, EndElement);
593 XML_SetCharacterDataHandler(parser, CharData);
594 success = (XML_Parse(parser, xml, (xmlLen == -1) ? (int)strlen(xml) : xmlLen, TRUE) != XML_STATUS_ERROR);
595
596 if (!success && (errorText != NULL))
597 {
598 snprintf(errorText, errBufSize, "%s at line %d",
599 XML_ErrorString(XML_GetErrorCode(parser)),
600 XML_GetCurrentLineNumber(parser));
601 }
602 XML_ParserFree(parser);
603
604 if (success && (state.state == XML_STATE_ERROR))
605 {
606 success = false;
607 if (errorText != NULL)
608 strncpy(errorText, state.errorText, errBufSize);
609 }
610
611 return success;
612 }
613
614
615 //
616 // Resolve event name
617 //
618
619 DWORD LogParser::resolveEventName(const TCHAR *name, DWORD defVal)
620 {
621 if (m_eventNameList != NULL)
622 {
623 int i;
624
625 for(i = 0; m_eventNameList[i].text != NULL; i++)
626 if (!_tcsicmp(name, m_eventNameList[i].text))
627 return m_eventNameList[i].code;
628 }
629
630 if (m_eventResolver != NULL)
631 {
632 DWORD val;
633
634 if (m_eventResolver(name, &val))
635 return val;
636 }
637
638 return defVal;
639 }