Unicode compilation issues
[public/netxms.git] / src / libnxlp / parser.cpp
CommitLineData
5039dede
AK
1/*
2** NetXMS - Network Management System
3** Log Parsing Library
2dd24569 4** Copyright (C) 2003-2012 Victor Kirhenshtein
5039dede
AK
5**
6** This program is free software; you can redistribute it and/or modify
65d2c384
VK
7** it under the terms of the GNU Lesser General Public License as published by
8** the Free Software Foundation; either version 3 of the License, or
5039dede
AK
9** (at your option) any later version.
10**
11** This program is distributed in the hope that it will be useful,
12** but WITHOUT ANY WARRANTY; without even the implied warranty of
13** MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14** GNU General Public License for more details.
15**
65d2c384 16** You should have received a copy of the GNU Lesser General Public License
5039dede
AK
17** along with this program; if not, write to the Free Software
18** Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
19**
20** File: parser.cpp
21**
22**/
23
24#include "libnxlp.h"
5039dede 25#include <expat.h>
5039dede
AK
26
27
28//
29// Context state texts
30//
31
ef856d02 32static const char *s_states[] = { "MANUAL", "AUTO", "INACTIVE" };
5039dede
AK
33
34
35//
36// XML parser state for creating LogParser object from XML
37//
38
39#define XML_STATE_INIT -1
40#define XML_STATE_END -2
41#define XML_STATE_ERROR -255
42#define XML_STATE_PARSER 0
43#define XML_STATE_RULES 1
44#define XML_STATE_RULE 2
45#define XML_STATE_MATCH 3
46#define XML_STATE_EVENT 4
47#define XML_STATE_FILE 5
48#define XML_STATE_ID 6
49#define XML_STATE_LEVEL 7
50#define XML_STATE_SOURCE 8
51#define XML_STATE_CONTEXT 9
52#define XML_STATE_MACROS 10
53#define XML_STATE_MACRO 11
54#define XML_STATE_DESCRIPTION 12
55
56
57typedef struct
58{
59 LogParser *parser;
60 int state;
61 String regexp;
62 String event;
63 String file;
64 String id;
65 String level;
66 String source;
67 String context;
68 String description;
69 int contextAction;
70 String ruleContext;
71 int numEventParams;
72 String errorText;
73 String macroName;
74 String macro;
6d738067
VK
75 bool invertedRule;
76 bool breakFlag;
5039dede
AK
77} XML_PARSER_STATE;
78
79
80//
81// Parser default constructor
82//
83
84LogParser::LogParser()
85{
86 m_numRules = 0;
87 m_rules = NULL;
88 m_cb = NULL;
89 m_userArg = NULL;
e464b7dc 90 m_name = NULL;
5039dede
AK
91 m_fileName = NULL;
92 m_eventNameList = NULL;
93 m_eventResolver = NULL;
94 m_thread = INVALID_THREAD_HANDLE;
95 m_recordsProcessed = 0;
96 m_recordsMatched = 0;
97 m_processAllRules = FALSE;
98 m_traceLevel = 0;
99 m_traceCallback = NULL;
e464b7dc 100 _tcscpy(m_status, LPS_INIT);
5039dede
AK
101}
102
103
104//
105// Destructor
106//
107
108LogParser::~LogParser()
109{
110 int i;
111
112 for(i = 0; i < m_numRules; i++)
113 delete m_rules[i];
114 safe_free(m_rules);
e464b7dc 115 safe_free(m_name);
5039dede
AK
116 safe_free(m_fileName);
117}
118
119
120//
8a5ba964 121// Trace
5039dede
AK
122//
123
fe1d4002 124void LogParser::trace(int level, const char *format, ...)
5039dede
AK
125{
126 va_list args;
127
128 if ((m_traceCallback == NULL) || (level > m_traceLevel))
129 return;
130
131 va_start(args, format);
132 m_traceCallback(format, args);
133 va_end(args);
134}
135
136
137//
138// Add rule
139//
140
6d738067 141bool LogParser::addRule(LogParserRule *rule)
5039dede 142{
6d738067 143 bool isOK;
5039dede 144
6d738067 145 isOK = rule->isValid();
5039dede
AK
146 if (isOK)
147 {
148 m_rules = (LogParserRule **)realloc(m_rules, sizeof(LogParserRule *) * (m_numRules + 1));
149 m_rules[m_numRules++] = rule;
150 }
151 else
152 {
153 delete rule;
154 }
155 return isOK;
156}
157
2dd24569 158bool LogParser::addRule(const char *regexp, DWORD eventCode, const char *eventName, int numParams)
5039dede 159{
2dd24569 160 return addRule(new LogParserRule(this, regexp, eventCode, eventName, numParams));
5039dede
AK
161}
162
163
164//
165// Check context
166//
167
ef856d02 168const char *LogParser::checkContext(LogParserRule *rule)
5039dede 169{
ef856d02 170 const char *state;
5039dede 171
6d738067 172 if (rule->getContext() == NULL)
32ec6391 173 {
6d738067
VK
174 trace(5, _T(" rule has no context"));
175 return s_states[CONTEXT_SET_MANUAL];
32ec6391 176 }
5039dede 177
fb986055 178 state = m_contexts.get(rule->getContext());
5039dede 179 if (state == NULL)
32ec6391 180 {
6d738067 181 trace(5, _T(" context '%s' inactive, rule should be skipped"), rule->getContext());
5039dede 182 return NULL; // Context inactive, don't use this rule
32ec6391 183 }
5039dede 184
6d738067 185 if (!_tcscmp(state, s_states[CONTEXT_CLEAR]))
32ec6391 186 {
6d738067 187 trace(5, _T(" context '%s' inactive, rule should be skipped"), rule->getContext());
32ec6391
VK
188 return NULL;
189 }
190 else
191 {
6d738067 192 trace(5, _T(" context '%s' active (mode=%s)"), rule->getContext(), state);
32ec6391
VK
193 return state;
194 }
5039dede
AK
195}
196
197
198//
6d738067 199// Match log record
5039dede
AK
200//
201
6d738067
VK
202bool LogParser::matchLogRecord(bool hasAttributes, const char *source, DWORD eventId,
203 DWORD level, const char *line, DWORD objectId)
5039dede
AK
204{
205 int i;
ef856d02 206 const char *state;
6d738067
VK
207 bool matched = false;
208
209 if (hasAttributes)
210 trace(2, _T("Match event: source=\"%s\" id=%u level=%d text=\"%s\""), source, eventId, level, line);
211 else
212 trace(2, _T("Match line: \"%s\""), line);
5039dede 213
5039dede
AK
214 m_recordsProcessed++;
215 for(i = 0; i < m_numRules; i++)
216 {
6d738067 217 trace(4, _T("checking rule %d \"%s\""), i + 1, m_rules[i]->getDescription());
7c09239b 218 if ((state = checkContext(m_rules[i])) != NULL)
5039dede 219 {
7c09239b
VK
220 bool ruleMatched = hasAttributes ?
221 m_rules[i]->matchEx(source, eventId, level, line, m_cb, objectId, m_userArg) :
222 m_rules[i]->match(line, m_cb, objectId, m_userArg);
223 if (ruleMatched)
5039dede 224 {
7c09239b
VK
225 trace(1, _T("rule %d \"%s\" matched"), i + 1, m_rules[i]->getDescription());
226 if (!matched)
227 m_recordsMatched++;
228
229 // Update context
230 if (m_rules[i]->getContextToChange() != NULL)
231 {
fb986055 232 m_contexts.set(m_rules[i]->getContextToChange(), s_states[m_rules[i]->getContextAction()]);
7c09239b
VK
233 trace(2, _T("rule %d \"%s\": context %s set to %s"), i + 1, m_rules[i]->getDescription(), m_rules[i]->getContextToChange(), s_states[m_rules[i]->getContextAction()]);
234 }
235
236 // Set context of this rule to inactive if rule context mode is "automatic reset"
237 if (!_tcscmp(state, s_states[CONTEXT_SET_AUTOMATIC]))
238 {
fb986055 239 m_contexts.set(m_rules[i]->getContext(), s_states[CONTEXT_CLEAR]);
7c09239b
VK
240 trace(2, _T("rule %d \"%s\": context %s cleared because it was set to automatic reset mode"),
241 i + 1, m_rules[i]->getDescription(), m_rules[i]->getContext());
242 }
243 matched = true;
244 if (!m_processAllRules || m_rules[i]->getBreakFlag())
245 break;
5039dede 246 }
5039dede
AK
247 }
248 }
249 if (i < m_numRules)
6d738067
VK
250 trace(2, _T("processing stopped at rule %d \"%s\"; result = %s"), i + 1,
251 m_rules[i]->getDescription(), matched ? _T("true") : _T("false"));
5039dede 252 else
6d738067 253 trace(2, _T("Processing stopped at end of rules list; result = %s"), matched ? _T("true") : _T("false"));
5039dede
AK
254 return matched;
255}
256
257
6d738067
VK
258//
259// Match simple log line
260//
261
262bool LogParser::matchLine(const char *line, DWORD objectId)
263{
264 return matchLogRecord(false, NULL, 0, 0, line, objectId);
265}
266
267
268//
269// Match log event (text with additional attributes - source, severity, event id)
270//
271
272bool LogParser::matchEvent(const char *source, DWORD eventId, DWORD level, const char *line, DWORD objectId)
273{
274 return matchLogRecord(true, source, eventId, level, line, objectId);
275}
276
277
5039dede
AK
278//
279// Set associated file name
280//
281
fe1d4002 282void LogParser::setFileName(const char *name)
5039dede
AK
283{
284 safe_free(m_fileName);
285 m_fileName = (name != NULL) ? _tcsdup(name) : NULL;
e464b7dc
VK
286 if (m_name == NULL)
287 m_name = _tcsdup(name); // Set parser name to file name
288}
289
290
291//
292// Set parser name
293//
294
295void LogParser::setName(const char *name)
296{
297 safe_free(m_name);
298 m_name = _tcsdup((name != NULL) ? name : CHECK_NULL(m_fileName));
5039dede
AK
299}
300
301
302//
303// Add macro
304//
305
fe1d4002 306void LogParser::addMacro(const char *name, const char *value)
5039dede 307{
fb986055 308 m_macros.set(name, value);
5039dede
AK
309}
310
311
312//
313// Get macro
314//
315
fe1d4002 316const char *LogParser::getMacro(const char *name)
5039dede
AK
317{
318 const TCHAR *value;
319
fb986055 320 value = m_macros.get(name);
5039dede
AK
321 return CHECK_NULL_EX(value);
322}
323
324
325//
326// Create parser configuration from XML
327//
328
5039dede
AK
329static void StartElement(void *userData, const char *name, const char **attrs)
330{
331 XML_PARSER_STATE *ps = (XML_PARSER_STATE *)userData;
332
333 if (!strcmp(name, "parser"))
334 {
335 ps->state = XML_STATE_PARSER;
6d738067
VK
336 ps->parser->setProcessAllFlag(XMLGetAttrBoolean(attrs, "processAll", false));
337 ps->parser->setTraceLevel(XMLGetAttrInt(attrs, "trace", 0));
e464b7dc
VK
338 const char *name = XMLGetAttr(attrs, "name");
339 if (name != NULL)
340 ps->parser->setName(name);
5039dede
AK
341 }
342 else if (!strcmp(name, "file"))
343 {
344 ps->state = XML_STATE_FILE;
345 }
346 else if (!strcmp(name, "macros"))
347 {
348 ps->state = XML_STATE_MACROS;
349 }
350 else if (!strcmp(name, "macro"))
351 {
352 const char *name;
353
354 ps->state = XML_STATE_MACRO;
355 name = XMLGetAttr(attrs, "name");
356 ps->macroName = CHECK_NULL_A(name);
357 ps->macro = NULL;
358 }
359 else if (!strcmp(name, "rules"))
360 {
361 ps->state = XML_STATE_RULES;
362 }
363 else if (!strcmp(name, "rule"))
364 {
365 ps->regexp = NULL;
6d738067 366 ps->invertedRule = false;
5039dede
AK
367 ps->event = NULL;
368 ps->context = NULL;
369 ps->contextAction = CONTEXT_SET_AUTOMATIC;
370 ps->description = NULL;
6d738067
VK
371 ps->id = NULL;
372 ps->source = NULL;
373 ps->level = NULL;
5039dede 374 ps->ruleContext = XMLGetAttr(attrs, "context");
6d738067 375 ps->breakFlag = XMLGetAttrBoolean(attrs, "break", false);
5039dede 376 ps->state = XML_STATE_RULE;
9eb288e6 377 ps->numEventParams = 0;
5039dede
AK
378 }
379 else if (!strcmp(name, "match"))
380 {
381 ps->state = XML_STATE_MATCH;
6d738067 382 ps->invertedRule = XMLGetAttrBoolean(attrs, "invert", false);
5039dede 383 }
d43aee56 384 else if (!strcmp(name, "id") || !strcmp(name, "facility"))
5039dede
AK
385 {
386 ps->state = XML_STATE_ID;
387 }
d43aee56 388 else if (!strcmp(name, "level") || !strcmp(name, "severity"))
5039dede
AK
389 {
390 ps->state = XML_STATE_LEVEL;
391 }
d43aee56 392 else if (!strcmp(name, "source") || !strcmp(name, "tag"))
5039dede
AK
393 {
394 ps->state = XML_STATE_SOURCE;
395 }
396 else if (!strcmp(name, "event"))
397 {
398 ps->numEventParams = XMLGetAttrDWORD(attrs, "params", 0);
399 ps->state = XML_STATE_EVENT;
400 }
401 else if (!strcmp(name, "context"))
402 {
403 const char *action;
404
405 ps->state = XML_STATE_CONTEXT;
406
407 action = XMLGetAttr(attrs, "action");
408 if (action == NULL)
409 action = "set";
410
411 if (!strcmp(action, "set"))
412 {
413 const char *mode;
414
415 mode = XMLGetAttr(attrs, "reset");
416 if (mode == NULL)
417 mode = "auto";
418
419 if (!strcmp(mode, "auto"))
420 {
421 ps->contextAction = CONTEXT_SET_AUTOMATIC;
422 }
423 else if (!strcmp(mode, "manual"))
424 {
425 ps->contextAction = CONTEXT_SET_MANUAL;
426 }
427 else
428 {
429 ps->errorText = _T("Invalid context reset mode");
430 ps->state = XML_STATE_ERROR;
431 }
432 }
433 else if (!strcmp(action, "clear"))
434 {
435 ps->contextAction = CONTEXT_CLEAR;
436 }
437 else
438 {
439 ps->errorText = _T("Invalid context action");
440 ps->state = XML_STATE_ERROR;
441 }
442 }
443 else if (!strcmp(name, "description"))
444 {
445 ps->state = XML_STATE_DESCRIPTION;
446 }
447 else
448 {
449 ps->state = XML_STATE_ERROR;
450 }
451}
452
453static void EndElement(void *userData, const char *name)
454{
455 XML_PARSER_STATE *ps = (XML_PARSER_STATE *)userData;
456
457 if (!strcmp(name, "parser"))
458 {
459 ps->state = XML_STATE_END;
460 }
461 else if (!strcmp(name, "file"))
462 {
6d738067 463 ps->parser->setFileName(ps->file);
5039dede
AK
464 ps->state = XML_STATE_PARSER;
465 }
466 else if (!strcmp(name, "macros"))
467 {
468 ps->state = XML_STATE_PARSER;
469 }
470 else if (!strcmp(name, "macro"))
471 {
6d738067 472 ps->parser->addMacro(ps->macroName, ps->macro);
5039dede
AK
473 ps->state = XML_STATE_MACROS;
474 }
475 else if (!strcmp(name, "rules"))
476 {
477 ps->state = XML_STATE_PARSER;
478 }
479 else if (!strcmp(name, "rule"))
480 {
2dd24569
VK
481 DWORD eventCode;
482#ifdef UNICODE
483 TCHAR *eventName = NULL;
484#else
485 const TCHAR *eventName = NULL;
486#endif
5039dede
AK
487 char *eptr;
488 LogParserRule *rule;
489
0bbab9d3 490 ps->event.trim();
2dd24569 491 eventCode = strtoul(ps->event, &eptr, 0);
5039dede 492 if (*eptr != 0)
2dd24569
VK
493 {
494 eventCode = ps->parser->resolveEventName(ps->event, 0);
495 if (eventCode == 0)
496 {
497#ifdef UNICODE
498 eventName = WideStringFromUTF8String(ps->event);
499#else
500 eventName = ps->event;
501#endif
502 }
503 }
ce7565e7 504 if (ps->regexp.isEmpty())
d43aee56 505 ps->regexp = _T(".*");
2dd24569
VK
506 rule = new LogParserRule(ps->parser, (const char *)ps->regexp, eventCode, eventName, ps->numEventParams);
507#ifdef UNICODE
508 safe_free(eventName);
509#endif
ce7565e7 510 if (!ps->ruleContext.isEmpty())
6d738067 511 rule->setContext(ps->ruleContext);
ce7565e7 512 if (!ps->context.isEmpty())
5039dede 513 {
6d738067
VK
514 rule->setContextToChange(ps->context);
515 rule->setContextAction(ps->contextAction);
5039dede 516 }
6d738067 517
ce7565e7 518 if (!ps->description.isEmpty())
6d738067
VK
519 rule->setDescription(ps->description);
520
ce7565e7 521 if (!ps->source.isEmpty())
6d738067
VK
522 rule->setSource(ps->source);
523
ce7565e7 524 if (!ps->level.isEmpty())
6d738067
VK
525 rule->setLevel(_tcstoul(ps->level, NULL, 0));
526
ce7565e7 527 if (!ps->id.isEmpty())
6d738067
VK
528 {
529 DWORD start, end;
530 TCHAR *eptr;
531
532 start = strtoul(ps->id, &eptr, 0);
533 if (*eptr == 0)
534 {
535 end = start;
536 }
537 else /* TODO: add better error handling */
538 {
539 while(!_istdigit(*eptr))
540 eptr++;
541 end = strtoul(eptr, NULL, 0);
542 }
543 rule->setIdRange(start, end);
544 }
545
546 rule->setInverted(ps->invertedRule);
547 rule->setBreakFlag(ps->breakFlag);
548
549 ps->parser->addRule(rule);
5039dede
AK
550 ps->state = XML_STATE_RULES;
551 }
552 else if (!strcmp(name, "match"))
553 {
554 ps->state = XML_STATE_RULE;
555 }
d43aee56 556 else if (!strcmp(name, "id") || !strcmp(name, "facility"))
5039dede
AK
557 {
558 ps->state = XML_STATE_RULE;
559 }
d43aee56 560 else if (!strcmp(name, "level") || !strcmp(name, "severity"))
5039dede
AK
561 {
562 ps->state = XML_STATE_RULE;
563 }
d43aee56 564 else if (!strcmp(name, "source") || !strcmp(name, "tag"))
5039dede
AK
565 {
566 ps->state = XML_STATE_RULE;
567 }
568 else if (!strcmp(name, "event"))
569 {
570 ps->state = XML_STATE_RULE;
571 }
572 else if (!strcmp(name, "context"))
573 {
574 ps->state = XML_STATE_RULE;
575 }
576 else if (!strcmp(name, "description"))
577 {
578 ps->state = XML_STATE_RULE;
579 }
580}
581
582static void CharData(void *userData, const XML_Char *s, int len)
583{
584 XML_PARSER_STATE *ps = (XML_PARSER_STATE *)userData;
585
586 switch(ps->state)
587 {
588 case XML_STATE_MATCH:
ce7565e7 589 ps->regexp.addString(s, len);
5039dede
AK
590 break;
591 case XML_STATE_ID:
ce7565e7 592 ps->id.addString(s, len);
5039dede
AK
593 break;
594 case XML_STATE_LEVEL:
ce7565e7 595 ps->level.addString(s, len);
5039dede
AK
596 break;
597 case XML_STATE_SOURCE:
ce7565e7 598 ps->source.addString(s, len);
5039dede
AK
599 break;
600 case XML_STATE_EVENT:
ce7565e7 601 ps->event.addString(s, len);
5039dede
AK
602 break;
603 case XML_STATE_FILE:
ce7565e7 604 ps->file.addString(s, len);
5039dede
AK
605 break;
606 case XML_STATE_CONTEXT:
ce7565e7 607 ps->context.addString(s, len);
5039dede
AK
608 break;
609 case XML_STATE_DESCRIPTION:
ce7565e7 610 ps->description.addString(s, len);
5039dede
AK
611 break;
612 case XML_STATE_MACRO:
ce7565e7 613 ps->macro.addString(s, len);
5039dede
AK
614 break;
615 default:
616 break;
617 }
618}
619
6d738067 620bool LogParser::createFromXml(const char *xml, int xmlLen, char *errorText, int errBufSize)
5039dede 621{
5039dede
AK
622 XML_Parser parser = XML_ParserCreate(NULL);
623 XML_PARSER_STATE state;
6d738067 624 bool success;
5039dede
AK
625
626 // Parse XML
627 state.parser = this;
628 state.state = -1;
629 XML_SetUserData(parser, &state);
630 XML_SetElementHandler(parser, StartElement, EndElement);
631 XML_SetCharacterDataHandler(parser, CharData);
3973f74b 632 success = (XML_Parse(parser, xml, (xmlLen == -1) ? (int)strlen(xml) : xmlLen, TRUE) != XML_STATUS_ERROR);
5039dede
AK
633
634 if (!success && (errorText != NULL))
635 {
636 snprintf(errorText, errBufSize, "%s at line %d",
637 XML_ErrorString(XML_GetErrorCode(parser)),
53c96e2d 638 (int)XML_GetCurrentLineNumber(parser));
5039dede
AK
639 }
640 XML_ParserFree(parser);
641
642 if (success && (state.state == XML_STATE_ERROR))
643 {
6d738067 644 success = false;
5039dede
AK
645 if (errorText != NULL)
646 strncpy(errorText, state.errorText, errBufSize);
647 }
648
649 return success;
5039dede
AK
650}
651
652
653//
654// Resolve event name
655//
656
fe1d4002 657DWORD LogParser::resolveEventName(const char *name, DWORD defVal)
5039dede
AK
658{
659 if (m_eventNameList != NULL)
660 {
661 int i;
662
663 for(i = 0; m_eventNameList[i].text != NULL; i++)
664 if (!_tcsicmp(name, m_eventNameList[i].text))
665 return m_eventNameList[i].code;
666 }
667
668 if (m_eventResolver != NULL)
669 {
670 DWORD val;
671
672 if (m_eventResolver(name, &val))
673 return val;
674 }
675
676 return defVal;
677}