From 4a52cee10938e535569bce9e46f82d8e05977200 Mon Sep 17 00:00:00 2001 From: Arjen Baart Date: Fri, 25 Sep 2020 15:33:03 +0200 Subject: [PATCH] Fix PR10: Gcm_input may loose its input message --- ChangeLog | 8 ++++ doc/test.xml | 39 +++++++++-------- src/gcm_input/gcm_input.cpp | 102 +++++++++++++++++++++++++------------------- src/gcm_input/message.cpp | 51 ++++++++++++++++++---- src/gcm_input/message.h | 1 + src/gnucomo.conf | 3 ++ src/lib/database.cpp | 34 +++++++-------- test/Makefile.am | 3 +- test/gnucomo.conf | 23 ++++++++++ test/gnucomo_nodb.conf | 26 +++++++++++ test/notification_sendmail | 47 ++++++++++++++++++++ test/read_no_database | 49 +++++++++++++++++++++ test/t0010.errors | 12 ------ test/t0010.expect | 13 ------ test/t0010.sh | 39 ----------------- 15 files changed, 294 insertions(+), 156 deletions(-) create mode 100644 test/gnucomo.conf create mode 100644 test/gnucomo_nodb.conf create mode 100755 test/notification_sendmail create mode 100755 test/read_no_database delete mode 100644 test/t0010.errors delete mode 100644 test/t0010.expect delete mode 100644 test/t0010.sh diff --git a/ChangeLog b/ChangeLog index f2cc77b..745fc65 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,8 +1,16 @@ +??? 12, 2020 - Release 0.0.13 +==================================== + gcm_input ---------- - logrunner is statically linked so it can be copied to different systems, 32 bit or 64 bit, without library dependencies. + - Save the input in an XML file if it can not be stored in the database + because the database is not available or the XML parser returns an error. + The directory where the XML file is stored is defined in the configuration file, + element "spool/directory". + This solved PR10. gcm_maintenance ----------- diff --git a/doc/test.xml b/doc/test.xml index aad6716..e55d57d 100644 --- a/doc/test.xml +++ b/doc/test.xml @@ -48,35 +48,45 @@ This chapter describes individual test cases.
-Test 0001: Database +Test createdb: Database Create and destroy the database.
-Test 0002a: Read syslog entries +Test upgradedb: Database upgrades -Create an object 'kithira.andromeda.nl' in the database and -parse a piece of a syslog file. +Test the database upgrades in gcm_daemon. +Create an old, version 1, database and run gcm_daemon to perform +an upgrade to the latest version. +Then, create a latest-version database and compare dumps of these +database. They should be essentially the same. + +
+ +
+Test read_messages: Read syslog entries + +Create an object in the database and parse a piece of a syslog file. This tests normal operation in which gcm_input reads a syslog file directly.
-Test 0003: Read corrupted syslog entries +Test read_bad_messages: Read corrupted syslog entries -Create an object 'vd.wt.tno.nl' in the database and +Create an object in the database and parse a piece of a syslog file with short and even empty lines. There should be warnings printed on stderr.
-Test 0004: Read syslog entries without hostname +Test read_without_hostname: Read syslog entries without hostname -Create an object 'kithira.andromeda.nl' in the database and +Create an object in the database and try to parse a piece of a syslog file without specifying the hostname. The message should be rejected on account of an unknown host. @@ -92,17 +102,6 @@ This tests wether an improper time still creates valid data.
-Test 0008: Database upgrades - -Test the database upgrades in gcm_daemon. -Create an old, version 1, database and run gcm_daemon to perform -an upgrade to the latest version. -Then, create a latest-version database and compare dumps of these -database. They should be essentially the same. - -
- -
Test 0009: Service check notifications Test the service check notifications. @@ -116,7 +115,7 @@ This tests problem report nr. 23
-Test 0010: Processing log entries +Test notification_sendmail: Processing log entries Test the log processing of gcm_daemon Create a database with one object and read a piece of sendmail log diff --git a/src/gcm_input/gcm_input.cpp b/src/gcm_input/gcm_input.cpp index 8f9a604..a4b8937 100644 --- a/src/gcm_input/gcm_input.cpp +++ b/src/gcm_input/gcm_input.cpp @@ -125,11 +125,10 @@ *****************************/ -static const char *RCSID = "$Id: gcm_input.cpp,v 1.16 2011-03-24 10:20:37 arjen Exp $"; - #include #include +#include #include "message.h" #include "log_filter.h" @@ -147,7 +146,7 @@ bool testmode = false; bool incremental = false; std::ostream *Log = &std::cerr; -static char Version[] = "gcm_input version 0.0.11 - Nov 22, 2007"; +static char Version[] = "gcm_input version 0.0.13 - Sep 22, 2020"; /*========================================================================= @@ -280,57 +279,72 @@ int main(int argc, char *argv[]) gnucomo_database db(&cfg); - if (db.is_connected()) - { + int gcm_input_result = 0; + + + client_message msg(&std::cin, db); + + double message_probability; - client_message msg(&std::cin, db); - - double message_probability; - - message_filter shortcircuit(hostname, arrival, service); - log_filter lf(hostname, arrival, service); - rpm_filter rf(hostname, arrival, service); - df_filter df(hostname, arrival, service); - - syslog_cooker slc; - irix_syslog_cooker islc; - access_cooker alc; - error_cooker elc; - xml_cooker xlc; - rpm_cooker rlc; - df_cooker dlc; - - msg.add_cooker(&xlc, &shortcircuit); - msg.add_cooker(&slc, &lf); - msg.add_cooker(&islc, &lf); - msg.add_cooker(&alc, &lf); - msg.add_cooker(&elc, &lf); - msg.add_cooker(&rlc, &rf); - msg.add_cooker(&dlc, &df); - - message_probability = msg.classify(hostname, arrival, service); - if (message_probability > 0.75) + message_filter shortcircuit(hostname, arrival, service); + log_filter lf(hostname, arrival, service); + rpm_filter rf(hostname, arrival, service); + df_filter df(hostname, arrival, service); + + syslog_cooker slc; + irix_syslog_cooker islc; + access_cooker alc; + error_cooker elc; + xml_cooker xlc; + rpm_cooker rlc; + df_cooker dlc; + + msg.add_cooker(&xlc, &shortcircuit); + msg.add_cooker(&slc, &lf); + msg.add_cooker(&islc, &lf); + msg.add_cooker(&alc, &lf); + msg.add_cooker(&elc, &lf); + msg.add_cooker(&rlc, &rf); + msg.add_cooker(&dlc, &df); + + message_probability = msg.classify(hostname, arrival, service); + if (message_probability > 0.75) + { + try { - try + if (msg.enter() < 0) { - msg.enter(); - } - catch (std::exception &e) - { - *Log << "Caught an exception: " << e.what() << "\n"; + // Can not store the input in the database. Dump the message in a file. + + String spool_dir = cfg.find_parameter("spool", "directory"); + String pid(getpid()); + String xmlfilename; + + if (spool_dir) + { + xmlfilename = spool_dir + "/"; + } + xmlfilename += "gnucomo" + pid + ".xml"; + + msg.saveXML(xmlfilename); + + // Report the error to the log and stderr. + *Log << "Entering the content into the database failed. XML content stored in " + xmlfilename + "\n"; + std::cerr << "Entering the content into the database failed. XML content stored in " + xmlfilename + "\n"; + + gcm_input_result = 1; } } - else + catch (std::exception &e) { - *Log << "Cannot determine message type with sufficient certainty.\n"; + *Log << "Caught an exception: " << e.what() << "\n"; } - return 0; } else { - *Log << "gcm_input: Can not connect to database.\n"; - *Log << "Gcm_input finished at " << Now() << ".\n"; - return 1; + *Log << "Cannot determine message type with sufficient certainty.\n"; } + + return gcm_input_result; } diff --git a/src/gcm_input/message.cpp b/src/gcm_input/message.cpp index 9e761b5..9393a16 100644 --- a/src/gcm_input/message.cpp +++ b/src/gcm_input/message.cpp @@ -116,11 +116,13 @@ *****************************/ -static const char *RCSID = "$Id: message.cpp,v 1.19 2011-03-24 10:20:37 arjen Exp $"; +#include +#include #include #include #include + #include "message.h" //#define DEBUG @@ -508,6 +510,8 @@ void client_message::enterXML() } if (strcmp((char *)node->name, "log") == 0) { + int log_entry_counter = 0; + // Each child contains a log entry, raw or cooked. node = node->children; @@ -515,12 +519,18 @@ void client_message::enterXML() { if (node->type == XML_ELEMENT_NODE) { + log_entry_counter++; + xmlNodePtr item; String log_hostname; UTC log_date; String raw("");; String log_service; + if (verbose) + { + *Log << "Log entry " << log_entry_counter << " is " << node->name << "\n"; + } if (strcmp((char *)node->name, "raw") == 0 && node->children != NULL) { item = node->children; @@ -598,7 +608,7 @@ void client_message::enterXML() } else { - *Log << " missing from cooked log element.\n"; + *Log << " missing from cooked log element nr " << log_entry_counter << ".\n"; } res = xmlXPathEval((const xmlChar *)"raw/text()", pathcontext); @@ -609,7 +619,7 @@ void client_message::enterXML() } else { - *Log << " missing from cooked log element.\n"; + *Log << " missing from cooked log element nr " << log_entry_counter << ".\n"; } } @@ -1102,11 +1112,23 @@ void client_message::enterXML() } } +void client_message::saveXML(String filename) +{ + std::ofstream xmlfile; + + xmlfile.open(filename); + xmlfile << xmlBuffer.str(); +} + /*========================================================================= ** NAME : enter ** SYNOPSIS : int enter() ** PARAMETERS : ** RETURN VALUE : The number of lines successfully parsed from the input +** or a negative number if an error is encountered. +** The error return values are: +** -1 No database connection. +** -2 XML parser error. ** ** DESCRIPTION : ** @@ -1114,12 +1136,14 @@ void client_message::enterXML() ** VARS CHANGED : ** FUNCTIONS USED : ** SEE ALSO : -** LAST MODIFIED : Nov 26, 2003 +** LAST MODIFIED : Sep 22, 2020 **========================================================================= */ int client_message::enter() { + int nr_lines = 0; + pan.mf->set_message_type(pan.lc->message_type()); pan.mf->construct_XML(input, xmlBuffer); @@ -1132,18 +1156,27 @@ int client_message::enter() xmlDom = xmlParseMemory(xmlBuffer.str(), xmlBuffer.pcount()); - if (xmlDom) + if (database.is_connected()) { - if (extractHeader()) + if (xmlDom) + { + if (extractHeader()) + { + enterXML(); + } + } + else { - enterXML(); + *Log << "XML parser FAILED.\n"; + nr_lines = -2; // XML parse error } } else { - *Log << "XML parser FAILED.\n"; + *Log << "Database connection FAILED.\n"; + nr_lines = -1; // No database connection } - return 0; + return nr_lines; } diff --git a/src/gcm_input/message.h b/src/gcm_input/message.h index 98f2d1b..51ff8d3 100644 --- a/src/gcm_input/message.h +++ b/src/gcm_input/message.h @@ -162,5 +162,6 @@ public: double classify(String host, UTC arrival = Now(), String serv = ""); int enter(); + void saveXML(String filename); }; diff --git a/src/gnucomo.conf b/src/gnucomo.conf index 59ff0a8..21cb7d1 100644 --- a/src/gnucomo.conf +++ b/src/gnucomo.conf @@ -18,6 +18,9 @@ brenno + + /var/spool/gnucomo + arjen test diff --git a/src/lib/database.cpp b/src/lib/database.cpp index 0980bf7..63384a8 100644 --- a/src/lib/database.cpp +++ b/src/lib/database.cpp @@ -70,8 +70,6 @@ *****************************/ -static const char *RCSID = "$Id: database.cpp,v 1.13 2011-03-24 10:21:47 arjen Exp $"; - #include //#define DEBUG @@ -93,7 +91,7 @@ extern std::ostream *Log; ** VARS CHANGED : ** FUNCTIONS USED : ** SEE ALSO : -** LAST MODIFIED : Aug 17, 2003 +** LAST MODIFIED : Sep 23, 2020 **========================================================================= */ @@ -101,30 +99,30 @@ static int gdb_refcount = 0; gnucomo_database::gnucomo_database(gnucomo_config *c) { + dbconn = 0; + dbxact = 0; cfg = c; - dbconn = new pqxx::connection(cfg->Database()); - - if (!dbconn->is_open()) + try { - std::cerr << "Connection to database failed.\n"; - } - else - { - try + dbconn = new pqxx::connection(cfg->Database()); + + if (!dbconn->is_open()) + { + std::cerr << "Connection to database failed.\n"; + } + else { // Create the transaction object //dbxact = new pqxx::transaction(*dbconn, "GnuCoMo"); dbxact = new pqxx::work(*dbconn, "GnuCoMo"); + gdb_refcount++; } - catch (std::exception &e) - { - *Log << "Cannot setup the database transaction: " << e.what() << "\n"; - *Log << "You are probably using incompatible versions of PostgreSQL an libpqxx.\n"; - exit(1); - } - gdb_refcount++; + } + catch (std::exception &e) + { + *Log << "Cannot setup the database transaction: " << e.what() << "\n"; } } diff --git a/test/Makefile.am b/test/Makefile.am index bfa8e8d..2c5d486 100644 --- a/test/Makefile.am +++ b/test/Makefile.am @@ -1,5 +1,6 @@ TESTS = createdb upgradedb read_messages read_bad_messages read_apache_error read_without_hostname \ - notifications + read_no_database \ + notifications notification_sendmail clean-local: rm -f gcm_input.log log.tbl diff --git a/test/gnucomo.conf b/test/gnucomo.conf new file mode 100644 index 0000000..20dec83 --- /dev/null +++ b/test/gnucomo.conf @@ -0,0 +1,23 @@ + + + + PostgreSQL + gnucomo_test + arjen + guess-again:-) + + + + file + ./gcm_input + 0 + + + arjen + test + + + diff --git a/test/gnucomo_nodb.conf b/test/gnucomo_nodb.conf new file mode 100644 index 0000000..6b130ba --- /dev/null +++ b/test/gnucomo_nodb.conf @@ -0,0 +1,26 @@ + + + + PostgreSQL + gnucomo_no_database + arjen + guess-again:-) + + + + file + ./gcm_input.log + 0 + + + . + + + arjen + test + + + diff --git a/test/notification_sendmail b/test/notification_sendmail new file mode 100755 index 0000000..bc83495 --- /dev/null +++ b/test/notification_sendmail @@ -0,0 +1,47 @@ +#!/bin/bash + +# +# Test the log processing of gcm_daemon +# Create a database with one object and read a piece of sendmail log +# +# This tests problem report nrs. 14 through 17 + + +rm -f gcm_input.log +createdb gnucomo_test + +result=0 + +if psql gnucomo_test -q <../src/database/create.sql >/dev/null +then + # Prepare the database + + psql gnucomo_test -q -c "insert into object (objectname) values ('schiza.andromeda.nl')" + + # read logs for both objects + + ../src/gcm_input/gcm_input -c gnucomo_test -h schiza.andromeda.nl -d 'oct 26 2003 20:30:45' /dev/null +then + psql gnucomo_test -q -c "insert into object (objectname) values ('example1.gnucomo.test')" + ../src/gcm_input/gcm_input -c gnucomo_nodb -h example1.gnucomo.test -d 'sep 5 2002 20:30:45' /dev/null -then - # Prepare the database - - psql gnucomo_test -q -c "insert into object (objectname) values ('schiza.andromeda.nl')" - - # read logs for both objects - - src/gcm_input/gcm_input -c gnucomo_test -h schiza.andromeda.nl -d 'oct 26 2003 20:30:45'