+??? 12, 2020 - Release 0.0.13
+====================================
+
gcm_input
----------
- logrunner is statically linked so it can be copied to different systems,
32 bit or 64 bit, without library dependencies.
+ - Save the input in an XML file if it can not be stored in the database
+ because the database is not available or the XML parser returns an error.
+ The directory where the XML file is stored is defined in the configuration file,
+ element "spool/directory".
+ This solved PR10.
gcm_maintenance
-----------
</para>
<section>
-<heading>Test 0001: Database</heading>
+<heading>Test createdb: Database</heading>
<para>
Create and destroy the database.
</para>
</section>
<section>
-<heading>Test 0002a: Read syslog entries</heading>
+<heading>Test upgradedb: Database upgrades</heading>
<para>
-Create an object 'kithira.andromeda.nl' in the database and
-parse a piece of a syslog file.
+Test the database upgrades in gcm_daemon.
+Create an old, version 1, database and run gcm_daemon to perform
+an upgrade to the latest version.
+Then, create a latest-version database and compare dumps of these
+database. They should be essentially the same.
+</para>
+</section>
+
+<section>
+<heading>Test read_messages: Read syslog entries</heading>
+<para>
+Create an object in the database and parse a piece of a syslog file.
This tests normal operation in which gcm_input reads a syslog file
directly.
</para>
</section>
<section>
-<heading>Test 0003: Read corrupted syslog entries</heading>
+<heading>Test read_bad_messages: Read corrupted syslog entries</heading>
<para>
-Create an object 'vd.wt.tno.nl' in the database and
+Create an object in the database and
parse a piece of a syslog file with short and even empty lines.
There should be warnings printed on stderr.
</para>
</section>
<section>
-<heading>Test 0004: Read syslog entries without hostname</heading>
+<heading>Test read_without_hostname: Read syslog entries without hostname</heading>
<para>
-Create an object 'kithira.andromeda.nl' in the database and
+Create an object in the database and
try to parse a piece of a syslog file without specifying the hostname.
The message should be rejected on account of an unknown host.
</para>
</section>
<section>
-<heading>Test 0008: Database upgrades</heading>
-<para>
-Test the database upgrades in gcm_daemon.
-Create an old, version 1, database and run gcm_daemon to perform
-an upgrade to the latest version.
-Then, create a latest-version database and compare dumps of these
-database. They should be essentially the same.
-</para>
-</section>
-
-<section>
<heading>Test 0009: Service check notifications</heading>
<para>
Test the service check notifications.
</section>
<section>
-<heading>Test 0010: Processing log entries</heading>
+<heading>Test notification_sendmail: Processing log entries</heading>
<para>
Test the log processing of gcm_daemon
Create a database with one object and read a piece of sendmail log
*****************************/
-static const char *RCSID = "$Id: gcm_input.cpp,v 1.16 2011-03-24 10:20:37 arjen Exp $";
-
#include <fstream>
#include <getopt.h>
+#include <unistd.h>
#include "message.h"
#include "log_filter.h"
bool incremental = false;
std::ostream *Log = &std::cerr;
-static char Version[] = "gcm_input version 0.0.11 - Nov 22, 2007";
+static char Version[] = "gcm_input version 0.0.13 - Sep 22, 2020";
/*=========================================================================
gnucomo_database db(&cfg);
- if (db.is_connected())
- {
+ int gcm_input_result = 0;
+
+
+ client_message msg(&std::cin, db);
+
+ double message_probability;
- client_message msg(&std::cin, db);
-
- double message_probability;
-
- message_filter shortcircuit(hostname, arrival, service);
- log_filter lf(hostname, arrival, service);
- rpm_filter rf(hostname, arrival, service);
- df_filter df(hostname, arrival, service);
-
- syslog_cooker slc;
- irix_syslog_cooker islc;
- access_cooker alc;
- error_cooker elc;
- xml_cooker xlc;
- rpm_cooker rlc;
- df_cooker dlc;
-
- msg.add_cooker(&xlc, &shortcircuit);
- msg.add_cooker(&slc, &lf);
- msg.add_cooker(&islc, &lf);
- msg.add_cooker(&alc, &lf);
- msg.add_cooker(&elc, &lf);
- msg.add_cooker(&rlc, &rf);
- msg.add_cooker(&dlc, &df);
-
- message_probability = msg.classify(hostname, arrival, service);
- if (message_probability > 0.75)
+ message_filter shortcircuit(hostname, arrival, service);
+ log_filter lf(hostname, arrival, service);
+ rpm_filter rf(hostname, arrival, service);
+ df_filter df(hostname, arrival, service);
+
+ syslog_cooker slc;
+ irix_syslog_cooker islc;
+ access_cooker alc;
+ error_cooker elc;
+ xml_cooker xlc;
+ rpm_cooker rlc;
+ df_cooker dlc;
+
+ msg.add_cooker(&xlc, &shortcircuit);
+ msg.add_cooker(&slc, &lf);
+ msg.add_cooker(&islc, &lf);
+ msg.add_cooker(&alc, &lf);
+ msg.add_cooker(&elc, &lf);
+ msg.add_cooker(&rlc, &rf);
+ msg.add_cooker(&dlc, &df);
+
+ message_probability = msg.classify(hostname, arrival, service);
+ if (message_probability > 0.75)
+ {
+ try
{
- try
+ if (msg.enter() < 0)
{
- msg.enter();
- }
- catch (std::exception &e)
- {
- *Log << "Caught an exception: " << e.what() << "\n";
+ // Can not store the input in the database. Dump the message in a file.
+
+ String spool_dir = cfg.find_parameter("spool", "directory");
+ String pid(getpid());
+ String xmlfilename;
+
+ if (spool_dir)
+ {
+ xmlfilename = spool_dir + "/";
+ }
+ xmlfilename += "gnucomo" + pid + ".xml";
+
+ msg.saveXML(xmlfilename);
+
+ // Report the error to the log and stderr.
+ *Log << "Entering the content into the database failed. XML content stored in " + xmlfilename + "\n";
+ std::cerr << "Entering the content into the database failed. XML content stored in " + xmlfilename + "\n";
+
+ gcm_input_result = 1;
}
}
- else
+ catch (std::exception &e)
{
- *Log << "Cannot determine message type with sufficient certainty.\n";
+ *Log << "Caught an exception: " << e.what() << "\n";
}
- return 0;
}
else
{
- *Log << "gcm_input: Can not connect to database.\n";
- *Log << "Gcm_input finished at " << Now() << ".\n";
- return 1;
+ *Log << "Cannot determine message type with sufficient certainty.\n";
}
+
+ return gcm_input_result;
}
*****************************/
-static const char *RCSID = "$Id: message.cpp,v 1.19 2011-03-24 10:20:37 arjen Exp $";
+#include <unistd.h>
+#include <fstream>
#include <algorithm>
#include <libxml/xpath.h>
#include <libxml/debugXML.h>
+
#include "message.h"
//#define DEBUG
}
if (strcmp((char *)node->name, "log") == 0)
{
+ int log_entry_counter = 0;
+
// Each child contains a log entry, raw or cooked.
node = node->children;
{
if (node->type == XML_ELEMENT_NODE)
{
+ log_entry_counter++;
+
xmlNodePtr item;
String log_hostname;
UTC log_date;
String raw("");;
String log_service;
+ if (verbose)
+ {
+ *Log << "Log entry " << log_entry_counter << " is " << node->name << "\n";
+ }
if (strcmp((char *)node->name, "raw") == 0 && node->children != NULL)
{
item = node->children;
}
else
{
- *Log << "<timestamp> missing from cooked log element.\n";
+ *Log << "<timestamp> missing from cooked log element nr " << log_entry_counter << ".\n";
}
res = xmlXPathEval((const xmlChar *)"raw/text()", pathcontext);
}
else
{
- *Log << "<raw> missing from cooked log element.\n";
+ *Log << "<raw> missing from cooked log element nr " << log_entry_counter << ".\n";
}
}
}
}
+void client_message::saveXML(String filename)
+{
+ std::ofstream xmlfile;
+
+ xmlfile.open(filename);
+ xmlfile << xmlBuffer.str();
+}
+
/*=========================================================================
** NAME : enter
** SYNOPSIS : int enter()
** PARAMETERS :
** RETURN VALUE : The number of lines successfully parsed from the input
+** or a negative number if an error is encountered.
+** The error return values are:
+** -1 No database connection.
+** -2 XML parser error.
**
** DESCRIPTION :
**
** VARS CHANGED :
** FUNCTIONS USED :
** SEE ALSO :
-** LAST MODIFIED : Nov 26, 2003
+** LAST MODIFIED : Sep 22, 2020
**=========================================================================
*/
int client_message::enter()
{
+ int nr_lines = 0;
+
pan.mf->set_message_type(pan.lc->message_type());
pan.mf->construct_XML(input, xmlBuffer);
xmlDom = xmlParseMemory(xmlBuffer.str(), xmlBuffer.pcount());
- if (xmlDom)
+ if (database.is_connected())
{
- if (extractHeader())
+ if (xmlDom)
+ {
+ if (extractHeader())
+ {
+ enterXML();
+ }
+ }
+ else
{
- enterXML();
+ *Log << "XML parser FAILED.\n";
+ nr_lines = -2; // XML parse error
}
}
else
{
- *Log << "XML parser FAILED.\n";
+ *Log << "Database connection FAILED.\n";
+ nr_lines = -1; // No database connection
}
- return 0;
+ return nr_lines;
}
double classify(String host, UTC arrival = Now(), String serv = "");
int enter();
+ void saveXML(String filename);
};
<gcm_input>
<dbuser>brenno</dbuser>
</gcm_input>
+ <spool>
+ <directory>/var/spool/gnucomo</directory>
+ </spool>
<gcm_daemon>
<user>arjen</user>
<password>test</password>
*****************************/
-static const char *RCSID = "$Id: database.cpp,v 1.13 2011-03-24 10:21:47 arjen Exp $";
-
#include <date.h>
//#define DEBUG
** VARS CHANGED :
** FUNCTIONS USED :
** SEE ALSO :
-** LAST MODIFIED : Aug 17, 2003
+** LAST MODIFIED : Sep 23, 2020
**=========================================================================
*/
gnucomo_database::gnucomo_database(gnucomo_config *c)
{
+ dbconn = 0;
+ dbxact = 0;
cfg = c;
- dbconn = new pqxx::connection(cfg->Database());
-
- if (!dbconn->is_open())
+ try
{
- std::cerr << "Connection to database failed.\n";
- }
- else
- {
- try
+ dbconn = new pqxx::connection(cfg->Database());
+
+ if (!dbconn->is_open())
+ {
+ std::cerr << "Connection to database failed.\n";
+ }
+ else
{
// Create the transaction object
//dbxact = new pqxx::transaction<pqxx::serializable>(*dbconn, "GnuCoMo");
dbxact = new pqxx::work(*dbconn, "GnuCoMo");
+ gdb_refcount++;
}
- catch (std::exception &e)
- {
- *Log << "Cannot setup the database transaction: " << e.what() << "\n";
- *Log << "You are probably using incompatible versions of PostgreSQL an libpqxx.\n";
- exit(1);
- }
- gdb_refcount++;
+ }
+ catch (std::exception &e)
+ {
+ *Log << "Cannot setup the database transaction: " << e.what() << "\n";
}
}
TESTS = createdb upgradedb read_messages read_bad_messages read_apache_error read_without_hostname \
- notifications
+ read_no_database \
+ notifications notification_sendmail
clean-local:
rm -f gcm_input.log log.tbl
--- /dev/null
+<?xml version='1.0'?>
+<gnucomo version='0.0.8'>
+ <database>
+ <type>PostgreSQL</type>
+ <name>gnucomo_test</name>
+ <user>arjen</user>
+ <password>guess-again:-)</password>
+ <!-- We do not use a TCP connection
+ <host>localhost</host>
+ <port>5432</port>
+ -->
+ </database>
+ <logging>
+ <method>file</method>
+ <destination>./gcm_input</destination>
+ <level>0</level>
+ </logging>
+ <gcm_daemon>
+ <user>arjen</user>
+ <password>test</password>
+ </gcm_daemon>
+</gnucomo>
+
--- /dev/null
+<?xml version='1.0'?>
+<gnucomo_nodb version='0.0.8'>
+ <database>
+ <type>PostgreSQL</type>
+ <name>gnucomo_no_database</name>
+ <user>arjen</user>
+ <password>guess-again:-)</password>
+ <!-- We do not use a TCP connection
+ <host>localhost</host>
+ <port>5432</port>
+ -->
+ </database>
+ <logging>
+ <method>file</method>
+ <destination>./gcm_input.log</destination>
+ <level>0</level>
+ </logging>
+ <spool>
+ <directory>.</directory>
+ </spool>
+ <gcm_daemon>
+ <user>arjen</user>
+ <password>test</password>
+ </gcm_daemon>
+</gnucomo_nodb>
+
--- /dev/null
+#!/bin/bash
+
+#
+# Test the log processing of gcm_daemon
+# Create a database with one object and read a piece of sendmail log
+#
+# This tests problem report nrs. 14 through 17
+
+
+rm -f gcm_input.log
+createdb gnucomo_test
+
+result=0
+
+if psql gnucomo_test -q <../src/database/create.sql >/dev/null
+then
+ # Prepare the database
+
+ psql gnucomo_test -q -c "insert into object (objectname) values ('schiza.andromeda.nl')"
+
+ # read logs for both objects
+
+ ../src/gcm_input/gcm_input -c gnucomo_test -h schiza.andromeda.nl -d 'oct 26 2003 20:30:45' <log0010
+
+ # Test gcm_daemon
+ cd ../src/gcm_daemon; ./gcm_daemon.php -c gnucomo_test; cd ../../test
+
+ # Check the notifications.
+
+ NR_NOTIF1=`psql gnucomo_test -q -t -c "select count(*) from notification"`
+ if [[ $NR_NOTIF1 -ne 4 ]]
+ then
+ echo "$NR_NOTIF1 notifications for object 1 (4 expected)."
+ result=2
+ fi
+
+ # Clean up.
+
+ psql gnucomo_test -f ../src/database/destroy.sql -q
+
+ dropdb gnucomo_test
+ exit $result
+else
+ echo Can not create test database
+ dropdb gnucomo_test
+ exit 1
+fi
--- /dev/null
+#!/bin/bash
+
+#
+# Try to read a syslog file and store in a database that does not exist.
+# gcm_input should return an error and save the input in an XML file.
+
+
+rm -f gcm_input.log
+
+createdb gnucomo_test
+result=1
+
+
+if psql gnucomo_test -q <../src/database/create.sql >/dev/null
+then
+ psql gnucomo_test -q -c "insert into object (objectname) values ('example1.gnucomo.test')"
+ ../src/gcm_input/gcm_input -c gnucomo_nodb -h example1.gnucomo.test -d 'sep 5 2002 20:30:45' <messages
+ result=$?
+ echo "gcm_input returned $result"
+
+ if [[ $result -eq 1 ]]
+ then
+ # check if nothing was stored in the (existing) database
+ NR_LOGS=`psql gnucomo_test -q -t -c "select count(*) from log"`
+ if [[ $NR_LOGS -eq 0 ]]
+ then
+ echo "No log entries in database"
+ result=0
+ fi
+ # check if the saved XML file exists
+ SAVEDFILE=`grep 'XML content stored' gcm_input.log |cut -d ' ' -f 12`
+ echo "Saved file is $SAVEDFILE"
+ if [[ ! -e $SAVEDFILE ]]
+ then
+ echo $SAVEDFILE does not exist.
+ result=2
+ fi
+ rm $SAVEDFILE # cleanup
+ else
+ echo "gcm_input did not return an error"
+ result=1
+ fi
+ psql gnucomo_test -f ../src/database/destroy.sql -q
+else
+ echo Can not create test database
+fi
+
+dropdb gnucomo_test
+exit $result
+++ /dev/null
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "object_abuse_pkey" for table "object_abuse"
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "object_statistics_pkey" for table "object_statistics"
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "parameter_pkey" for table "parameter"
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "parameter_class_pkey" for table "parameter_class"
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "parameter_notification_pkey" for table "parameter_notification"
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "property_pkey" for table "property"
-NOTICE: CREATE TABLE / PRIMARY KEY will create implicit index "service_pattern_pkey" for table "service_pattern"
-ERROR: role "view" already exists
-ERROR: role "ops" already exists
-ERROR: role "admin" already exists
-ERROR: role "daemon" already exists
-ERROR: role "gnucomo" already exists
+++ /dev/null
-Processing logs...
-Last processed logid = 0
-Running service check from log id 0.
-Gathering statistics for object 1
-
- notificationid | objectid | type_of_issueid
-----------------+----------+-----------------
- 1 | 1 | 8
- 2 | 1 | 10
- 3 | 1 | 11
- 4 | 1 | 6
-(4 rows)
-
+++ /dev/null
-#!/bin/sh
-
-#
-# Test the log processing of gcm_daemon
-# Create a database with one object and read a piece of sendmail log
-#
-# This tests problem report nrs. 14 through 17
-
-LD_LIBRARY_PATH=/usr/local/pqxx/lib
-export LD_LIBRARY_PATH
-
-cd ..
-
-if psql gnucomo_test -q <src/database/create.sql >/dev/null
-then
- # Prepare the database
-
- psql gnucomo_test -q -c "insert into object (objectname) values ('schiza.andromeda.nl')"
-
- # read logs for both objects
-
- src/gcm_input/gcm_input -c gnucomo_test -h schiza.andromeda.nl -d 'oct 26 2003 20:30:45' <test/log0010
-
- # Test gcm_daemon
- cd src/gcm_daemon; ./gcm_daemon.php -c gnucomo_test; cd ../..
-
- # Check the notifications.
-
- psql gnucomo_test -c "select notificationid,objectid,type_of_issueid from notification"
-
- # Clean up.
-
- psql gnucomo_test -f src/database/destroy.sql -q
-
- exit 0
-else
- echo Can not create test database
- exit 1
-fi
projects / gnucomo.git / commitdiff