* logrunner.c
* (c) Peter Roozemaal, feb 2003
*
- * $Id: logrunner.cpp,v 1.1 2003-08-11 16:56:16 arjen Exp $
+ * $Id: logrunner.cpp,v 1.6 2007-12-10 16:12:37 arjen Exp $
*
* 1) compile,
- * 2) create 'logrunner.conf' containing a list of filenames and types of the
- * logfiles to scan; one file per line:
- * <gnucomo_messagetype> TAB <logfilename>
+ * 2) Add 'logfile' elements to the gnucomo configuration file
+ * defining the name and type of each logfile to scan.
+ * 'filter' elements can be addes to pre-filter lines from the log.
+ *
* example:
- * syslog /var/log/messages
- * syslog /var/log/secure
- * 3) run executable
+ * <logfile>
+ * <name>/var/log/httpd/access_log</name>
+ * <type>apache access log</type>
+ * </logfile>
+ * <logfile>
+ * <name>/var/log/messages</name>
+ * <type>system log</type>
+ * <filter>open_scanner</filter>
+ * <filter>session closed</filter>
+ * </logfile>
+ *
+ * 3) run executable and feed the output to gcm_input
*/
#include <sys/types.h>
#include <sys/stat.h>
#include <stdlib.h>
#include <string.h>
+#include <fstream>
+#include <list>
+
#include <AXE/String.h>
+#include "gnucomo_config.h"
+
extern String XML_Entities(String s);
static const char* const usage =
"<hostname> is the gnucomo data collection server\n"
"Options are:\n"
" -1: one-shot; stop after a single pass over one of the configured logfiles\n"
+ " -c <name>: specify alternative configuration name, default is \'gnucomo\'\n"
+ " -l <file>: Read input from logfile <file>.\n"
" -p <number>: set IP port number to connect to\n"
- " -c <file>: specify alternative configuration file, default is \'logrunner.conf\'\n"
+ " -s <file>: Use <file> as the status file instead of the default.\n"
+ " -v Verbose output\n"
"If no hostname is specified, logrunner sends output to stdout\n";
-struct fileinfo {
- struct fileinfo* next;
- char* name; /* strdupped name */
- char* type; /* strdupped logtype */
+class LogFile {
+ String name;
+ String type;
+ String fromhost;
+
int fd;
ino_t inode;
off_t position;
+
+ std::list<regex> filter;
+
+ void copy_data();
+
+public:
+ LogFile(String n, String t, String fh)
+ {
+ name = n;
+ type = t;
+ fromhost = fh;
+ inode = 0;
+ position = 0;
+ fd = -1;
+ }
+
+ LogFile(const LogFile &lf)
+ {
+ name = lf.name;
+ type = lf.type;
+ fromhost = lf.fromhost;
+
+ fd = lf.fd;
+ inode = lf.inode;
+ position = lf.position;
+ filter = lf.filter;
+ }
+
+ String pathname()
+ {
+ return name;
+ }
+
+ void update_status(ino_t i, off_t p)
+ {
+ inode = i;
+ position = p;
+ }
+
+ String status()
+ {
+ return name + " " + String(inode) + " " + String(position);
+ }
+
+ void add_filter(String filter_expression)
+ {
+ filter.push_back(filter_expression);
+ }
+
+ void do_file();
};
-static const char* confname = "logrunner.conf";
-static const char* statusname = "logrunner.status";
-static const char* newstatusname = "logrunner.status.new";
+static std::list<LogFile> logs_to_run;
+
+String confname = "gnucomo";
+static char* statusname = "/var/lib/logrunner.status";
+static char* newstatusname = "/var/lib/logrunner.status.new";
static const char *hostname = NULL;
static int port = 2996; /* random magic number */
static int out_stream = 1;
static int oneshot = 0;
-
-static struct fileinfo* filelist = NULL;
+static int verbose = 0;
static int something_has_changed = 0;
signal(SIGTERM, sighandler);
}
+
void open_output()
{
struct hostent* hostptr;
write(out_stream, str, strlen(str));
}
-void xml_header(struct fileinfo *f)
+void xml_header(String type)
{
+ struct hostent *host;
char buffer[256];
*buffer = 0;
gethostname(buffer, sizeof(buffer));
- xsend("<?xml version='1.0'?>\n");
+
+ // Try to obtain the official name of the host (FQDN)
+
+ host = gethostbyname(buffer);
+ if (host != NULL)
+ {
+ strcpy(buffer, host->h_name);
+ }
+
+ xsend("<?xml version='1.0'?>\n");
xsend("<gcmt:message xmlns:gcmt=\"http://gnucomo.org/transport/\">\n");
xsend("<gcmt:header>\n<gcmt:hostname>");
xsend(buffer);
xsend("</gcmt:hostname>\n<gcmt:messagetype>");
- xsend(f->type);
+ xsend(type);
xsend("</gcmt:messagetype>\n</gcmt:header>\n");
xsend("<gcmt:data><gcmt:log>");
}
void output(const char* str)
{
- write(2, str, strlen(str));
+ std::cerr << str;
}
void output_error(const char* str)
{
- char error[80];
- sprintf(error, " errno=%d\n", errno);
- output(str);
- output(error);
-}
-
-void write_status_file(struct fileinfo* fl)
-{
- FILE* dumpfile;
- int localerror = 0;
- dumpfile = fopen(newstatusname, "w");
- if ( !dumpfile )
- {
- char error[80];
- sprintf(error, ", errno=%d\n", errno);
- output("!!! dumpstatus: open failed: ");
- output(newstatusname);
- output(error);
- something_has_changed = 0;
- return;
- }
- while ( fl )
- {
- if ( fprintf(dumpfile, "%s %li %lu\n",
- fl->name, (long) fl->inode,
- (unsigned long) fl->position) < 0 )
- {
- output_error("!!! dumpstatus: write failed");
- localerror = 1;
- break;
- }
- fl = fl->next;
- }
- if ( fclose(dumpfile) && localerror == 0 )
- {
- output_error("!!! dumpstatus: close failed");
- localerror = 1;
- }
- if ( localerror == 0 )
- {
- if ( rename(newstatusname, statusname) )
- {
- output_error("!!! dumpstatus: rename failed");
- }
- }
- something_has_changed = 0;
+ std::cerr << str << " errno=" << errno << "\n";
}
-void read_status()
+void LogFile::do_file()
{
- FILE* statusfile;
- char buffer[4096];
- char* xp;
- long ino;
- unsigned long pos;
- struct fileinfo* fp;
+ struct stat statinfo;
+ char buffer[80];
- statusfile = fopen(statusname, "r");
- if ( statusfile == NULL )
+ /* inode check */
+ if ( stat(name, &statinfo) )
{
- fprintf(stderr, "logrunner: can\'t open status file \'%s\': ",
- statusname);
- perror("");
- return;
+ std::cerr << "!!! logfile: stat failed: " << name << ", errno=" << errno << "\n";
}
- while ( fgets(buffer, sizeof(buffer), statusfile ) )
+ else
{
- xp = strchr(buffer, ' ');
- if ( xp )
+ if ( statinfo.st_ino != inode )
{
- *xp = 0;
- sscanf(xp+1, "%ld %lu", &ino, &pos);
- fp = filelist;
- while ( fp )
+ if ( fd >= 0 )
{
- if ( strcmp(buffer, fp->name) == 0 )
- {
- fp->inode = ino;
- fp->position = pos;
- break;
- }
- fp = fp->next;
+ copy_data();
+ close(fd);
+ fd = -1;
}
+ if (verbose)
+ {
+ std::cerr << "@@@ logfile: logrotate detected: " << name << "\n";
+ }
+ inode = statinfo.st_ino;
+ position = 0;
}
}
- fclose(statusfile);
-}
-
-struct fileinfo* new_info()
-{
- struct fileinfo* rv;
- rv = (struct fileinfo *)malloc(sizeof(struct fileinfo));
- if ( rv )
+ if ( fd < 0 )
{
- rv->next = NULL;
- rv->name = NULL;
- rv->type = NULL;
- rv->fd = -1;
- rv->position = 0;
- rv->inode = 0;
- }
- return rv;
-}
-
-void read_config()
-{
- /* The config file now is a simple list of type, filenames;
- * it should be more user-friendly in the future!
- */
- FILE* conffile;
- char buffer[4096];
-
- conffile = fopen(confname, "r");
- if ( !conffile )
- {
- fprintf(stderr,
- "logrunner: can\'t open configuration file \'%s\': ",
- confname);
- perror("");
- exit(2);
- }
- while ( fgets(buffer, sizeof(buffer), conffile) )
- {
- int len = strlen(buffer);
- char *fname;
- if ( buffer[len-1] == '\n' )
- {
- buffer[--len] = 0;
- }
- fname = strchr(buffer, '\t');
- if ( fname )
+ /* attempt to open the file */
+ fd = open(name, O_RDONLY);
+ if ( fd < 0 )
{
- struct fileinfo* fi;
- fi = new_info();
- *(fname++) = 0;
- if ( fi )
- {
- fi->name = strdup(fname);
- fi->type = strdup(buffer);
- fi->next = filelist;
- filelist = fi;
- }
- else
- {
- fprintf(stderr, "logrunner: out of memory\n");
- exit(2);
- }
+ std::cerr << "!!! logfile: open failed: " << name << ", " << strerror(errno) << "\n";
+ return;
}
- else if ( len > 0 )
+ lseek(fd, position, SEEK_SET);
+
+ if (verbose)
{
- fprintf(stderr, "logrunner: WARNING: bad config line: %s\n", buffer);
+ std::cerr << "*** logfile: opened: " << name;
+ std::cerr << "\n*** logfile: resumed read from position ";
+ std::cerr << position << "\n";
+ std::cerr << "This logfile has " << filter.size() << " filters.\n";
}
}
- fclose(conffile);
+
+ copy_data();
}
-void copy_data(struct fileinfo *f)
+void LogFile::copy_data()
{
char buffer[4096];
int ndata;
+ String logline("");
/* read data and dump to output */
- ndata = read(f->fd, buffer, sizeof(buffer));
+ ndata = read(fd, buffer, sizeof(buffer)-1);
if ( ndata > 0 )
{
- xml_header(f);
+ xml_header(type);
while ( ndata > 0 )
{
// Make a separate <gcmt:raw> element from each line
{
// Another line found - make the split.
*nextline++ = '\0';
- write(out_stream, "<gcmt:raw>", 10);
- String logline(line);
- logline = XML_Entities(logline);
- write(out_stream, logline, ~logline);
- write(out_stream, "</gcmt:raw>\n", 12);
+ logline += line;
+
+ // See if we have to select the host and apply filters to this log entry
+
+ bool filtered_out = false;
+
+ if (fromhost && (fromhost.in(logline) == -1 || fromhost.in(logline) > 20))
+ {
+ filtered_out = true;
+ }
+
+ std::list<regex>::iterator f = filter.begin();
+ while (f != filter.end())
+ {
+ if (logline == *f)
+ {
+ filtered_out = true;
+ }
+ f++;
+ }
+
+ if (!filtered_out)
+ {
+ logline = XML_Entities(logline);
+
+ write(out_stream, "<gcmt:raw>", 10);
+ write(out_stream, logline, ~logline);
+ write(out_stream, "</gcmt:raw>\n", 12);
+ }
+ else if (verbose)
+ {
+ std::cerr << logline << " is filtered out.\n";
+ }
+
+ line = nextline;
+ logline = "";
+ }
+ else
+ {
+ // We have a buffer full of data but no newline.
+ // Flush the buffer into logline and continue reading.
+
+ nextline[0] = '\0';
+ logline += line;
line = nextline;
}
}
// There is still an incomplete line in the buffer.
memmove(buffer, line, nextline - line);
}
- f->position += ndata - (nextline - line);
+ position += ndata - (nextline - line);
ndata -= line - buffer;
- ndata += read(f->fd, buffer + (nextline - line), sizeof(buffer) - (nextline - line));
+ ndata += read(fd, buffer + (nextline - line), sizeof(buffer) - 1 - (nextline - line));
}
xml_footer();
something_has_changed = 1;
}
if ( ndata < 0 )
{
- char error[80];
- sprintf(error, ", errno=%d\n", errno);
- output("!!! logfile: read failed: ");
- output(f->name);
- output(error);
+ std::cerr << "!!! logfile: read failed: " << name << ", " << strerror(errno) << "\n";
+ std::cerr << " file descriptor = " << fd << "\n";
}
}
-void do_file(struct fileinfo *f)
+void write_status_file()
{
- struct stat statinfo;
- char buffer[80];
+ FILE* dumpfile;
+ int localerror = 0;
- /* inode check */
- if ( stat(f->name, &statinfo) )
+ std::ofstream statusfile(newstatusname);
+ std::list<LogFile>::iterator lf = logs_to_run.begin();
+ while (lf != logs_to_run.end())
{
- sprintf(buffer, ", errno=%d\n", errno);
- output("!!! logfile: stat failed: ");
- output(f->name);
- output(buffer);
+ if (verbose)
+ {
+ std::cerr << "Write status for " << lf->pathname() << "\n";
+ }
+ statusfile << lf->status() << "\n";
+ lf++;
}
- else
+
+ if ( localerror == 0 )
+ {
+ if ( rename(newstatusname, statusname) )
+ {
+ output_error("!!! dumpstatus: rename failed");
+ }
+ }
+ something_has_changed = 0;
+}
+
+void read_status()
+{
+ FILE* statusfile;
+ char buffer[4096];
+ char* xp;
+ long ino;
+ unsigned long pos;
+ struct fileinfo* fp;
+
+ statusfile = fopen(statusname, "r");
+ if ( statusfile == NULL )
{
- if ( statinfo.st_ino != f->inode )
+ fprintf(stderr, "logrunner: can\'t open status file \'%s\': ",
+ statusname);
+ perror("");
+ return;
+ }
+ while ( fgets(buffer, sizeof(buffer), statusfile ) )
+ {
+ xp = strchr(buffer, ' ');
+ if ( xp )
{
- if ( f->fd >= 0 )
+ *xp = 0;
+ sscanf(xp+1, "%ld %lu", &ino, &pos);
+
+ // Search for the logfile in the list of logs to run
+
+ std::list<LogFile>::iterator lf = logs_to_run.begin();
+ while (lf != logs_to_run.end())
{
- copy_data(f);
- close(f->fd);
- f->fd = -1;
+ if (lf->pathname() == String(buffer))
+ {
+ if (verbose)
+ {
+ std::cerr << "Read status for " << lf->pathname() << "\n";
+ }
+ lf->update_status(ino, pos);
+ }
+ lf++;
}
- output("@@@ logfile: logrotate detected: ");
- output(f->name);
- output("\n");
- f->inode = statinfo.st_ino;
- f->position = 0;
+
}
}
- if ( f->fd < 0 )
+ fclose(statusfile);
+}
+
+void read_config(gnucomo_config cfg)
+{
+ /*
+ * The configuration for logrunner is stored in the central Gnucomo
+ * configuration file. Multiple 'logfile' elements can be put in this
+ * XML file, one for each logfile to scan.
+ * Each 'logfile' element has at least a 'name' and a 'type' element
+ * that denote the pathname of the logfile and the type of its content.
+ */
+
+ String logfilename;
+ String logfiletype;
+ String fromhost;
+
+ int l = 0;
+
+ logfilename = cfg.find_parameter("logfile", "name", l);
+ while (logfilename != String(""))
{
- /* attempt to open the file */
- f->fd = open(f->name, O_RDONLY);
- if ( f->fd < 0 )
+ logfiletype = cfg.find_parameter("logfile", "type", l);
+ fromhost = cfg.find_parameter("logfile", "fromhost", l);
+ if (verbose)
{
- sprintf(buffer, ", errno=%d\n", errno);
- output("!!! logfile: open failed: ");
- output(f->name);
- output(buffer);
- return;
+ std::cerr << "LogFile " << logfilename << " of type "
+ << logfiletype << " from host " << fromhost << "\n";
}
- output("*** logfile: opened: ");
- output(f->name);
- sprintf(buffer,
- "\n*** logfile: resumed read from position %ld\n",
- (long) lseek(f->fd, f->position, SEEK_SET) );
- output(buffer);
+
+ LogFile lf(logfilename, logfiletype, fromhost);
+
+ int f = 0;
+ String exp = cfg.find_parameter("logfile", "filter", l, f);
+ while (exp != "")
+ {
+ lf.add_filter(exp);
+
+ f++;
+ exp = cfg.find_parameter("logfile", "filter", l, f);
+ }
+
+ logs_to_run.push_back(lf);
+
+ l++;
+ logfilename = cfg.find_parameter("logfile", "name", l);
}
- copy_data(f);
}
void process_options(int argc, char* argv[])
{
- const char* const options = "1c:p:";
+ const char* const options = "1c:l:p:s:v";
int opt;
+ String logfilename;
+
opt = getopt(argc, argv, options);
while ( opt != -1 )
{
case 'c':
confname = strdup(optarg);
break;
+ case 'l':
+ logfilename = optarg;
+ break;
case 'p':
port = atoi(optarg);
break;
+ case 's':
+ statusname = strdup(optarg);
+ newstatusname = (char *)malloc(strlen(statusname) + 6);
+ strcpy(newstatusname, statusname);
+ strcat(newstatusname, ".new");
+ break;
+ case 'v':
+ verbose = 1;
+ break;
default:
fputs(usage, stderr);
exit(2);
fputs(usage, stderr);
exit(2);
}
+
+ if (logfilename)
+ {
+ LogFile lf(logfilename, String("system log"), String(""));
+ logs_to_run.push_back(lf);
+ }
}
int main(int argc, char* argv[])
{
+
+ gnucomo_config cfg;
+
process_options(argc, argv);
open_output();
- read_config();
+ if (!cfg.read(confname))
+ {
+ std::cerr << "Can not read Gnucomo configuration file for " << confname << ".\n";
+ exit(1);
+ }
+
+ if (logs_to_run.empty())
+ {
+ read_config(cfg);
+ }
read_status();
set_signal_handler();
while ( sig_seen == 0 )
{
- struct fileinfo* fip;
- fip = filelist;
- while ( fip )
+ std::list<LogFile>::iterator lf = logs_to_run.begin();
+ while (lf != logs_to_run.end() && !(something_has_changed && oneshot))
{
- do_file(fip);
- if (something_has_changed && oneshot)
- {
- break;
- }
- fip = fip->next;
+ lf->do_file();
+ lf++;
}
+
if ( something_has_changed )
{
- write_status_file(filelist);
+ write_status_file();
}
if ( oneshot )
{