2 /**************************************************************************
3 ** (c) Copyright 2002, Andromeda Technology & Automation
4 ** This is free software; you can redistribute it and/or modify it under the
5 ** terms of the GNU General Public License, see the file COPYING.
6 ***************************************************************************
7 ** MODULE INFORMATION *
8 ***********************
9 ** FILE NAME : message.cpp
10 ** SYSTEM NAME : Gnucomo - Gnu Computer Monitoring
11 ** VERSION NUMBER : $Revision: 1.7 $
13 ** DESCRIPTION : Implementation of the message handling classes
18 ***************************************************************************
19 ** ADMINISTRATIVE INFORMATION *
20 ********************************
21 ** ORIGINAL AUTHOR : Arjen Baart - arjen@andromeda.nl
22 ** CREATION DATE : Sep 16, 2002
23 ** LAST UPDATE : Feb 19, 2003
25 **************************************************************************/
27 /*****************************
29 Revision 1.7 2003-02-21 08:08:05 arjen
30 Gcm_input also detects packages that are removed from the system.
31 Determining the version number of a package in a RPM
32 list is improved. Only the last one or two parts of the string that
33 begin with a '-' and a number are considered the version.
35 Revision 1.6 2003/02/05 09:37:51 arjen
36 Create notifications when a new package is discovered
37 in a 'rpm -qa' list or when the version of a package is changed.
39 Revision 1.4 2002/12/06 22:26:28 arjen
40 Set the value of log.processed to FALSE when inserting a
41 new log entry into the database
42 When a syslog entry arrives from last year, gcm_input subtracts one from the
43 year of arrival to create the year of the log entry.
44 Read output from "rpm -qa" and enter packages in the parameter table.
46 Revision 1.3 2002/11/09 08:04:27 arjen
47 Added a reference to the GPL
49 Revision 1.2 2002/11/04 10:13:36 arjen
50 Use proper namespace for iostream classes
52 Revision 1.1 2002/10/05 10:25:49 arjen
53 Creation of gcm_input and a first approach to a web interface
55 *****************************/
57 static const char *RCSID = "$Id: message.cpp,v 1.7 2003-02-21 08:08:05 arjen Exp $";
62 extern bool verbose; /* Defined in the main application */
65 /* Utility functions */
67 String SQL_Escape(String s);
69 /*=========================================================================
71 ** SYNOPSIS : bool operator >> (message_buffer &, String &)
73 ** RETURN VALUE : True if input was available.
75 ** DESCRIPTION : Input operator. Read the next line from the message.
81 ** LAST MODIFIED : Nov 04, 2002
82 **=========================================================================
85 bool operator >> (message_buffer &b, String &s)
87 bool input_ok = false;
89 if (b.next_line == b.buffer.end())
95 b.buffer.push_back(l);
97 // next_line keeps pointing to the end.
112 client_message::client_message(std::istream *in, gnucomo_database db)
119 gpg_encrypted = false;
120 classification = UNKNOWN;
124 static const String syslog_date_re("[[:alpha:]]{3} [ 123][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}");
125 static const String mail_date_re("[[:alpha:]]{3}, [ 123]?[0-9] [[:alpha:]]{3} [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}");
126 static const String unix_date_re("[[:alpha:]]{3} [[:alpha:]]{3} [ 123][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}");
128 static const regex re_syslog(syslog_date_re + " [[:alnum:]]+ [[:alpha:]]+.*:.+");
129 static const regex re_PGP("-----BEGIN PGP MESSAGE-----");
130 static const regex re_dump("^ *DUMP: Date of this level");
131 static const regex re_accesslog("(GET|POST) .+ HTTP");
132 static const regex re_errorlog("^\\[" + unix_date_re + "\\] \\[(error|notice)\\] .+");
133 static const regex re_rpm("[[:alnum:]+-]+-[0-9][[:alnum:].-]");
135 static const regex re_syslog_date("[[:alpha:]]{3} [ 123][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}");
136 static const regex re_uxmail_from("^From - " + unix_date_re);
137 static const regex re_mail_From("^From:[[:blank:]]+");
138 static const regex re_mail_Date("^Date:[[:blank:]]+" + mail_date_re);
139 static const regex re_email_address("[[:alnum:]_.-]+@[[:alnum:]_.-]+");
140 static const regex re_email_user("[[:alnum:]_.-]+@");
142 /*=========================================================================
144 ** SYNOPSIS : double classify(String host, date arriv_d, hour arriv_t, String serv)
146 ** RETURN VALUE : The certainty with which the message is classified.
154 ** LAST MODIFIED : Nov 16, 2002
155 **=========================================================================
158 double client_message::classify(String host, UTC arriv, String serv)
166 /* First, check if the message has a mail header. */
168 if (input >> line && line == re_uxmail_from)
174 /* Scan ahead for the hostname and date of arrival. */
176 while (input >> line && line != "")
178 if (line == re_mail_From)
180 from_address = line(re_email_address);
181 from_address(re_email_user) = ""; // Remove the user part;
182 hostname = from_address;
184 if (line == re_mail_Date)
186 arrival = UTC(line(regex(mail_date_re)));
192 // Push the first line back, we need to read it again.
197 * Now that we have the mail header out of the way, try to figure
198 * out what the content of the message is.
202 while (input >> line && certainty < 0.9)
204 std::cout << " testing: " << line << "\n";
205 if (line == re_syslog)
208 classification = SYSLOG;
211 std::cout << "Syslog detected.\n";
214 else if (line == re_PGP)
217 gpg_encrypted = true;
218 std::cerr << "The message is PGP/GnuPG encrypted.\n";
220 else if (line == re_dump)
225 std::cout << "DUMP output detected.\n";
228 else if (line == re_accesslog)
231 classification = ACCESSLOG;
235 std::cout << "HTTP access log detected.\n";
238 else if (line == re_errorlog)
241 classification = ERRORLOG;
245 std::cout << "HTTP error log detected.\n";
248 else if (line == re_rpm)
251 classification = RPMLIST;
255 std::cout << "RPM package list detected.\n";
263 std::cerr << "Can not determine the hostname where the message came from.\n";
266 else if (!arrival.proper())
268 std::cerr << "Arrival time is not knwon.\n";
279 /*=========================================================================
281 ** SYNOPSIS : int enter()
283 ** RETURN VALUE : The number of lines successfully parsed from the input
291 ** LAST MODIFIED : Feb 19, 2003
292 **=========================================================================
295 int client_message::enter()
301 String change_notification("");
302 String create_notification("");
303 bool initial_entry = false;
305 std::list<String> packages;
308 /* Double-check the classification of the message */
310 if (classification == UNKNOWN || certainty < 0.9 || gpg_encrypted)
317 // Skip the mail header.
319 while (input >> line && line != "");
322 /* Try to find the host in the database */
326 objectid = database.find_host(hostname);
329 std::cerr << "Please define the host " << hostname << " in the database.\n";
334 std::cout << "Object id for " << hostname << " is " << objectid << "\n";
337 if (classification == RPMLIST)
342 /* Read all packages, so we will know which ones are */
343 /* missing at the end. */
345 qry = "select name from parameter where objectid='";
346 qry += objectid + "' and class='package'";
347 n_packages = database.Query(qry);
348 initial_entry = n_packages == 0;
350 std::cout << n_packages << " packages in database.\n";
351 for (int t = 0; t < n_packages; t++)
353 packages.push_back(database.Field(t, "name"));
355 std::cout << "Package list built: " << packages.size() << ".\n";
358 /* Scan the input line by line, entring records into the database */
360 String rest; // Rest of the line to be parsed
362 while (input >> line)
366 std::cout << line << "\n";
370 /* Check each line if it contains valid information */
374 switch (classification)
380 check = &re_accesslog;
383 check = &re_errorlog;
396 String insertion("insert into log (objectid, servicecode,"
397 " object_timestamp, timestamp, rawdata, processed) values (");
400 switch (classification)
405 if (log_date.Year() < 0 || log_date.Year() > 2500)
407 // The year is not in the log file. Assume the year of arrival,
408 // unless this puts the log entry at a later date than the arrival date.
409 // This happens e.g. when a log entry from December arrives in Januari.
411 log_date = date(log_date.Day(), log_date.Month(), date(arrival).Year());
412 if (log_date > date(arrival))
414 log_date = date(log_date.Day(), log_date.Month(), date(arrival).Year() - 1);
420 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
424 if (rest(0,i) == hostname(0,i))
429 std::cout << " Hostname matches.\n";
430 std::cout << " rest = " << rest << "\n";
432 for (i = 0; isalpha(rest[i]) && i < ~rest; i++);
435 std::cout << " Service name = " << rest(0,i) << "\n";
438 /* Insert a new record into the log table */
440 insertion += "'" + objectid + "',";
441 insertion += "'" + rest(0,i) + "',";
442 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
443 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
444 insertion += "'" + SQL_Escape(line) + "',FALSE";
449 std::cout << insertion << "\n";
453 database.Query(insertion);
465 std::cerr << " Hostname " << rest(0,i) << " does not match.\n";
470 datestring = line(regex("\\[.+\\]"));
473 datestring[datestring.index(':')] = ' ';
474 log_date = datestring;
475 log_time = datestring;
478 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
480 insertion += "'" + objectid + "',";
481 insertion += "'" + service + "',";
482 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
483 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
484 insertion += "'" + SQL_Escape(line) + "',FALSE";
489 std::cout << insertion << "\n";
493 database.Query(insertion);
505 datestring = line(regex("\\[.+\\]"));
508 log_date = datestring;
509 log_time = datestring;
512 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
514 insertion += "'" + objectid + "',";
515 insertion += "'" + service + "',";
516 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
517 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
518 insertion += "'" + SQL_Escape(line) + "',FALSE";
523 std::cout << insertion << "\n";
527 database.Query(insertion);
539 // Scan a list of packages and versions from "rpm -a".
540 // A similar listing can be created on IRIX 6.5 by using the
541 // command "showprods -3 -n|awk '{printf "%s-%s\n",$2,$3}'|grep -v '^[-=]' \
542 // |grep -v Version-Description".
544 // We have to separate the package name and the version.
545 // The separation is marked by a '-', followed by a digit.
546 // However, there may be other sequences of '-'digit in the package name,
547 // do we have to scan ahead until there is at most one such sequence
548 // left in the version string. The '-'digit seqeunce inside the
549 // version usually separates the version and the release number.
551 int version_start, next_version_start;
555 next_version_start = i;
557 while (i < ~line - 1)
559 while (i < ~line - 1 && !(line[i] == '-' && isdigit(line[i + 1])))
565 version_start = next_version_start;
566 next_version_start = i;
571 String package(line(0,version_start));
572 String version(line(version_start + 1, ~line));
579 std::cout << "Package is " << package;
580 std::cout << ", version is " << version << "\n";
583 // Construct a qry to check the package's existance
585 qry = "select paramid from parameter where objectid='";
586 qry += objectid + "' and class='package' and name='";
587 qry += package + "'";
589 if (database.Query(qry) == 1)
591 std::list<String>::iterator lp;
593 lp = find(packages.begin(), packages.end(), package);
594 if (lp != packages.end())
600 std::cerr << "Could NOT find " << package << " in list.\n";
603 paramid = database.Field(0, "paramid");
604 qry = "select value from property where paramid='";
605 qry += paramid + "' and name='version'";
606 if (database.Query(qry) == 0)
608 std::cerr << "Database corruption: Package " << package;
609 std::cerr << " does not have a 'version' property.\n";
611 else if (database.Field(0, "value") != version)
615 std::cout << " Parameter " << package << " has different version\n";
617 insertion = "update property set value='";
618 insertion += version + "' where paramid='";
619 insertion += paramid + "' and name='version'";
621 insert_h = "insert into history (paramid, modified, change_nature, changed_property, new_value)";
622 insert_h += " values ('";
623 insert_h += paramid + "', '" + arrival.format("%Y-%m-%d %T") + "', 'MODIFIED', 'version', '";
624 insert_h += version + "')";
626 database.Query(insertion);
627 database.Query(insert_h);
629 if (change_notification == "")
631 remark = "Gnucomo detected a different version for package parameter(s) ";
632 change_notification = database.new_notification(objectid, "property modified", remark);
635 if (change_notification != "")
637 insertion = "insert into parameter_notification (notificationid, paramid) values ('";
638 insertion += change_notification + "', '";
639 insertion += paramid + "')";
641 database.Query(insertion);
645 std::cerr << "gcm_input ERROR: Cannot create 'property modified' notification.\n";
652 std::cout << " Parameter " << package << " has not changed.\n";
661 std::cout << " Parameter " << package << " does not exist.\n";
663 // Create a new package parameter, including version property and history record
665 insertion = "insert into parameter (objectid, name, class, description) values ('";
666 insertion += objectid + "', '" + package + "', 'package', 'RPM package " + package + "')";
670 std::cout << insertion << "\n";
674 database.Query(insertion);
675 qry = "select paramid from parameter where objectid='";
676 qry += objectid + "' and class='package' and name='";
677 qry += package + "'";
679 paramid = database.Field(0, "paramid");
682 insertion = "insert into property (paramid, name, value, type) values ('";
683 insertion += paramid + "', 'version', '";
684 insertion += version + "', 'STATIC')";
685 insert_h = "insert into history (paramid, modified, change_nature, changed_property, new_value)";
686 insert_h += " values ('";
687 insert_h += paramid + "', '" + arrival.format("%Y-%m-%d %T") + "', 'CREATED', 'version', '";
688 insert_h += version + "')";
692 std::cout << insertion << "\n" << insert_h << "\n";
696 database.Query(insertion);
697 database.Query(insert_h);
700 if (create_notification == "")
702 remark = "Gnucomo detected new parameter(s) of class package";
703 create_notification = database.new_notification(objectid, "parameter created", remark);
705 if (create_notification != "")
707 insertion = "insert into parameter_notification (notificationid, paramid) values ('";
708 insertion += create_notification + "', '";
709 insertion += paramid + "')";
711 database.Query(insertion);
715 std::cerr << "gcm_input ERROR: Cannot create 'parameter created' notification.\n";
733 std::cerr << "gcm_input WARNING: Not a valid line: " << line << "\n";
737 if (classification == RPMLIST)
739 std::list<String>::iterator lp;
740 String remove_notification("");
743 * If there are any packages left in the list, they seem to have
744 * disappeared from the system.
747 for (lp = packages.begin(); lp != packages.end(); lp++)
753 // Construct a qry to check the package's existance
755 qry = "select paramid from parameter where objectid='";
756 qry += objectid + "' and class='package' and name='";
759 if (database.Query(qry) == 1)
761 paramid = database.Field(0, "paramid");
762 qry ="select change_nature from history where paramid='";
763 qry += paramid + "' order by modified desc";
764 if (database.Query(qry) <= 0)
766 std::cerr << "Database ERROR: no history record for parameter " << *lp << ".\n";
768 else if (database.Field(0, "change_nature") != "REMOVED")
772 std::cout << "Removing parameter " << *lp << ".\n";
775 insert = "insert into history (paramid, modified, change_nature)";
776 insert += " values ('";
777 insert += paramid + "', '" + arrival.format("%Y-%m-%d %T") + "', 'REMOVED')";
779 database.Query(insert);
781 if (remove_notification == "")
783 remark = "Gnucomo detected that package(s) have disappeared ";
784 remove_notification = database.new_notification(objectid, "parameter removed", remark);
787 if (remove_notification != "")
789 insert = "insert into parameter_notification (notificationid, paramid) values ('";
790 insert += remove_notification + "', '";
791 insert += paramid + "')";
793 database.Query(insert);
797 std::cerr << "gcm_input ERROR: Cannot create 'parameter removed' notification.\n";
806 std::cout << nr_lines << " lines parsed from the log file.\n";
811 /*=========================================================================
813 ** SYNOPSIS : String SQL_Escape(String)
817 ** DESCRIPTION : Insert backslashes before single quotes.
824 **=========================================================================
827 String SQL_Escape(String s)
831 for (i = 0; i < ~s; i++)