2 /**************************************************************************
3 ** (c) Copyright 2002, Andromeda Technology & Automation
4 ** This is free software; you can redistribute it and/or modify it under the
5 ** terms of the GNU General Public License, see the file COPYING.
6 ***************************************************************************
7 ** MODULE INFORMATION *
8 ***********************
9 ** FILE NAME : message.cpp
10 ** SYSTEM NAME : Gnucomo - Gnu Computer Monitoring
11 ** VERSION NUMBER : $Revision: 1.8 $
13 ** DESCRIPTION : Implementation of the message handling classes
18 ***************************************************************************
19 ** ADMINISTRATIVE INFORMATION *
20 ********************************
21 ** ORIGINAL AUTHOR : Arjen Baart - arjen@andromeda.nl
22 ** CREATION DATE : Sep 16, 2002
23 ** LAST UPDATE : Feb 28, 2003
25 **************************************************************************/
27 /*****************************
29 Revision 1.8 2003-03-16 09:42:40 arjen
30 Read IRIX system logs.
32 Revision 1.7 2003/02/21 08:08:05 arjen
33 Gcm_input also detects packages that are removed from the system.
34 Determining the version number of a package in a RPM
35 list is improved. Only the last one or two parts of the string that
36 begin with a '-' and a number are considered the version.
38 Revision 1.6 2003/02/05 09:37:51 arjen
39 Create notifications when a new package is discovered
40 in a 'rpm -qa' list or when the version of a package is changed.
42 Revision 1.4 2002/12/06 22:26:28 arjen
43 Set the value of log.processed to FALSE when inserting a
44 new log entry into the database
45 When a syslog entry arrives from last year, gcm_input subtracts one from the
46 year of arrival to create the year of the log entry.
47 Read output from "rpm -qa" and enter packages in the parameter table.
49 Revision 1.3 2002/11/09 08:04:27 arjen
50 Added a reference to the GPL
52 Revision 1.2 2002/11/04 10:13:36 arjen
53 Use proper namespace for iostream classes
55 Revision 1.1 2002/10/05 10:25:49 arjen
56 Creation of gcm_input and a first approach to a web interface
58 *****************************/
60 static const char *RCSID = "$Id: message.cpp,v 1.8 2003-03-16 09:42:40 arjen Exp $";
65 extern bool verbose; /* Defined in the main application */
68 /* Utility functions */
70 String SQL_Escape(String s);
72 /*=========================================================================
74 ** SYNOPSIS : bool operator >> (message_buffer &, String &)
76 ** RETURN VALUE : True if input was available.
78 ** DESCRIPTION : Input operator. Read the next line from the message.
84 ** LAST MODIFIED : Nov 04, 2002
85 **=========================================================================
88 bool operator >> (message_buffer &b, String &s)
90 bool input_ok = false;
92 if (b.next_line == b.buffer.end())
98 b.buffer.push_back(l);
100 // next_line keeps pointing to the end.
115 client_message::client_message(std::istream *in, gnucomo_database db)
122 gpg_encrypted = false;
123 classification = UNKNOWN;
127 static const String syslog_date_re("[[:alpha:]]{3} [ 123][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}");
128 static const String mail_date_re("[[:alpha:]]{3}, [ 123]?[0-9] [[:alpha:]]{3} [0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2} [+-][0-9]{4}");
129 static const String unix_date_re("[[:alpha:]]{3} [[:alpha:]]{3} [ 123][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2} [0-9]{4}");
131 static const regex re_syslog(syslog_date_re + " [[:alnum:]]+ [[:alpha:]]+.*:.+");
132 static const regex re_syslog_irix(syslog_date_re + " [0-7][A-T]:[[:alnum:]]+ [[:alpha:]]+.*:.+");
133 static const regex re_PGP("-----BEGIN PGP MESSAGE-----");
134 static const regex re_dump("^ *DUMP: Date of this level");
135 static const regex re_accesslog("(GET|POST) .+ HTTP");
136 static const regex re_errorlog("^\\[" + unix_date_re + "\\] \\[(error|notice)\\] .+");
137 static const regex re_rpm("[[:alnum:]+-]+-[0-9][[:alnum:].-]");
139 static const regex re_syslog_date("[[:alpha:]]{3} [ 123][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}");
140 static const regex re_uxmail_from("^From - " + unix_date_re);
141 static const regex re_mail_From("^From:[[:blank:]]+");
142 static const regex re_mail_Date("^Date:[[:blank:]]+" + mail_date_re);
143 static const regex re_email_address("[[:alnum:]_.-]+@[[:alnum:]_.-]+");
144 static const regex re_email_user("[[:alnum:]_.-]+@");
146 /*=========================================================================
148 ** SYNOPSIS : double classify(String host, date arriv_d, hour arriv_t, String serv)
150 ** RETURN VALUE : The certainty with which the message is classified.
158 ** LAST MODIFIED : Nov 16, 2002
159 **=========================================================================
162 double client_message::classify(String host, UTC arriv, String serv)
170 /* First, check if the message has a mail header. */
172 if (input >> line && line == re_uxmail_from)
178 /* Scan ahead for the hostname and date of arrival. */
180 while (input >> line && line != "")
182 if (line == re_mail_From)
184 from_address = line(re_email_address);
185 from_address(re_email_user) = ""; // Remove the user part;
186 hostname = from_address;
188 if (line == re_mail_Date)
190 arrival = UTC(line(regex(mail_date_re)));
196 // Push the first line back, we need to read it again.
201 * Now that we have the mail header out of the way, try to figure
202 * out what the content of the message is.
206 while (input >> line && certainty < 0.9)
208 std::cout << " testing: " << line << "\n";
209 if (line == re_syslog)
212 classification = SYSLOG;
215 std::cout << "Syslog detected.\n";
218 else if (line == re_syslog_irix)
221 classification = SYSLOG_IRIX;
224 std::cout << "IRIX Syslog detected.\n";
227 else if (line == re_PGP)
230 gpg_encrypted = true;
231 std::cerr << "The message is PGP/GnuPG encrypted.\n";
233 else if (line == re_dump)
238 std::cout << "DUMP output detected.\n";
241 else if (line == re_accesslog)
244 classification = ACCESSLOG;
248 std::cout << "HTTP access log detected.\n";
251 else if (line == re_errorlog)
254 classification = ERRORLOG;
258 std::cout << "HTTP error log detected.\n";
261 else if (line == re_rpm)
264 classification = RPMLIST;
268 std::cout << "RPM package list detected.\n";
276 std::cerr << "Can not determine the hostname where the message came from.\n";
279 else if (!arrival.proper())
281 std::cerr << "Arrival time is not knwon.\n";
292 /*=========================================================================
294 ** SYNOPSIS : int enter()
296 ** RETURN VALUE : The number of lines successfully parsed from the input
304 ** LAST MODIFIED : Feb 19, 2003
305 **=========================================================================
308 int client_message::enter()
314 String change_notification("");
315 String create_notification("");
316 bool initial_entry = false;
318 std::list<String> packages;
321 /* Double-check the classification of the message */
323 if (classification == UNKNOWN || certainty < 0.9 || gpg_encrypted)
330 // Skip the mail header.
332 while (input >> line && line != "");
335 /* Try to find the host in the database */
339 objectid = database.find_host(hostname);
342 std::cerr << "Please define the host " << hostname << " in the database.\n";
347 std::cout << "Object id for " << hostname << " is " << objectid << "\n";
350 if (classification == RPMLIST)
355 /* Read all packages, so we will know which ones are */
356 /* missing at the end. */
358 qry = "select name from parameter where objectid='";
359 qry += objectid + "' and class='package'";
360 n_packages = database.Query(qry);
361 initial_entry = n_packages == 0;
363 std::cout << n_packages << " packages in database.\n";
364 for (int t = 0; t < n_packages; t++)
366 packages.push_back(database.Field(t, "name"));
368 std::cout << "Package list built: " << packages.size() << ".\n";
371 /* Scan the input line by line, entring records into the database */
373 String rest; // Rest of the line to be parsed
375 while (input >> line)
379 std::cout << line << "\n";
383 /* Check each line if it contains valid information */
387 switch (classification)
393 check = &re_syslog_irix;
396 check = &re_accesslog;
399 check = &re_errorlog;
412 String insertion("insert into log (objectid, servicecode,"
413 " object_timestamp, timestamp, rawdata, processed) values (");
416 switch (classification)
421 if (log_date.Year() < 0 || log_date.Year() > 2500)
423 // The year is not in the log file. Assume the year of arrival,
424 // unless this puts the log entry at a later date than the arrival date.
425 // This happens e.g. when a log entry from December arrives in Januari.
427 log_date = date(log_date.Day(), log_date.Month(), date(arrival).Year());
428 if (log_date > date(arrival))
430 log_date = date(log_date.Day(), log_date.Month(), date(arrival).Year() - 1);
436 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
440 if (rest(0,i) == hostname(0,i))
445 std::cout << " Hostname matches.\n";
446 std::cout << " rest = " << rest << "\n";
448 for (i = 0; isalpha(rest[i]) && i < ~rest; i++);
451 std::cout << " Service name = " << rest(0,i) << "\n";
454 /* Insert a new record into the log table */
456 insertion += "'" + objectid + "',";
457 insertion += "'" + rest(0,i) + "',";
458 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
459 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
460 insertion += "'" + SQL_Escape(line) + "',FALSE";
465 std::cout << insertion << "\n";
469 database.Query(insertion);
481 std::cerr << " Hostname " << rest(0,i) << " does not match.\n";
488 if (log_date.Year() < 0 || log_date.Year() > 2500)
490 // The year is not in the log file. Assume the year of arrival,
491 // unless this puts the log entry at a later date than the arrival date.
492 // This happens e.g. when a log entry from December arrives in Januari.
494 log_date = date(log_date.Day(), log_date.Month(), date(arrival).Year());
495 if (log_date > date(arrival))
497 log_date = date(log_date.Day(), log_date.Month(), date(arrival).Year() - 1);
503 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
507 if (rest(0,i) == hostname(0,i))
512 std::cout << " Hostname matches.\n";
513 std::cout << " rest = " << rest << "\n";
515 for (i = 0; isalpha(rest[i]) && i < ~rest; i++);
518 std::cout << " Service name = " << rest(0,i) << "\n";
521 /* Insert a new record into the log table */
523 insertion += "'" + objectid + "',";
524 insertion += "'" + rest(0,i) + "',";
525 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
526 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
527 insertion += "'" + SQL_Escape(line) + "',FALSE";
532 std::cout << insertion << "\n";
536 database.Query(insertion);
548 std::cerr << " Hostname " << rest(0,i) << " does not match.\n";
553 datestring = line(regex("\\[.+\\]"));
556 datestring[datestring.index(':')] = ' ';
557 log_date = datestring;
558 log_time = datestring;
561 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
563 insertion += "'" + objectid + "',";
564 insertion += "'" + service + "',";
565 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
566 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
567 insertion += "'" + SQL_Escape(line) + "',FALSE";
572 std::cout << insertion << "\n";
576 database.Query(insertion);
588 datestring = line(regex("\\[.+\\]"));
591 log_date = datestring;
592 log_time = datestring;
595 std::cout << " Log timestamp = " << log_date << " " << log_time << "\n";
597 insertion += "'" + objectid + "',";
598 insertion += "'" + service + "',";
599 insertion += "'" + log_date.format("%Y-%m-%d") + " " + log_time.format() + "',";
600 insertion += "'" + arrival.format("%Y-%m-%d %T") + "',";
601 insertion += "'" + SQL_Escape(line) + "',FALSE";
606 std::cout << insertion << "\n";
610 database.Query(insertion);
622 // Scan a list of packages and versions from "rpm -a".
623 // A similar listing can be created on IRIX 6.5 by using the
624 // command "showprods -3 -n|awk '{printf "%s-%s\n",$2,$3}'|grep -v '^[-=]' \
625 // |grep -v Version-Description".
627 // We have to separate the package name and the version.
628 // The separation is marked by a '-', followed by a digit.
629 // However, there may be other sequences of '-'digit in the package name,
630 // do we have to scan ahead until there is at most one such sequence
631 // left in the version string. The '-'digit seqeunce inside the
632 // version usually separates the version and the release number.
634 int version_start, next_version_start;
638 next_version_start = i;
640 while (i < ~line - 1)
642 while (i < ~line - 1 && !(line[i] == '-' && isdigit(line[i + 1])))
648 version_start = next_version_start;
649 next_version_start = i;
654 String package(line(0,version_start));
655 String version(line(version_start + 1, ~line));
662 std::cout << "Package is " << package;
663 std::cout << ", version is " << version << "\n";
666 // Construct a qry to check the package's existance
668 qry = "select paramid from parameter where objectid='";
669 qry += objectid + "' and class='package' and name='";
670 qry += package + "'";
672 if (database.Query(qry) == 1)
674 std::list<String>::iterator lp;
676 lp = find(packages.begin(), packages.end(), package);
677 if (lp != packages.end())
683 std::cerr << "Could NOT find " << package << " in list.\n";
686 paramid = database.Field(0, "paramid");
687 qry = "select value from property where paramid='";
688 qry += paramid + "' and name='version'";
689 if (database.Query(qry) == 0)
691 std::cerr << "Database corruption: Package " << package;
692 std::cerr << " does not have a 'version' property.\n";
694 else if (database.Field(0, "value") != version)
698 std::cout << " Parameter " << package << " has different version\n";
700 insertion = "update property set value='";
701 insertion += version + "' where paramid='";
702 insertion += paramid + "' and name='version'";
704 insert_h = "insert into history (paramid, modified, change_nature, changed_property, new_value)";
705 insert_h += " values ('";
706 insert_h += paramid + "', '" + arrival.format("%Y-%m-%d %T") + "', 'MODIFIED', 'version', '";
707 insert_h += version + "')";
709 database.Query(insertion);
710 database.Query(insert_h);
712 if (change_notification == "")
714 remark = "Gnucomo detected a different version for package parameter(s) ";
715 change_notification = database.new_notification(objectid, "property modified", remark);
718 if (change_notification != "")
720 insertion = "insert into parameter_notification (notificationid, paramid) values ('";
721 insertion += change_notification + "', '";
722 insertion += paramid + "')";
724 database.Query(insertion);
728 std::cerr << "gcm_input ERROR: Cannot create 'property modified' notification.\n";
735 std::cout << " Parameter " << package << " has not changed.\n";
744 std::cout << " Parameter " << package << " does not exist.\n";
746 // Create a new package parameter, including version property and history record
748 insertion = "insert into parameter (objectid, name, class, description) values ('";
749 insertion += objectid + "', '" + package + "', 'package', 'RPM package " + package + "')";
753 std::cout << insertion << "\n";
757 database.Query(insertion);
758 qry = "select paramid from parameter where objectid='";
759 qry += objectid + "' and class='package' and name='";
760 qry += package + "'";
762 paramid = database.Field(0, "paramid");
765 insertion = "insert into property (paramid, name, value, type) values ('";
766 insertion += paramid + "', 'version', '";
767 insertion += version + "', 'STATIC')";
768 insert_h = "insert into history (paramid, modified, change_nature, changed_property, new_value)";
769 insert_h += " values ('";
770 insert_h += paramid + "', '" + arrival.format("%Y-%m-%d %T") + "', 'CREATED', 'version', '";
771 insert_h += version + "')";
775 std::cout << insertion << "\n" << insert_h << "\n";
779 database.Query(insertion);
780 database.Query(insert_h);
783 if (create_notification == "")
785 remark = "Gnucomo detected new parameter(s) of class package";
786 create_notification = database.new_notification(objectid, "parameter created", remark);
788 if (create_notification != "")
790 insertion = "insert into parameter_notification (notificationid, paramid) values ('";
791 insertion += create_notification + "', '";
792 insertion += paramid + "')";
794 database.Query(insertion);
798 std::cerr << "gcm_input ERROR: Cannot create 'parameter created' notification.\n";
816 std::cerr << "gcm_input WARNING: Not a valid line: " << line << "\n";
820 if (classification == RPMLIST)
822 std::list<String>::iterator lp;
823 String remove_notification("");
826 * If there are any packages left in the list, they seem to have
827 * disappeared from the system.
830 for (lp = packages.begin(); lp != packages.end(); lp++)
836 // Construct a qry to check the package's existance
838 qry = "select paramid from parameter where objectid='";
839 qry += objectid + "' and class='package' and name='";
842 if (database.Query(qry) == 1)
844 paramid = database.Field(0, "paramid");
845 qry ="select change_nature from history where paramid='";
846 qry += paramid + "' order by modified desc";
847 if (database.Query(qry) <= 0)
849 std::cerr << "Database ERROR: no history record for parameter " << *lp << ".\n";
851 else if (database.Field(0, "change_nature") != "REMOVED")
855 std::cout << "Removing parameter " << *lp << ".\n";
858 insert = "insert into history (paramid, modified, change_nature)";
859 insert += " values ('";
860 insert += paramid + "', '" + arrival.format("%Y-%m-%d %T") + "', 'REMOVED')";
862 database.Query(insert);
864 if (remove_notification == "")
866 remark = "Gnucomo detected that package(s) have disappeared ";
867 remove_notification = database.new_notification(objectid, "parameter removed", remark);
870 if (remove_notification != "")
872 insert = "insert into parameter_notification (notificationid, paramid) values ('";
873 insert += remove_notification + "', '";
874 insert += paramid + "')";
876 database.Query(insert);
880 std::cerr << "gcm_input ERROR: Cannot create 'parameter removed' notification.\n";
889 std::cout << nr_lines << " lines parsed from the log file.\n";
894 /*=========================================================================
896 ** SYNOPSIS : String SQL_Escape(String)
900 ** DESCRIPTION : Insert backslashes before single quotes.
907 **=========================================================================
910 String SQL_Escape(String s)
914 for (i = 0; i < ~s; i++)