3 /**********************************************************************************
4 ** (c) Copyright 2002, Brenno J.S.A.A.F. de Winter, De Winter Information Solutions
5 ** This is free software; you can redistribute it and/or modify it under the
6 ** terms of the GNU General Public License, see the file COPYING.
7 ***********************************************************************************/
12 AUTHOR : Brenno J.S.A.A.F. de Winter
13 De Winter Information Solutions
14 COPYRIGHT : 2002 - De Winter Information Solutions,
15 Brenno J.S.A.A.F. de Winter
18 First : November 8th 2002
19 Gnucomo-0.0.3: December 6th 2002
20 Gnucomo-0.0.8: September 4th 2003
22 $Log: gcm_daemon.php,v $
23 Revision 1.26 2011-03-24 09:46:36 arjen
24 Resolve a hostname before adding to the abuses.
26 Revision 1.25 2007/12/12 09:06:21 arjen
27 Added a new script gcm_maintenance.php to cleanup the database
28 and check referential integrity. Purging old log entries is
29 removed from the gcm_daemon script.
31 Revision 1.24 2007/11/17 09:34:07 arjen
32 Cleaned up some leftovers
34 Revision 1.23 2007/11/03 10:31:12 arjen
35 Added the class definition for a filesystem parameter.
36 New issue type: 'property out of range'.
38 Revision 1.22 2007/10/23 11:23:52 arjen
39 Record the date when adding the number of abuses for an IP address
42 Revision 1.21 2007/01/11 13:47:41 arjen
43 Log_adv and derived tables removed.
44 Create notifications from log entries with pattern matching.
46 Revision 1.20 2005/06/04 07:15:16 arjen
47 Added pattern check on log entries with the service_pattern table.
49 Revision 1.19 2004/01/10 20:04:12 arjen
50 Send email about open notifications to an object's users.
52 Revision 1.18 2003/12/03 08:07:21 arjen
53 Changed the type of log_adv_daemon_email.delay and log_adv_daemon_email.xdelay
54 from time to interval. These delays can be more than 24 hours.
56 Revision 1.17 2003/10/29 09:58:29 arjen
57 Create separate notifications for different objects in service_check().
59 Revision 1.16 2003/09/03 12:48:48 arjen
60 Check the log table against the servies running on an object and
61 create notifications if a service is not supposed to be available
62 or is not known at all.
64 Revision 1.15 2003/09/02 12:48:09 arjen
65 BUGFIX: Secondary indices on log_notification were unique.
66 Additional information in the 'usr' table: 'display_name' and 'email'.
67 Added new issues and services.
69 Revision 1.14 2003/09/01 06:51:07 arjen
70 Accept command argument '-c config' to use an alternate
71 gnucomo configuration.
73 Revision 1.13 2003/08/14 10:22:42 arjen
76 Revision 1.12 2003/08/05 07:46:37 arjen
77 BUGFIX: Print an error message if a parameter does not have
80 Revision 1.11 2003/07/09 07:25:02 arjen
81 Gcm_daemon gathers statistics on parameters, notifications, etc. for all objects.
83 Revision 1.10 2003/03/29 08:33:58 arjen
84 In phpclasses/db.class.php: Added the database connection string as
85 an argument to the function copy_db_class.
86 Fixed the PHP member function db::db_connect(). The Postgres connection
87 string is now passed as an argument to that function.
89 Revision 1.9 2003/02/21 08:37:59 arjen
90 Added new table to the database: log_adv_daemon_email.
95 // $Id: gcm_daemon.php,v 1.26 2011-03-24 09:46:36 arjen Exp $
97 ini_set('include_path', '.:./classes:../phpclasses');
98 ini_set('html_errors', 'false');
100 define("BATCHSIZE", 50000);
102 require_once "gnucomo_config.php";
103 require_once "db.class.php";
105 // Set the standard variables //
107 $project_name = "gnucomo"; // name of the entire project
108 $app_name = "gcm_daemon"; // name of the application running
109 $developrelease = "FALSE"; // Indicates if special debug settings are needed
110 $db_version = 54; // The db_version indicates what the level of
111 // the database should be. If the database is
112 // old an update will be generated.
113 $gcmd_version = 5; // This value indicates the active version of
114 // the gcm_daemon, which is saved in the database.
115 // Log records that were not recognized before
116 // will now be recognized. The version doesn't
117 // mean anything in the overall gnucomo project.
119 //Avoid time-limit issues
122 // Scan the command arguments
124 for ($argi = 1; $argi < $argc; $argi++)
126 switch ($argv[$argi])
130 $project_name = $argv[$argi];
134 echo "Usage: gcm_daemon [-c configname]\n";
140 // Read the database settings //
141 $class_settings = new gnucomo_config();
142 if (!$class_settings->read($project_name))
144 echo "Can not read Gnucomo configuration file for $project_name.\n";
148 openlog("gnucomo", LOG_PID, LOG_DAEMON);
149 syslog(LOG_INFO, "gcm_daemon started");
151 //Open an connection to the database
152 $dbms_type = $class_settings->find_parameter("database", "type");
153 $dbms_host = $class_settings->find_parameter("database", "host");
154 $dbms_name = $class_settings->find_parameter("database", "name");
155 $dbms_user = $class_settings->find_parameter("gcm_daemon", "user");
156 $dbms_password = $class_settings->find_parameter("gcm_daemon", "password");
158 db_select($dbms_type);
160 $dbms->db_host = $dbms_host;
161 $dbms->db_name = $dbms_name;
162 $dbms->db_user = $dbms_user;
163 $dbms->db_password = $dbms_password;
164 $dbms->db_connect($class_settings->database());
166 if ($dbms->have_db_connection() == "FALSE")
168 exit ("Database connection failed.");
172 // The database connection has been made.
173 $dbms_working = copy_db_class($dbms, $class_settings->database());
176 // Verify if the database is up-to-date by checking the versionnumber
178 $local_sql = "SELECT setting_value FROM db_value WHERE setting = 'db_version' ";
179 $dbms->query($local_sql);
181 if ($dbms->fetch_row() == "TRUE")
183 $active_version = $dbms->db_result_row[0];
185 // Update the database to the most recent version.
187 if ($active_version < $db_version)
189 include ("gnucomo_db_version.php");
194 syslog (LOG_INFO, "Couldn't initialize database version. Is this a gnucomo database?");
195 die ("Couldn't initialize database version.\n");
198 // The gcm_daemon version is maintained in the database to enable
199 // automatic update actions.
201 $local_sql = "SELECT setting_value FROM db_value
202 WHERE setting = 'gcm_daemon_version'";
203 $dbms->query($local_sql);
205 if ($dbms->fetch_row() == "TRUE")
207 if ($dbms->db_result_row[0] < $gcmd_version)
210 //Update de gcm_daemon version in the database
211 $local_sql = "UPDATE db_value SET setting_value = '".$gcmd_version;
212 $local_sql .= "' WHERE setting = 'gcm_daemon_version'";
213 $dbms->query($local_sql);
219 // Now we loop the tasks that we have to do.
225 //echo "Processing logs...\n";
228 //mail_notifications();
230 // Gather the statistics for each object
232 $obj_result = $dbms->query("SELECT objectid FROM object");
233 for ($obj = 0; $obj < $dbms->num_rows($obj_result); $obj++)
235 $object = $dbms->fetch_object($obj_result, $obj);
236 // echo "Gathering statistics for object " . $object->objectid . "\n";
237 GatherStatistics($object->objectid);
240 $keep_running = false;
242 } while ($keep_running == true);
244 //Tell the log that we're ending our efforts in a nice way
246 syslog (LOG_INFO, "gcm_daemon ended nicely");
248 function process_log ()
251 /* This function will walk through the log-records that haven't been processed
252 * first a snapshot will be created of a the non-processed records.
253 * sequentially each record will dealt with. By doing that changes will be made
254 * in several log_adv_xxx tables
260 global $dbms_working;
261 global $class_settings;
265 // Find records in log that still have to be processed.
267 $local_sql = "SELECT setting_value FROM db_value WHERE setting = 'log_processing'";
268 $dbms->query($local_sql);
270 if ($dbms->fetch_row() == "TRUE")
272 $last_log = $dbms->db_result_row[0];
275 //echo "Last processed logid = $last_log \n";
277 //Query the log-table
278 $log_limit = $last_log + BATCHSIZE;
279 $local_sql = "SELECT * FROM log WHERE logid > CAST(".$last_log." AS BIGINT)
280 AND logid <= $log_limit ORDER BY logid";
281 $dbms->query($local_sql);
283 //Update the log-statistics in the object-table
284 $local_statistics_db = copy_db_class($dbms, $class_settings->database());
285 $local_findobject_db = copy_db_class($dbms, $class_settings->database());
288 $local_upper_row = $dbms->num_rows() + $last_log + 1;
289 $local_sql = "SELECT COUNT(logid), objectid from log WHERE logid > CAST(". $last_log .
290 " AS BIGINT) AND logid < CAST (" . $local_upper_row . " AS BIGINT) GROUP BY objectid";
291 $local_statistics_db->query ($local_sql);
294 for ($i = 1; $i <= $local_statistics_db->num_rows(); $i++)
296 $local_object_row = $local_statistics_db->fetch_row();
298 $local_sql = "UPDATE object SET log_count = log_count + " .
299 $local_statistics_db->db_result_row[0] . " WHERE objectid = '" .
300 $local_statistics_db->db_result_row[1] . "'";
302 $local_findobject_db->query($local_sql);
307 if ($dbms->num_rows() > 0)
310 //Create a database connection for changes in the database.
311 $dbms_changes = copy_db_class($dbms, $class_settings->database());
312 if ($dbms_changes->have_db_connection() == 'TRUE')
316 $local_sql_statistics = "";
317 $local_object_os = "";
318 $local_object_os_version = "";
320 match_log_patterns($last_log);
322 // Register that the logrecords have been processed.
324 $local_sql = "UPDATE db_value SET setting_value = '"
325 . $local_upper_row ."' where setting = 'log_processing'";
326 $dbms->query($local_sql);
329 // Update the statistics for the object-table
335 syslog (LOG_INFO, "Couldn't clone database connection.");
336 die ("Couldn't reconnect to the database.\n");
343 * Update a single statistic for some object.
344 * If it does not yet exist, it will be created.
347 function UpdateStatistic($objectid, $name, $value)
351 $result = $dbms->query("SELECT objectid FROM object_statistics WHERE
352 objectid='$objectid' AND statname='$name'");
353 if ($dbms->num_rows() == 0)
355 $dbms->query("INSERT INTO object_statistics VALUES
356 ('$objectid', '$name', '$value')");
360 $dbms->query("UPDATE object_statistics SET statvalue='$value' WHERE
361 statname='$name' AND objectid='$objectid'");
366 * Gather the statistics for a single object ($objectid).
367 * We count the number of parameters, removed parameters, notifications
368 * closed notifications and log entries. The totals of these are
369 * maintained in a separate table: object_statistics.
372 function GatherStatistics($objectid)
376 // Gather statistics on parameters
378 $r = $dbms->query("SELECT paramid FROM parameter WHERE objectid=CAST('"
379 . $objectid . "' AS BIGINT)");
380 $nr_parameters = $dbms->num_rows($r);
382 $removed_parameters = 0;
383 for ($p = 0; $p < $nr_parameters; $p++)
385 $param = pg_fetch_object($r, $p);
386 $qry ="select change_nature from history where paramid= CAST('";
387 $qry .= $param->paramid . "' AS BIGINT) order by modified desc";
388 $rhist = $dbms->query($qry);
389 if ($dbms->num_rows($rhist) == 0)
391 echo "ERROR: No history for parameter id " . $param->paramid . "\n";
395 $hist = $dbms->fetch_object($rhist, 0);
396 if ($hist->change_nature == "REMOVED")
398 $removed_parameters++;
403 UpdateStatistic($objectid, 'parameters', $nr_parameters);
404 UpdateStatistic($objectid, 'removed_parameters', $removed_parameters);
406 // Gather statistics on notifications
408 $r = $dbms->query("SELECT count(notificationid) FROM notification WHERE
409 objectid = CAST('" . $objectid . "' AS BIGINT)");
410 $cnt = $dbms->fetch_object($r, 0);
411 UpdateStatistic($objectid, 'notifications', $cnt->count);
413 $r = $dbms->query("SELECT count(notificationid) FROM notification WHERE
414 objectid = CAST('" . $objectid . "' AS BIGINT) AND statuscode ='cls'");
415 $cnt = $dbms->fetch_object($r, 0);
416 UpdateStatistic($objectid, 'closed_notifications', $cnt->count);
418 // Gather statistics on log entries
420 $r = $dbms->query("SELECT count(logid) FROM log WHERE
421 objectid = CAST('" . $objectid . "' AS BIGINT)");
422 $cnt = $dbms->fetch_object($r, 0);
423 UpdateStatistic($objectid, 'logs', $cnt->count);
426 function match_log_patterns($logstart)
430 $notifications = array();
432 $log_limit = $logstart + BATCHSIZE;
433 $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log
434 where logid > $logstart and logid <= $log_limit
436 for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++)
438 $logentry = $dbms->fetch_object($noqueue_res, $row);
439 //echo "\n----------\n" . $logentry->rawdata . "\n----------\n";
440 $service = $logentry->servicecode;
441 $pattern_res = $dbms->query("select * from service_pattern where service='$service'
442 OR service='ANY' order by rank");
444 $match_found = false;
445 for ($patnr = 0; !$match_found && $patnr < $dbms->num_rows($pattern_res); $patnr++)
447 $srv_pat = $dbms->fetch_object($pattern_res, $patnr);
448 //echo " Checking with pattern " . $srv_pat->pattern . "\n";
449 if (ereg($srv_pat->pattern, $logentry->rawdata, $matches))
451 // Scan the argument for '$n' expressions and expand
453 $srv_pat->argument = expand_arguments($srv_pat->argument, $matches);
454 //echo " " . $srv_pat->pattern . " matches.\n";
455 //echo " Matched string: " . $matches[0] . "\n";
456 //echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n";
459 switch ($srv_pat->action)
465 $notif = $srv_pat->argument;
466 if (!isset($notifications[$logentry->objectid][$notif]))
468 //echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
469 $remark = "Notification generated from Gnucomo pattern match.";
470 $notifications[$logentry->objectid][$notif] =
471 $dbms->new_notification($logentry->objectid, $notif, $remark);
473 if (isset($notifications[$logentry->objectid][$notif]))
475 //echo "Notification $notif for object " . $logentry->objectid . " already created.\n";
476 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
477 $insertion .= $notifications[$logentry->objectid][$notif] . "', '";
478 $insertion .= $logentry->logid . "')";
479 $dbms->query($insertion);
484 //echo "Recording abuse for address ", $srv_pat->argument, "\n Log entry:\n ";
485 //echo $logentry->rawdata, "\n Pattern:\n ", $srv_pat->pattern, "\n\n";
487 $nr_abuses = record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, 1);
490 echo "ERROR in recording abuse for address ", $srv_pat->argument, "\n Log entry:\n ";
491 echo $logentry->rawdata, "\n Pattern:\n ", $srv_pat->pattern, "\n\n";
493 if ($nr_abuses >= 32)
495 $source_ip = gethostbyname($srv_pat->argument);
496 $notif = 'abuses exceeded';
497 if (!isset($notifications[$logentry->objectid][$notif][$source_ip]))
499 //echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
500 $remark = "Abuses from IP address $source_ip exceeded the limit.";
501 $notifid = $dbms->new_notification($logentry->objectid, $notif, $remark);
502 $notifications[$logentry->objectid][$notif][$source_ip] = $notifid;
504 // Add log entries from previously detected abuses
506 //echo " Add log entries from previously detected abuses\n";
507 $abuses = $dbms->query("SELECT logid FROM log_abuse WHERE objectid = '" .
508 $logentry->objectid . "' AND source = '$source_ip'");
509 for ($abusenr = 0; $abusenr < $dbms->num_rows($abuses); $abusenr++)
511 $log_abuse = $dbms->fetch_object($abuses, $abusenr);
512 if ($log_abuse->logid != $logentry->logid)
514 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
515 $insertion .= $notifid . "', '";
516 $insertion .= $log_abuse->logid . "')";
517 $dbms->query($insertion);
521 if (isset($notifications[$logentry->objectid][$notif][$source_ip]))
523 //echo "Notification $notif for object " . $logentry->objectid . " already created.\n";
524 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
525 $insertion .= $notifications[$logentry->objectid][$notif][$source_ip] . "', '";
526 $insertion .= $logentry->logid . "')";
527 $dbms->query($insertion);
532 record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, -4);
535 echo "Error: unrecognized action in service pattern: " . $srv_pat->action . "\n";
541 // echo " " . $srv_pat->pattern . " does not match.\n";
549 * Some IP address abused us. Record the event.
550 * Return the number of abuse points recorded so far for the address
553 function record_abuse($logid, $objectid, $sourceip, $points)
557 $abuse_points = $points;
559 $ipaddress = gethostbyname($sourceip);
560 //echo " IP address for $sourceip is $ipaddress.\n";
561 $sourceip = $ipaddress;
563 $abres = $dbms->query("SELECT * FROM object_abuse WHERE objectid='$objectid' AND source='$sourceip'");
567 $abuse_points = -1; // return an error
571 if (pg_numrows($abres) == 0 && $points > 0)
573 //echo "$sourceip is new.\n";
574 $dbms->query("INSERT INTO object_abuse VALUES ('$objectid', '$sourceip', '$points', '', NOW())");
575 $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')");
577 else if (pg_numrows($abres) != 0)
579 $abuse = $dbms->fetch_object($abres, 0);
580 if ($abuse->status == '' || $abuse->status == 'dropped')
582 $abuse_points = $abuse->nr_abuses + $points;
583 if ($abuse_points < 0)
587 //echo $sourceip . " will get " . $abuse_points . " abuse points, ";
588 //echo "Status was " . $abuse->status . "\n";
589 $dbms->query("UPDATE object_abuse SET nr_abuses='$abuse_points'" .
590 ", last_change=NOW() WHERE objectid='$objectid' AND source='$sourceip'");
594 $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')");
596 if ($abuse_points >= 32)
598 //echo " BLOCK IP adrress $sourceip on the firewall.\n";
599 $dbms->query("UPDATE object_abuse SET status='dropped'" .
600 " WHERE objectid='$objectid' AND source='$sourceip'");
606 return $abuse_points;
611 * Service_check - Check the log entries if there are any unknown
615 function service_check()
619 $unknown_notification = array();
620 $unused_notification = array();
623 // How far did we get last time ?
625 $lastlogres = $dbms->query("SELECT setting_value FROM db_value
626 WHERE setting = 'log_servicecheck'");
628 if ($dbms->num_rows($lastlogres) == 1)
630 $last_log = $dbms->Field($lastlogres, 0, 'setting_value');
634 $dbms->query("INSERT INTO db_value (setting, setting_value)
635 VALUES ('log_servicecheck', '0')");
638 //echo "Running service check from log id $last_log.\n";
639 // Query the log-table
641 $log_limit = $last_log + BATCHSIZE;
642 $qry = "SELECT logid, objectid, servicecode FROM log
643 WHERE logid > CAST(".$last_log." AS BIGINT) AND logid <= $log_limit
645 $log_res = $dbms->query($qry);
646 //$log_res = $dbms->query("SELECT logid, objectid, servicecode,rawdata FROM log");
648 for ($log_row = 0; $log_row < $dbms->num_rows($log_res); $log_row++)
650 $log_entry = $dbms->fetch_object($log_res, $log_row);
651 $last_log = $log_entry->logid;
653 // Check if the service is used on the object.
655 $qry = "SELECT * FROM object_service WHERE objectid='";
656 $qry .= $log_entry->objectid . "' AND servicecode='";
657 $qry .= $log_entry->servicecode . "'";
659 $os_res = $dbms->query($qry);
660 if ($dbms->num_rows($os_res) == 0)
662 // Service is not found for the object, check if the service
665 $qry = "SELECT * FROM service WHERE servicecode='";
666 $qry .= $log_entry->servicecode . "'";
668 if ($dbms->num_rows($dbms->query($qry)) == 0)
670 if (!isset($unknown_notification[$log_entry->objectid]))
672 $remark = "One or more log entries from a service that is not in the database";
673 $unknown_notification[$log_entry->objectid] =
674 $dbms->new_notification($log_entry->objectid, 'service unknown', $remark);
676 if (isset($unknown_notification[$log_entry->objectid]))
678 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
679 $insertion .= $unknown_notification[$log_entry->objectid] . "', '";
680 $insertion .= $log_entry->logid . "')";
681 $dbms->query($insertion);
686 if (!isset($unused_notification[$log_entry->objectid]))
688 $remark = "One or more log entries from a service not running on this object";
689 $unused_notification[$log_entry->objectid] =
690 $dbms->new_notification($log_entry->objectid, 'service not used', $remark);
692 if (isset($unused_notification[$log_entry->objectid]))
694 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
695 $insertion .= $unused_notification[$log_entry->objectid] . "', '";
696 $insertion .= $log_entry->logid . "')";
697 $dbms->query($insertion);
703 $qry = "UPDATE db_value SET setting_value = '"
704 . $last_log . "' WHERE setting = 'log_servicecheck'";
709 * find open notifications and send an email to the object's users.
712 function mail_notifications ()
716 $notifres = $dbms->query("SELECT notificationid, objectid, type_of_issueid FROM notification
717 WHERE statuscode != 'cls'");
719 for ($notifrow = 0; $notifrow < pg_numrows($notifres); $notifrow++)
721 $notification = pg_fetch_object($notifres, $notifrow);
723 $issue = pg_fetch_object($dbms->query("SELECT description FROM type_of_issue
724 WHERE type_of_issueid='" . $notification->type_of_issueid . "'"), 0);
725 echo "Mailing Notification for object id " . $notification->objectid . "\n";
726 $object = pg_fetch_object($dbms->query("SELECT objectname FROM object
727 WHERE objectid='" . $notification->objectid ."'"), 0);
729 $users = $dbms->query("SELECT username FROM object_user WHERE objectid='" . $notification->objectid . "'");
731 for ($userrow = 0; $userrow < pg_numrows($users); $userrow++)
733 $objusr = pg_fetch_object($users, $userrow);
734 $usr = pg_fetch_object($dbms->query("SELECT email FROM usr
735 WHERE username='" . $objusr->username . "'"), 0);
737 $message = "Notification " . $notification->notificationid . ": " . $issue->description;
738 $message .= " for object " . $object->objectname . "\n";
740 mail($usr->email, "GnuCoMo Notification", $message);
746 * The 'command' may contain positional parameters such as '$1' and '$3',
747 * just like the shell. These parameters are replaced by content from
751 function expand_arguments($command, $args)
753 while (ereg('\$([0-9]+)', $command, $match))
756 if ($index >= count($args))
758 echo "Error: Argument $index not found for $command.\n";
759 $command = ereg_replace('\$' . $index, "", $command);
763 $command = ereg_replace('\$' . $index, $args[$index], $command);
769 // ereg function are removed from PHP since version 7.0
770 // These wrappers use the alternative preg functions.
772 function ereg($pattern, $string, &$matches)
774 $pattern = "#" . $pattern . "#";
775 return preg_match($pattern, $string, $matches);
778 function ereg_replace($pattern, $replacement, $string)
780 $pattern = "#" . $pattern . "#";
781 return preg_replace($pattern, $replacement, $string);