3 /**********************************************************************************
4 ** (c) Copyright 2002, Brenno J.S.A.A.F. de Winter, De Winter Information Solutions
5 ** This is free software; you can redistribute it and/or modify it under the
6 ** terms of the GNU General Public License, see the file COPYING.
7 ***********************************************************************************/
12 AUTHOR : Brenno J.S.A.A.F. de Winter
13 De Winter Information Solutions
14 COPYRIGHT : 2002 - De Winter Information Solutions,
15 Brenno J.S.A.A.F. de Winter
18 First : November 8th 2002
19 Gnucomo-0.0.3: December 6th 2002
20 Gnucomo-0.0.8: September 4th 2003
22 $Log: gcm_daemon.php,v $
23 Revision 1.26 2011-03-24 09:46:36 arjen
24 Resolve a hostname before adding to the abuses.
26 Revision 1.25 2007/12/12 09:06:21 arjen
27 Added a new script gcm_maintenance.php to cleanup the database
28 and check referential integrity. Purging old log entries is
29 removed from the gcm_daemon script.
31 Revision 1.24 2007/11/17 09:34:07 arjen
32 Cleaned up some leftovers
34 Revision 1.23 2007/11/03 10:31:12 arjen
35 Added the class definition for a filesystem parameter.
36 New issue type: 'property out of range'.
38 Revision 1.22 2007/10/23 11:23:52 arjen
39 Record the date when adding the number of abuses for an IP address
42 Revision 1.21 2007/01/11 13:47:41 arjen
43 Log_adv and derived tables removed.
44 Create notifications from log entries with pattern matching.
46 Revision 1.20 2005/06/04 07:15:16 arjen
47 Added pattern check on log entries with the service_pattern table.
49 Revision 1.19 2004/01/10 20:04:12 arjen
50 Send email about open notifications to an object's users.
52 Revision 1.18 2003/12/03 08:07:21 arjen
53 Changed the type of log_adv_daemon_email.delay and log_adv_daemon_email.xdelay
54 from time to interval. These delays can be more than 24 hours.
56 Revision 1.17 2003/10/29 09:58:29 arjen
57 Create separate notifications for different objects in service_check().
59 Revision 1.16 2003/09/03 12:48:48 arjen
60 Check the log table against the servies running on an object and
61 create notifications if a service is not supposed to be available
62 or is not known at all.
64 Revision 1.15 2003/09/02 12:48:09 arjen
65 BUGFIX: Secondary indices on log_notification were unique.
66 Additional information in the 'usr' table: 'display_name' and 'email'.
67 Added new issues and services.
69 Revision 1.14 2003/09/01 06:51:07 arjen
70 Accept command argument '-c config' to use an alternate
71 gnucomo configuration.
73 Revision 1.13 2003/08/14 10:22:42 arjen
76 Revision 1.12 2003/08/05 07:46:37 arjen
77 BUGFIX: Print an error message if a parameter does not have
80 Revision 1.11 2003/07/09 07:25:02 arjen
81 Gcm_daemon gathers statistics on parameters, notifications, etc. for all objects.
83 Revision 1.10 2003/03/29 08:33:58 arjen
84 In phpclasses/db.class.php: Added the database connection string as
85 an argument to the function copy_db_class.
86 Fixed the PHP member function db::db_connect(). The Postgres connection
87 string is now passed as an argument to that function.
89 Revision 1.9 2003/02/21 08:37:59 arjen
90 Added new table to the database: log_adv_daemon_email.
95 // $Id: gcm_daemon.php,v 1.26 2011-03-24 09:46:36 arjen Exp $
97 ini_set('include_path', '.:./classes:../phpclasses');
98 ini_set('html_errors', 'false');
100 define("BATCHSIZE", 50000);
102 //Tell the log that we're up.
103 define_syslog_variables();
105 require_once "gnucomo_config.php";
106 require_once "db.class.php";
108 // Set the standard variables //
110 $project_name = "gnucomo"; // name of the entire project
111 $app_name = "gcm_daemon"; // name of the application running
112 $developrelease = "FALSE"; // Indicates if special debug settings are needed
113 $db_version = 53; // The db_version indicates what the level of
114 // the database should be. If the database is
115 // old an update will be generated.
116 $gcmd_version = 5; // This value indicates the active version of
117 // the gcm_daemon, which is saved in the database.
118 // Log records that were not recognized before
119 // will now be recognized. The version doesn't
120 // mean anything in the overall gnucomo project.
122 //Avoid time-limit issues
125 // Scan the command arguments
127 for ($argi = 1; $argi < $argc; $argi++)
129 switch ($argv[$argi])
133 $project_name = $argv[$argi];
137 echo "Usage: gcm_daemon [-c configname]\n";
143 // Read the database settings //
144 $class_settings = new gnucomo_config();
145 if (!$class_settings->read($project_name))
147 echo "Can not read Gnucomo configuration file for $project_name.\n";
151 openlog("gnucomo", LOG_PID, LOG_DAEMON);
152 syslog(LOG_INFO, "gcm_daemon started");
154 //Open an connection to the database
155 $dbms_type = $class_settings->find_parameter("database", "type");
156 $dbms_host = $class_settings->find_parameter("database", "host");
157 $dbms_name = $class_settings->find_parameter("database", "name");
158 $dbms_user = $class_settings->find_parameter("gcm_daemon", "user");
159 $dbms_password = $class_settings->find_parameter("gcm_daemon", "password");
161 db_select($dbms_type);
163 $dbms->db_host = $dbms_host;
164 $dbms->db_name = $dbms_name;
165 $dbms->db_user = $dbms_user;
166 $dbms->db_password = $dbms_password;
167 $dbms->db_connect($class_settings->database());
169 if ($dbms->have_db_connection() == "FALSE")
171 exit ("Database connection failed.");
175 // The database connection has been made.
176 $dbms_working = copy_db_class($dbms, $class_settings->database());
179 // Verify if the database is up-to-date by checking the versionnumber
181 $local_sql = "SELECT setting_value FROM db_value WHERE setting = 'db_version' ";
182 $dbms->query($local_sql);
184 if ($dbms->fetch_row() == "TRUE")
186 $active_version = $dbms->db_result_row[0];
188 // Update the database to the most recent version.
190 if ($active_version < $db_version)
192 include ("gnucomo_db_version.php");
197 syslog (LOG_INFO, "Couldn't initialize database version. Is this a gnucomo database?");
198 die ("Couldn't initialize database version.\n");
201 // The gcm_daemon version is maintained in the database to enable
202 // automatic update actions.
204 $local_sql = "SELECT setting_value FROM db_value
205 WHERE setting = 'gcm_daemon_version'";
206 $dbms->query($local_sql);
208 if ($dbms->fetch_row() == "TRUE")
210 if ($dbms->db_result_row[0] < $gcmd_version)
213 //Update de gcm_daemon version in the database
214 $local_sql = "UPDATE db_value SET setting_value = '".$gcmd_version;
215 $local_sql .= "' WHERE setting = 'gcm_daemon_version'";
216 $dbms->query($local_sql);
222 // Now we loop the tasks that we have to do.
228 echo "Processing logs...\n";
231 //mail_notifications();
233 // Gather the statistics for each object
235 $obj_result = $dbms->query("SELECT objectid FROM object");
236 for ($obj = 0; $obj < $dbms->num_rows($obj_result); $obj++)
238 $object = $dbms->fetch_object($obj_result, $obj);
239 echo "Gathering statistics for object " . $object->objectid . "\n";
240 GatherStatistics($object->objectid);
243 $keep_running = false;
245 } while ($keep_running == true);
247 //Tell the log that we're ending our efforts in a nice way
249 syslog (LOG_INFO, "gcm_daemon ended nicely");
251 function process_log ()
254 /* This function will walk through the log-records that haven't been processed
255 * first a snapshot will be created of a the non-processed records.
256 * sequentially each record will dealt with. By doing that changes will be made
257 * in several log_adv_xxx tables
263 global $dbms_working;
264 global $class_settings;
268 // Find records in log that still have to be processed.
270 $local_sql = "SELECT setting_value FROM db_value WHERE setting = 'log_processing'";
271 $dbms->query($local_sql);
273 if ($dbms->fetch_row() == "TRUE")
275 $last_log = $dbms->db_result_row[0];
278 echo "Last processed logid = $last_log \n";
280 //Query the log-table
281 $log_limit = $last_log + BATCHSIZE;
282 $local_sql = "SELECT * FROM log WHERE logid > CAST(".$last_log." AS BIGINT)
283 AND logid <= $log_limit ORDER BY logid";
284 $dbms->query($local_sql);
286 //Update the log-statistics in the object-table
287 $local_statistics_db = copy_db_class($dbms, $class_settings->database());
288 $local_findobject_db = copy_db_class($dbms, $class_settings->database());
291 $local_upper_row = $dbms->num_rows() + $last_log + 1;
292 $local_sql = "SELECT COUNT(logid), objectid from log WHERE logid > CAST(". $last_log .
293 " AS BIGINT) AND logid < CAST (" . $local_upper_row . " AS BIGINT) GROUP BY objectid";
294 $local_statistics_db->query ($local_sql);
297 for ($i = 1; $i <= $local_statistics_db->num_rows(); $i++)
299 $local_object_row = $local_statistics_db->fetch_row();
301 $local_sql = "UPDATE object SET log_count = log_count + " .
302 $local_statistics_db->db_result_row[0] . " WHERE objectid = '" .
303 $local_statistics_db->db_result_row[1] . "'";
305 $local_findobject_db->query($local_sql);
310 if ($dbms->num_rows() > 0)
313 //Create a database connection for changes in the database.
314 $dbms_changes = copy_db_class($dbms, $class_settings->database());
315 if ($dbms_changes->have_db_connection() == 'TRUE')
319 $local_sql_statistics = "";
320 $local_object_os = "";
321 $local_object_os_version = "";
323 match_log_patterns($last_log);
325 // Register that the logrecords have been processed.
327 $local_sql = "UPDATE db_value SET setting_value = '"
328 . $local_upper_row ."' where setting = 'log_processing'";
329 $dbms->query($local_sql);
332 // Update the statistics for the object-table
338 syslog (LOG_INFO, "Couldn't clone database connection.");
339 die ("Couldn't reconnect to the database.\n");
346 * Update a single statistic for some object.
347 * If it does not yet exist, it will be created.
350 function UpdateStatistic($objectid, $name, $value)
354 $result = $dbms->query("SELECT objectid FROM object_statistics WHERE
355 objectid='$objectid' AND statname='$name'");
356 if ($dbms->num_rows() == 0)
358 $dbms->query("INSERT INTO object_statistics VALUES
359 ('$objectid', '$name', '$value')");
363 $dbms->query("UPDATE object_statistics SET statvalue='$value' WHERE
364 statname='$name' AND objectid='$objectid'");
369 * Gather the statistics for a single object ($objectid).
370 * We count the number of parameters, removed parameters, notifications
371 * closed notifications and log entries. The totals of these are
372 * maintained in a separate table: object_statistics.
375 function GatherStatistics($objectid)
379 // Gather statistics on parameters
381 $r = $dbms->query("SELECT paramid FROM parameter WHERE objectid=CAST('"
382 . $objectid . "' AS BIGINT)");
383 $nr_parameters = $dbms->num_rows($r);
385 $removed_parameters = 0;
386 for ($p = 0; $p < $nr_parameters; $p++)
388 $param = pg_fetch_object($r, $p);
389 $qry ="select change_nature from history where paramid= CAST('";
390 $qry .= $param->paramid . "' AS BIGINT) order by modified desc";
391 $rhist = $dbms->query($qry);
392 if ($dbms->num_rows($rhist) == 0)
394 echo "ERROR: No history for parameter id " . $param->paramid . "\n";
398 $hist = $dbms->fetch_object($rhist, 0);
399 if ($hist->change_nature == "REMOVED")
401 $removed_parameters++;
406 UpdateStatistic($objectid, 'parameters', $nr_parameters);
407 UpdateStatistic($objectid, 'removed_parameters', $removed_parameters);
409 // Gather statistics on notifications
411 $r = $dbms->query("SELECT count(notificationid) FROM notification WHERE
412 objectid = CAST('" . $objectid . "' AS BIGINT)");
413 $cnt = $dbms->fetch_object($r, 0);
414 UpdateStatistic($objectid, 'notifications', $cnt->count);
416 $r = $dbms->query("SELECT count(notificationid) FROM notification WHERE
417 objectid = CAST('" . $objectid . "' AS BIGINT) AND statuscode ='cls'");
418 $cnt = $dbms->fetch_object($r, 0);
419 UpdateStatistic($objectid, 'closed_notifications', $cnt->count);
421 // Gather statistics on log entries
423 $r = $dbms->query("SELECT count(logid) FROM log WHERE
424 objectid = CAST('" . $objectid . "' AS BIGINT)");
425 $cnt = $dbms->fetch_object($r, 0);
426 UpdateStatistic($objectid, 'logs', $cnt->count);
429 function match_log_patterns($logstart)
433 $notifications = array();
435 $log_limit = $logstart + BATCHSIZE;
436 $noqueue_res = $dbms->query("select logid, objectid, servicecode, rawdata from log
437 where logid > $logstart and logid <= $log_limit
439 for ($row = 0; $row < $dbms->num_rows($noqueue_res); $row++)
441 $logentry = $dbms->fetch_object($noqueue_res, $row);
442 //echo "\n----------\n" . $logentry->rawdata . "\n----------\n";
443 $service = $logentry->servicecode;
444 $pattern_res = $dbms->query("select * from service_pattern where service='$service'
445 OR service='ANY' order by rank");
447 $match_found = false;
448 for ($patnr = 0; !$match_found && $patnr < $dbms->num_rows($pattern_res); $patnr++)
450 $srv_pat = $dbms->fetch_object($pattern_res, $patnr);
451 //echo " Checking with pattern " . $srv_pat->pattern . "\n";
452 if (ereg($srv_pat->pattern, $logentry->rawdata, $matches))
454 // Scan the argument for '$n' expressions and expand
456 $srv_pat->argument = expand_arguments($srv_pat->argument, $matches);
457 //echo " " . $srv_pat->pattern . " matches.\n";
458 //echo " Matched string: " . $matches[0] . "\n";
459 //echo " Action = " . $srv_pat->action . "(" . $srv_pat->argument . ")\n\n";
462 switch ($srv_pat->action)
468 $notif = $srv_pat->argument;
469 if (!isset($notifications[$logentry->objectid][$notif]))
471 //echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
472 $remark = "Notification generated from Gnucomo pattern match.";
473 $notifications[$logentry->objectid][$notif] =
474 $dbms->new_notification($logentry->objectid, $notif, $remark);
476 if (isset($notifications[$logentry->objectid][$notif]))
478 //echo "Notification $notif for object " . $logentry->objectid . " already created.\n";
479 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
480 $insertion .= $notifications[$logentry->objectid][$notif] . "', '";
481 $insertion .= $logentry->logid . "')";
482 $dbms->query($insertion);
487 //echo "Recording abuse for address ", $srv_pat->argument, "\n Log entry:\n ";
488 //echo $logentry->rawdata, "\n Pattern:\n ", $srv_pat->pattern, "\n\n";
490 if (record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, 1) >= 32)
492 $source_ip = gethostbyname($srv_pat->argument);
493 $notif = 'abuses exceeded';
494 if (!isset($notifications[$logentry->objectid][$notif][$source_ip]))
496 echo "Creating notification $notif for object " . $logentry->objectid . ".\n";
497 $remark = "Abuses from IP address $source_ip exceeded the limit.";
498 $notifid = $dbms->new_notification($logentry->objectid, $notif, $remark);
499 $notifications[$logentry->objectid][$notif][$source_ip] = $notifid;
501 // Add log entries from previously detected abuses
503 echo " Add log entries from previously detected abuses\n";
504 $abuses = $dbms->query("SELECT logid FROM log_abuse WHERE objectid = '" .
505 $logentry->objectid . "' AND source = '$source_ip'");
506 for ($abusenr = 0; $abusenr < $dbms->num_rows($abuses); $abusenr++)
508 $log_abuse = $dbms->fetch_object($abuses, $abusenr);
509 if ($log_abuse->logid != $logentry->logid)
511 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
512 $insertion .= $notifid . "', '";
513 $insertion .= $log_abuse->logid . "')";
514 $dbms->query($insertion);
518 if (isset($notifications[$logentry->objectid][$notif][$source_ip]))
520 //echo "Notification $notif for object " . $logentry->objectid . " already created.\n";
521 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
522 $insertion .= $notifications[$logentry->objectid][$notif][$source_ip] . "', '";
523 $insertion .= $logentry->logid . "')";
524 $dbms->query($insertion);
529 record_abuse($logentry->logid, $logentry->objectid, $srv_pat->argument, -4);
532 echo "Error: unrecognized action in service pattern: " . $srv_pat->action . "\n";
538 // echo " " . $srv_pat->pattern . " does not match.\n";
546 * Some IP address abused us. Record the event.
547 * Return the number of abuse points recorded so far for the address
550 function record_abuse($logid, $objectid, $sourceip, $points)
554 $abuse_points = $points;
556 $ipaddress = gethostbyname($sourceip);
557 //echo " IP address for $sourceip is $ipaddress.\n";
558 $sourceip = $ipaddress;
560 $abres = $dbms->query("SELECT * FROM object_abuse WHERE objectid='$objectid' AND source='$sourceip'");
562 if (pg_numrows($abres) == 0 && $points > 0)
564 //echo "$sourceip is new.\n";
565 $dbms->query("INSERT INTO object_abuse VALUES ('$objectid', '$sourceip', '$points', '', NOW())");
566 $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')");
568 else if (pg_numrows($abres) != 0)
570 $abuse = $dbms->fetch_object($abres, 0);
571 if ($abuse->status == '' || $abuse->status == 'dropped')
573 $abuse_points = $abuse->nr_abuses + $points;
574 if ($abuse_points < 0)
578 //echo $sourceip . " will get " . $abuse_points . " abuse points, ";
579 //echo "Status was " . $abuse->status . "\n";
580 $dbms->query("UPDATE object_abuse SET nr_abuses='$abuse_points'" .
581 ", last_change=NOW() WHERE objectid='$objectid' AND source='$sourceip'");
585 $dbms->query("INSERT INTO log_abuse VALUES ('$logid', '$objectid', '$sourceip')");
587 if ($abuse_points >= 32)
589 //echo " BLOCK IP adrress $sourceip on the firewall.\n";
590 $dbms->query("UPDATE object_abuse SET status='dropped'" .
591 " WHERE objectid='$objectid' AND source='$sourceip'");
596 return $abuse_points;
601 * Service_check - Check the log entries if there are any unknown
605 function service_check()
609 $unknown_notification = array();
610 $unused_notification = array();
613 // How far did we get last time ?
615 $lastlogres = $dbms->query("SELECT setting_value FROM db_value
616 WHERE setting = 'log_servicecheck'");
618 if ($dbms->num_rows($lastlogres) == 1)
620 $last_log = $dbms->Field($lastlogres, 0, 'setting_value');
624 $dbms->query("INSERT INTO db_value (setting, setting_value)
625 VALUES ('log_servicecheck', '0')");
628 echo "Running service check from log id $last_log.\n";
629 // Query the log-table
631 $log_limit = $last_log + BATCHSIZE;
632 $qry = "SELECT logid, objectid, servicecode FROM log
633 WHERE logid > CAST(".$last_log." AS BIGINT) AND logid <= $log_limit
635 $log_res = $dbms->query($qry);
636 //$log_res = $dbms->query("SELECT logid, objectid, servicecode,rawdata FROM log");
638 for ($log_row = 0; $log_row < $dbms->num_rows($log_res); $log_row++)
640 $log_entry = $dbms->fetch_object($log_res, $log_row);
641 $last_log = $log_entry->logid;
643 // Check if the service is used on the object.
645 $qry = "SELECT * FROM object_service WHERE objectid='";
646 $qry .= $log_entry->objectid . "' AND servicecode='";
647 $qry .= $log_entry->servicecode . "'";
649 $os_res = $dbms->query($qry);
650 if ($dbms->num_rows($os_res) == 0)
652 // Service is not found for the object, check if the service
655 $qry = "SELECT * FROM service WHERE servicecode='";
656 $qry .= $log_entry->servicecode . "'";
658 if ($dbms->num_rows($dbms->query($qry)) == 0)
660 if (!isset($unknown_notification[$log_entry->objectid]))
662 $remark = "One or more log entries from a service that is not in the database";
663 $unknown_notification[$log_entry->objectid] =
664 $dbms->new_notification($log_entry->objectid, 'service unknown', $remark);
666 if (isset($unknown_notification[$log_entry->objectid]))
668 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
669 $insertion .= $unknown_notification[$log_entry->objectid] . "', '";
670 $insertion .= $log_entry->logid . "')";
671 $dbms->query($insertion);
676 if (!isset($unused_notification[$log_entry->objectid]))
678 $remark = "One or more log entries from a service not running on this object";
679 $unused_notification[$log_entry->objectid] =
680 $dbms->new_notification($log_entry->objectid, 'service not used', $remark);
682 if (isset($unused_notification[$log_entry->objectid]))
684 $insertion = "INSERT INTO log_notification (notificationid, logid) VALUES ('";
685 $insertion .= $unused_notification[$log_entry->objectid] . "', '";
686 $insertion .= $log_entry->logid . "')";
687 $dbms->query($insertion);
693 $qry = "UPDATE db_value SET setting_value = '"
694 . $last_log . "' WHERE setting = 'log_servicecheck'";
699 * find open notifications and send an email to the object's users.
702 function mail_notifications ()
706 $notifres = $dbms->query("SELECT notificationid, objectid, type_of_issueid FROM notification
707 WHERE statuscode != 'cls'");
709 for ($notifrow = 0; $notifrow < pg_numrows($notifres); $notifrow++)
711 $notification = pg_fetch_object($notifres, $notifrow);
713 $issue = pg_fetch_object($dbms->query("SELECT description FROM type_of_issue
714 WHERE type_of_issueid='" . $notification->type_of_issueid . "'"), 0);
715 echo "Mailing Notification for object id " . $notification->objectid . "\n";
716 $object = pg_fetch_object($dbms->query("SELECT objectname FROM object
717 WHERE objectid='" . $notification->objectid ."'"), 0);
719 $users = $dbms->query("SELECT username FROM object_user WHERE objectid='" . $notification->objectid . "'");
721 for ($userrow = 0; $userrow < pg_numrows($users); $userrow++)
723 $objusr = pg_fetch_object($users, $userrow);
724 $usr = pg_fetch_object($dbms->query("SELECT email FROM usr
725 WHERE username='" . $objusr->username . "'"), 0);
727 $message = "Notification " . $notification->notificationid . ": " . $issue->description;
728 $message .= " for object " . $object->objectname . "\n";
730 mail($usr->email, "GnuCoMo Notification", $message);
736 * The 'command' may contain positional parameters such as '$1' and '$3',
737 * just like the shell. These parameters are replaced by content from
741 function expand_arguments($command, $args)
743 while (ereg('\$([0-9]+)', $command, $match))
746 if ($index >= count($args))
748 echo "Error: Argument $index not found for $command.\n";
749 $command = ereg_replace('\$' . $index, "", $command);
753 $command = ereg_replace('\$' . $index, $args[$index], $command);