ZaneYork
/
zoneminder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'fix_zmaudit' into storageareas

This commit is contained in:
Isaac Connor 2016-06-03 11:57:38 -04:00
parent ef9c0ca33a 575180ba5e
commit ad185809f8
1 changed files with 484 additions and 536 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

328
scripts/zmaudit.pl.in
View File

@ -122,20 +122,18 @@ my $max_image_age = 6/24; # 6 hours
my $max_swap_age = 24/24; # 24 hours
my $image_path = IMAGE_PATH;
my $loop = 1;
my $cleaned = 0;
MAIN: while( $loop ) {
while ( ! ( $dbh and $dbh->ping() ) ) {
$dbh = zmDbConnect();
if ( $continuous ) {
Error("Unable to connect to database");
# if we are running continuously, then just skip to the next
# interval, otherwise we are a one off run, so wait a second and
# retry until someone kills us.
# if we are running continuously, then just skip to the next
# interval, otherwise we are a one off run, so wait a second and
# retry until someone kills us.
sleep( $Config{ZM_AUDIT_CHECK_INTERVAL} );
} else {
Fatal("Unable to connect to database");
@ -169,16 +167,14 @@ MAIN: while( $loop ) {
$cleaned = 0;
my $res = $monitorSelectSth->execute()
or Fatal( "Can't execute: ".$monitorSelectSth->errstr() );
while( my $monitor = $monitorSelectSth->fetchrow_hashref() )
{
while( my $monitor = $monitorSelectSth->fetchrow_hashref() ) {
$Monitors{$$monitor{Id}} = $monitor;
Debug( "Found database monitor '$monitor->{Id}'" );
my $db_events = $db_monitors->{$monitor->{Id}} = {};
my $res = $eventSelectSth->execute( $monitor->{Id} )
or Fatal( "Can't execute: ".$eventSelectSth->errstr() );
while ( my $event = $eventSelectSth->fetchrow_hashref() )
{
while ( my $event = $eventSelectSth->fetchrow_hashref() ) {
$db_events->{$event->{Id}} = $event->{Age};
}
Debug( "Got ".int(keys(%$db_events))." events\n" );
@ -194,18 +190,16 @@ MAIN: while( $loop ) {
chdir( $Storage->Path() );
# Please note that this glob will take all files beginning with a digit.
foreach my $monitor ( glob("[0-9]*") )
{
foreach my $monitor ( glob("[0-9]*") ) {
next if $monitor =~ /\D/;
Debug( "Found filesystem monitor '$monitor'" );
my $fs_events = $fs_monitors->{$monitor} = {};
( my $monitor_dir ) = ( $monitor =~ /^(.*)$/ ); # De-taint
# De-taint
( my $monitor_dir ) = ( $monitor =~ /^(.*)$/ );
if ( $Config{ZM_USE_DEEP_STORAGE} )
{
foreach my $day_dir ( glob("$monitor_dir/*/*/*") )
{
if ( $Config{ZM_USE_DEEP_STORAGE} ) {
foreach my $day_dir ( glob("$monitor_dir/*/*/*") ) {
Debug( "Checking $day_dir" );
( $day_dir ) = ( $day_dir =~ /^(.*)$/ ); # De-taint
chdir( $day_dir );
@ -214,8 +208,7 @@ MAIN: while( $loop ) {
my @event_links = sort { $b <=> $a } grep { -l $_ } readdir( DIR );
closedir( DIR );
my $count = 0;
foreach my $event_link ( @event_links )
{
foreach my $event_link ( @event_links ) {
if ( $event_link =~ /[^\d\.]/ ) {
Warning("Non-event link found $event_link in $day_dir, skipping");
next;
@ -223,49 +216,36 @@ MAIN: while( $loop ) {
Debug( "Checking link $event_link" );
( my $event = $event_link ) =~ s/^.*\.//;
my $event_path = readlink( $event_link );
if ( $count++ > MAX_AGED_DIRS )
{
if ( $count++ > MAX_AGED_DIRS ) {
$fs_events->{$event} = -1;
}
else
{
if ( !-e $event_path )
{
} else {
if ( !-e $event_path ) {
aud_print( "Event link $day_dir/$event_link does not point to valid target" );
if ( confirm() )
{
if ( confirm() ) {
( $event_link ) = ( $event_link =~ /^(.*)$/ ); # De-taint
unlink( $event_link );
$cleaned = 1;
}
}
else
{
} else {
$fs_events->{$event} = (time() - ($^T - ((-M $event_path) * 24*60*60)));
} # event path exists
}
}
}
} # end foreach event_link
chdir( $Storage->Path() );
}
}
else
{
} # end foreach day dir
} else {
chdir( $monitor_dir );
opendir( DIR, "." ) or Fatal( "Can't open directory '$monitor_dir': $!" );
my @temp_events = sort { $b <=> $a } grep { -d $_ && $_ =~ /^\d+$/ } readdir( DIR );
closedir( DIR );
my $count = 0;
foreach my $event ( @temp_events )
{
if ( $count++ > MAX_AGED_DIRS )
{
foreach my $event ( @temp_events ) {
if ( $count++ > MAX_AGED_DIRS ) {
$fs_events->{$event} = -1;
}
else
{
} else {
$fs_events->{$event} = (time() - ($^T - ((-M $event) * 24*60*60)));
}
}
} # end foreach event
chdir( $Storage->Path() );
} # if USE_DEEP_STORAGE
Debug( "Got ".int(keys(%$fs_events))." events\n" );
@ -273,52 +253,42 @@ MAIN: while( $loop ) {
redo MAIN if ( $cleaned );
$cleaned = 0;
while ( my ( $fs_monitor, $fs_events ) = each(%$fs_monitors) )
{
if ( my $db_events = $db_monitors->{$fs_monitor} )
{
if ( $fs_events )
{
while ( my ( $fs_monitor, $fs_events ) = each(%$fs_monitors) ) {
if ( my $db_events = $db_monitors->{$fs_monitor} ) {
next if ! $fs_events;
foreach my $fs_event ( sort { $a <=> $b } keys %$fs_events ) {
my $age = $fs_events->{$fs_event};
if ( !defined($db_events->{$fs_event}) && ($age < 0 || ($age > $Config{ZM_AUDIT_MIN_AGE})) )
{
if ( !defined($db_events->{$fs_event}) && ($age < 0 || ($age > $Config{ZM_AUDIT_MIN_AGE})) ) {
aud_print( "Filesystem event '$fs_monitor/$fs_event' does not exist in database" );
if ( confirm() )
{
if ( confirm() ) {
my $Event = new ZoneMinder::Event( $fs_event );
# Must set these because the db event does not exist
# Must set these because the db event does not exist
$Event->MonitorId( $fs_monitor );
$Event->StorageId( $Storage->Id() );
$Event->delete_files();
$cleaned = 1;
delete $fs_events->{$fs_event};
}
}
}
}
}
else
{
} # end if confirm
} # end if ! in db events and old enough
} # end foreach fs event
} else {
aud_print( "Filesystem monitor '$fs_monitor' in $$Storage{Path} does not exist in database" );
if ( confirm() )
{
if ( confirm() ) {
my $command = "rm -rf $fs_monitor";
executeShellCommand( $command );
$cleaned = 1;
}
}
}
} # end foreach monitor/filesystem events
my $monitor_links;
foreach my $link ( glob("*") )
{
foreach my $link ( glob("*") ) {
next if ( !-l $link );
next if ( -e $link );
aud_print( "Filesystem monitor link '$link' does not point to valid monitor directory" );
if ( confirm() )
{
if ( confirm() ) {
( $link ) = ( $link =~ /^(.*)$/ ); # De-taint
my $command = "rm $link";
executeShellCommand( $command );
@ -341,19 +311,17 @@ MAIN: while( $loop ) {
my $deleteStatsSql = "delete low_priority from Stats where EventId = ?";
my $deleteStatsSth = $dbh->prepare_cached( $deleteStatsSql )
or Fatal( "Can't prepare '$deleteStatsSql': ".$dbh->errstr() );
while ( my ( $db_monitor, $db_events ) = each(%$db_monitors) )
{
if ( my $fs_events = $fs_monitors->{$db_monitor} )
{
if ( $db_events )
{
while ( my ( $db_event, $age ) = each(%$db_events ) )
{
while ( my ( $db_monitor, $db_events ) = each(%$db_monitors) ) {
if ( my $fs_events = $fs_monitors->{$db_monitor} ) {
next if ! $db_events;
while ( my ( $db_event, $age ) = each(%$db_events ) ) {
if ( !defined($fs_events->{$db_event}) ) {
my $Event = new ZoneMinder::Event( $db_event );
if ( $Event->check_for_in_filesystem() ) {
Debug("Database events apparent exists at " . $Event->Path() );
} else {
my $Event = new ZoneMinder::Event( $db_event );
if ( $Event->check_for_in_filesystem() ) {
Debug("Database events apparent exists at " . $Event->Path() );
} else {
if ( $age > $Config{ZM_AUDIT_MIN_AGE} ) {
aud_print( "Database event '$db_monitor/$db_event' does not exist at " . $Event->Path()." in filesystem" );
if ( confirm() ) {
@ -370,29 +338,26 @@ Debug("Database events apparent exists at " . $Event->Path() );
aud_print( "Database event '".$Event->getPath()."/$db_monitor/$db_event' does not exist in filesystem but too young to delete.\n" );
}
}
}
}
}
}
else
{
} # end if exists in filesystem
} # end if ! in fs_events
} # foreach db_event
} else {
my $Monitor = new ZoneMinder::Monitor( $db_monitor );
my $Storage = $Monitor->Storage();
aud_print( "Database monitor '$db_monitor' does not exist in filesystem, should have been at ".$Storage->Path()."\n" );
#if ( confirm() )
#{
# We don't actually do this in case it's new
#my $res = $deleteMonitorSth->execute( $db_monitor )
# or Fatal( "Can't execute: ".$deleteMonitorSth->errstr() );
#$cleaned = 1;
#}
#if ( confirm() )
#{
# We don't actually do this in case it's new
#my $res = $deleteMonitorSth->execute( $db_monitor )
# or Fatal( "Can't execute: ".$deleteMonitorSth->errstr() );
#$cleaned = 1;
#}
}
} # end foreach db monitors
redo MAIN if ( $cleaned );
# Remove orphaned events (with no monitor)
# Remove orphaned events (with no monitor)
$cleaned = 0;
my $selectOrphanedEventsSql = "SELECT Events.Id, Events.Name
FROM Events LEFT JOIN Monitors ON (Events.MonitorId = Monitors.Id)
@ -401,11 +366,10 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't prepare '$selectOrphanedEventsSql': ".$dbh->errstr() );
$res = $selectOrphanedEventsSth->execute()
or Fatal( "Can't execute: ".$selectOrphanedEventsSth->errstr() );
while( my $event = $selectOrphanedEventsSth->fetchrow_hashref() )
{
while( my $event = $selectOrphanedEventsSth->fetchrow_hashref() ) {
aud_print( "Found orphaned event with no monitor '$event->{Id}'" );
if ( confirm() )
{
if ( confirm() ) {
$res = $deleteEventSth->execute( $event->{Id} )
or Fatal( "Can't execute: ".$deleteEventSth->errstr() );
$cleaned = 1;
@ -413,7 +377,7 @@ Debug("Database events apparent exists at " . $Event->Path() );
}
redo MAIN if ( $cleaned );
# Remove empty events (with no frames)
# Remove empty events (with no frames)
$cleaned = 0;
my $selectEmptyEventsSql = "SELECT E.Id AS Id, E.StartTime, F.EventId FROM Events as E LEFT JOIN Frames as F ON (E.Id = F.EventId)
WHERE isnull(F.EventId) AND now() - interval ".$Config{ZM_AUDIT_MIN_AGE}." second > E.StartTime";
@ -421,11 +385,9 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't prepare '$selectEmptyEventsSql': ".$dbh->errstr() );
$res = $selectEmptyEventsSth->execute()
or Fatal( "Can't execute: ".$selectEmptyEventsSth->errstr() );
while( my $event = $selectEmptyEventsSth->fetchrow_hashref() )
{
while( my $event = $selectEmptyEventsSth->fetchrow_hashref() ) {
aud_print( "Found empty event with no frame records '$event->{Id}'" );
if ( confirm() )
{
if ( confirm() ) {
$res = $deleteEventSth->execute( $event->{Id} )
or Fatal( "Can't execute: ".$deleteEventSth->errstr() );
$cleaned = 1;
@ -433,7 +395,7 @@ Debug("Database events apparent exists at " . $Event->Path() );
}
redo MAIN if ( $cleaned );
# Remove orphaned frame records
# Remove orphaned frame records
$cleaned = 0;
my $selectOrphanedFramesSql = "SELECT DISTINCT EventId FROM Frames
WHERE EventId NOT IN (SELECT Id FROM Events)";
@ -441,11 +403,9 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't prepare '$selectOrphanedFramesSql': ".$dbh->errstr() );
$res = $selectOrphanedFramesSth->execute()
or Fatal( "Can't execute: ".$selectOrphanedFramesSth->errstr() );
while( my $frame = $selectOrphanedFramesSth->fetchrow_hashref() )
{
while( my $frame = $selectOrphanedFramesSth->fetchrow_hashref() ) {
aud_print( "Found orphaned frame records for event '$frame->{EventId}'" );
if ( confirm() )
{
if ( confirm() ) {
$res = $deleteFramesSth->execute( $frame->{EventId} )
or Fatal( "Can't execute: ".$deleteFramesSth->errstr() );
$cleaned = 1;
@ -453,7 +413,7 @@ Debug("Database events apparent exists at " . $Event->Path() );
}
redo MAIN if ( $cleaned );
# Remove orphaned stats records
# Remove orphaned stats records
$cleaned = 0;
my $selectOrphanedStatsSql = "SELECT DISTINCT EventId FROM Stats
WHERE EventId NOT IN (SELECT Id FROM Events)";
@ -461,11 +421,9 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't prepare '$selectOrphanedStatsSql': ".$dbh->errstr() );
$res = $selectOrphanedStatsSth->execute()
or Fatal( "Can't execute: ".$selectOrphanedStatsSth->errstr() );
while( my $stat = $selectOrphanedStatsSth->fetchrow_hashref() )
{
while( my $stat = $selectOrphanedStatsSth->fetchrow_hashref() ) {
aud_print( "Found orphaned statistic records for event '$stat->{EventId}'" );
if ( confirm() )
{
if ( confirm() ) {
$res = $deleteStatsSth->execute( $stat->{EventId} )
or Fatal( "Can't execute: ".$deleteStatsSth->errstr() );
$cleaned = 1;
@ -473,21 +431,21 @@ Debug("Database events apparent exists at " . $Event->Path() );
}
redo MAIN if ( $cleaned );
# New audit to close any events that were left open for longer than MIN_AGE seconds
# New audit to close any events that were left open for longer than MIN_AGE seconds
my $selectUnclosedEventsSql =
#"SELECT E.Id, ANY_VALUE(E.MonitorId),
#"SELECT E.Id, ANY_VALUE(E.MonitorId),
#
#max(F.TimeStamp) as EndTime,
#unix_timestamp(max(F.TimeStamp)) - unix_timestamp(E.StartTime) as Length,
#max(F.FrameId) as Frames,
#count(if(F.Score>0,1,NULL)) as AlarmFrames,
#sum(F.Score) as TotScore,
#max(F.Score) as MaxScore
#FROM Events as E
#INNER JOIN Frames as F on E.Id = F.EventId
#WHERE isnull(E.Frames) or isnull(E.EndTime)
#GROUP BY E.Id HAVING EndTime < (now() - interval ".$Config{ZM_AUDIT_MIN_AGE}." second)"
#;
#max(F.TimeStamp) as EndTime,
#unix_timestamp(max(F.TimeStamp)) - unix_timestamp(E.StartTime) as Length,
#max(F.FrameId) as Frames,
#count(if(F.Score>0,1,NULL)) as AlarmFrames,
#sum(F.Score) as TotScore,
#max(F.Score) as MaxScore
#FROM Events as E
#INNER JOIN Frames as F on E.Id = F.EventId
#WHERE isnull(E.Frames) or isnull(E.EndTime)
#GROUP BY E.Id HAVING EndTime < (now() - interval ".$Config{ZM_AUDIT_MIN_AGE}." second)"
#;
"SELECT *, unix_timestamp(StartTime) AS TimeStamp FROM Events WHERE EndTime IS NULL AND StartTime < (now() - interval ".$Config{ZM_AUDIT_MIN_AGE}." second)";
my $selectFrameDataSql = "SELECT max(TimeStamp) as EndTime, unix_timestamp(max(TimeStamp)) AS EndTimeStamp, max(FrameId) as Frames,
@ -518,16 +476,13 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't prepare '$updateUnclosedEventsSql': ".$dbh->errstr() );
$res = $selectUnclosedEventsSth->execute()
or Fatal( "Can't execute: ".$selectUnclosedEventsSth->errstr() );
while( my $event = $selectUnclosedEventsSth->fetchrow_hashref() )
{
while( my $event = $selectUnclosedEventsSth->fetchrow_hashref() ) {
aud_print( "Found open event '$event->{Id}'" );
if ( confirm( 'close', 'closing' ) )
{
if ( confirm( 'close', 'closing' ) ) {
$res = $selectFrameDataSth->execute( $event->{Id} ) or Fatal( "Can't execute: ".$selectFrameDataSth->errstr() );
my $frame = $selectFrameDataSth->fetchrow_hashref();
if ( $frame ) {
$res = $updateUnclosedEventsSth->execute
(
$res = $updateUnclosedEventsSth->execute(
sprintf("%s%d%s",
$Monitors{$event->{MonitorId}}->{EventPrefix},
$event->{Id},
@ -550,11 +505,10 @@ Debug("Database events apparent exists at " . $Event->Path() );
Error("SHOULD DELETE");
} # end if has frame data
}
}
} # end while unclosed event
# Now delete any old image files
if ( my @old_files = grep { -M > $max_image_age } <$image_path/*.{jpg,gif,wbmp}> )
{
# Now delete any old image files
if ( my @old_files = grep { -M > $max_image_age } <$image_path/*.{jpg,gif,wbmp}> ) {
aud_print( "Deleting ".int(@old_files)." old images\n" );
my $untainted_old_files = join( ";", @old_files );
( $untainted_old_files ) = ( $untainted_old_files =~ /^(.*)$/ );
@ -566,10 +520,8 @@ Debug("Database events apparent exists at " . $Event->Path() );
File::Find::find( { wanted=>\&deleteSwapImage, untaint=>1 }, $swap_image_root );
# Prune the Logs table if required
if ( $Config{ZM_LOG_DATABASE_LIMIT} )
{
if ( $Config{ZM_LOG_DATABASE_LIMIT} =~ /^\d+$/ )
{
if ( $Config{ZM_LOG_DATABASE_LIMIT} ) {
if ( $Config{ZM_LOG_DATABASE_LIMIT} =~ /^\d+$/ ) {
# Number of rows
my $selectLogRowCountSql = "SELECT count(*) as Rows from Logs";
my $selectLogRowCountSth = $dbh->prepare_cached( $selectLogRowCountSql )
@ -578,23 +530,18 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't execute: ".$selectLogRowCountSth->errstr() );
my $row = $selectLogRowCountSth->fetchrow_hashref();
my $logRows = $row->{Rows};
if ( $logRows > $Config{ZM_LOG_DATABASE_LIMIT} )
{
if ( $logRows > $Config{ZM_LOG_DATABASE_LIMIT} ) {
my $deleteLogByRowsSql = "DELETE low_priority FROM Logs ORDER BY TimeKey ASC LIMIT ?";
my $deleteLogByRowsSth = $dbh->prepare_cached( $deleteLogByRowsSql )
or Fatal( "Can't prepare '$deleteLogByRowsSql': ".$dbh->errstr() );
$res = $deleteLogByRowsSth->execute( $logRows - $Config{ZM_LOG_DATABASE_LIMIT} )
or Fatal( "Can't execute: ".$deleteLogByRowsSth->errstr() );
if ( $deleteLogByRowsSth->rows() )
{
if ( $deleteLogByRowsSth->rows() ) {
aud_print( "Deleted ".$deleteLogByRowsSth->rows()
." log table entries by count\n" )
;
." log table entries by count\n" );
}
}
}
else
{
} else {
# Time of record
my $deleteLogByTimeSql =
"DELETE low_priority FROM Logs
@ -605,8 +552,7 @@ Debug("Database events apparent exists at " . $Event->Path() );
or Fatal( "Can't execute: ".$deleteLogByTimeSth->errstr() );
if ( $deleteLogByTimeSth->rows() ){
aud_print( "Deleted ".$deleteLogByTimeSth->rows()
." log table entries by time\n" )
;
." log table entries by time\n" );
}
}
} # end if ZM_LOG_DATABASE_LIMIT
@ -617,52 +563,37 @@ Debug("Database events apparent exists at " . $Event->Path() );
exit( 0 );
sub aud_print
{
sub aud_print {
my $string = shift;
if ( !$continuous )
{
if ( !$continuous ) {
print( $string );
}
else
{
} else {
Info( $string );
}
}
sub confirm
{
sub confirm {
my $prompt = shift || "delete";
my $action = shift || "deleting";
my $yesno = 0;
if ( $report )
{
if ( $report ) {
print( "\n" );
}
elsif ( $interactive )
{
} elsif ( $interactive ) {
print( ", $prompt y/n: " );
my $char = <>;
chomp( $char );
if ( $char eq 'q' )
{
if ( $char eq 'q' ) {
exit( 0 );
}
if ( !$char )
{
if ( !$char ) {
$char = 'y';
}
$yesno = ( $char =~ /[yY]/ );
}
else
{
if ( !$continuous )
{
} else {
if ( !$continuous ) {
print( ", $action\n" );
}
else
{
} else {
Info( $action );
}
$yesno = 1;
@ -670,24 +601,41 @@ sub confirm
return( $yesno );
}
sub deleteSwapImage
{
sub deleteSwapImage {
my $file = $_;
if ( $file !~ /^zmswap-/ )
{
if ( $file !~ /^zmswap-/ ) {
return;
}
# Ignore directories
if ( -d $file )
{
# Ignore directories
if ( -d $file ) {
return;
}
if ( -M $file > $max_swap_age )
{
if ( -M $file > $max_swap_age ) {
Debug( "Deleting $file" );
#unlink( $file );
#unlink( $file );
}
}
sub delete_empty_directories {
if ( ! opendir( DIR, $_[0] ) ) {
Error( "Can't open directory '$_[0]': $!" );
return;
}
my @contents = map { $_ eq '.' or $_ eq '..' ? () : $_ } readdir( DIR );
my @dirs = map { -d $_ ? $_ : () } @contents;
if ( @dirs ) {
foreach ( @dirs ) {
delete_empty_directories( $_[0].'/'.$_ );
}
#Reload, since we may now be empty
@contents = map { $_ eq '.' or $_ eq '..' ? () : $_ } readdir( DIR );
}
if ( ! @contents ) {
unlink $_[0];
}
closedir( DIR );
} # end sub delete_empty_directories