ZaneYork
/
zoneminder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

add StorageScheme to Storage and Events. Deprecate ZM_USE_DEEP_STORAGE

This commit is contained in:
Isaac Connor 2017-12-18 12:52:26 -05:00
parent be017d526e
commit d312482a2b
14 changed files with 146 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
db/zm_create.sql.in
View File

@ -639,6 +639,7 @@ CREATE TABLE `Storage` (
`Name` varchar(64) NOT NULL default '',
`Type` enum('local','s3fs') NOT NULL default 'local',
`DiskSpace` bigint unsigned default NULL,
`Scheme enum('Deep','Medium','Shallow') NOT NULL default 'Medium',
PRIMARY KEY (`Id`)
) ENGINE=@ZM_MYSQL_ENGINE@;

28
db/zm_update-1.31.17.sql Normal file
View File

@ -0,0 +1,28 @@
alter table Events modify Id int(10) unsigned;
alter table Events DROP Primary key;
alter table Events Add Primary key(Id);
alter table Events modify Id int(10) unsigned auto_incremement;
SET @s = (SELECT IF(
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = DATABASE()
AND table_name = 'Storage'
AND column_name = 'Scheme'
) > 0,
"SELECT 'Column Scheme already exists in Storage'",
"ALTER TABLE Storage ADD `Scheme enum('Deep','Medium','Shallow') NOT NULL default 'Medium' AFTER `DiskSpace`"
));
PREPARE stmt FROM @s;
EXECUTE stmt;
SET @s = (SELECT IF(
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = DATABASE()
AND table_name = 'Events'
AND column_name = 'StorageScheme'
) > 0,
"SELECT 'Column StorageScheme already exists in Events'",
"ALTER TABLE Events ADD `StorageScheme enum('Deep','Medium','Shallow') NOT NULL default 'Deep' AFTER `DiskSpace`"
));
PREPARE stmt FROM @s;
EXECUTE stmt;

37
scripts/ZoneMinder/lib/ZoneMinder/Event.pm
View File

@ -154,13 +154,14 @@ sub Path {
if ( ! $$event{Path} ) {
my $Storage = $event->Storage();
if ( $Config{ZM_USE_DEEP_STORAGE} ) {
if ( $$events{Scheme} eq 'Deep' ) {
if ( $event->Time() ) {
$$event{Path} = join('/',
$Storage->Path(),
$event->{MonitorId},
strftime( "%y/%m/%d/%H/%M/%S",
strftime( '%y/%m/%d/%H/%M/%S',
localtime($event->Time())
),
);
@ -168,7 +169,18 @@ sub Path {
Error("Event $$event{Id} has no value for Time(), unable to determine path");
$$event{Path} = '';
}
} else {
} elsif ( $$events{Scheme} eq 'Medium' ) {
if ( $event->Time() ) {
$$event{Path} = join('/',
$Storage->Path(),
$event->{MonitorId},
strftime( '%y-%m-%d', localtime($event->Time())),
$event->{Id},
);
} else {
Error("Event $$event{Id} has no value for Time(), unable to determine path");
$$event{Path} = '';
} else { # Shallow
$$event{Path} = join('/',
$Storage->Path(),
$event->{MonitorId},
@ -307,24 +319,25 @@ sub delete {
} # end sub delete
sub delete_files {
my $event = shift;
my $Storage = @_ > 1 ? $_[1] : new ZoneMinder::Storage( $_[0]{StorageId} );
my $Storage = @_ ? $_[0] : new ZoneMinder::Storage( $$event{StorageId} );
my $storage_path = $Storage->Path();
if ( ! $storage_path ) {
Fatal("Empty storage path when deleting files for event $_[0]{Id} with storage id $_[0]{StorageId} ");
Fatal("Empty storage path when deleting files for event $$event{Id} with storage id $$event{StorageId} ");
return;
}
chdir( $storage_path );
if ( $Config{ZM_USE_DEEP_STORAGE} ) {
if ( ! $_[0]{MonitorId} ) {
Error("No monitor id assigned to event $_[0]{Id}");
if ( $$event{Scheme} eq 'Deep' ) {
if ( ! $$event{MonitorId} ) {
Error("No monitor id assigned to event $$event{Id}");
return;
}
Debug("Deleting files for Event $_[0]{Id} from $storage_path.");
my $link_path = $_[0]{MonitorId}."/*/*/*/.".$_[0]{Id};
Debug("Deleting files for Event $$event{Id} from $storage_path.");
my $link_path = $$event{MonitorId}."/*/*/*/.".$$event{Id};
#Debug( "LP1:$link_path" );
my @links = glob($link_path);
#Debug( "L:".$links[0].": $!" );
@ -356,9 +369,9 @@ sub delete_files {
my $command = "/bin/rm -rf $storage_path/$delete_path";
ZoneMinder::General::executeShellCommand( $command );
}
}
} # end if links
} else {
my $command = "/bin/rm -rf $storage_path/$_[0]{MonitorId}/$_[0]{Id}";
my $command = "/bin/rm -rf ". $event->Path();
ZoneMinder::General::executeShellCommand( $command );
}
} # end sub delete_files

11
scripts/zmaudit.pl.in
View File

@ -191,7 +191,7 @@ MAIN: while( $loop ) {
# De-taint
( my $monitor_dir ) = ( $monitor =~ /^(.*)$/ );
if ( $Config{ZM_USE_DEEP_STORAGE} ) {
if ( $$Storage{Scheme} eq 'Deep' ) {
foreach my $day_dir ( glob("$monitor_dir/*/*/*") ) {
Debug( "Checking day dir $day_dir" );
( $day_dir ) = ( $day_dir =~ /^(.*)$/ ); # De-taint
@ -235,6 +235,15 @@ MAIN: while( $loop ) {
} # end foreach event_link
chdir( $Storage->Path() );
} # end foreach day dir
} elsif ( $$Storage{Scheme} eq 'Medium' ) {
foreach my $event_dir ( glob("$monitor_dir/*/*") ) {
next if ! -d $event_dir;
my $Event = $fs_events->{$event} = new ZoneMinder::Event();
$$Event{Id} = $event;
$$Event{Path} = $event_dir;
$Event->MonitorId( $monitor_dir );
$Event->StorageId( $Storage->Id() );
} # end foreach event
} else {
if ( ! chdir( $monitor_dir ) ) {
Error( "Can't chdir directory '$$Storage{Path}/$monitor_dir': $!" );

41
src/zm_event.cpp
View File

@ -71,7 +71,7 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string
static char sql[ZM_SQL_MED_BUFSIZ];
struct tm *stime = localtime( &start_time.tv_sec );
snprintf( sql, sizeof(sql), "INSERT INTO Events ( MonitorId, StorageId, Name, StartTime, Width, Height, Cause, Notes, StateId, Orientation, Videoed, DefaultVideo, SaveJPEGs ) values ( %d, %d, 'New Event', from_unixtime( %ld ), %d, %d, '%s', '%s', %d, %d, %d, '', %d )",
snprintf( sql, sizeof(sql), "INSERT INTO Events ( MonitorId, StorageId, Name, StartTime, Width, Height, Cause, Notes, StateId, Orientation, Videoed, DefaultVideo, SaveJPEGs, StorageScheme ) values ( %d, %d, 'New Event', from_unixtime( %ld ), %d, %d, '%s', '%s', %d, %d, %d, '', %d, '%s' )",
monitor->Id(),
storage->Id(),
start_time.tv_sec,
@ -82,7 +82,8 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string
state_id,
monitor->getOrientation(),
videoEvent,
monitor->GetOptSaveJPEGs()
monitor->GetOptSaveJPEGs(),
storage->SchemeString()
);
if ( mysql_query( &dbconn, sql ) ) {
Error( "Can't insert event: %s. sql was (%s)", mysql_error( &dbconn ), sql );
@ -101,7 +102,7 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string
struct stat statbuf;
char id_file[PATH_MAX];
if ( config.use_deep_storage ) {
if ( storage->Scheme() == Storage::Schemes::DEEP ) {
char *path_ptr = path;
path_ptr += snprintf( path_ptr, sizeof(path), "%s/%d", storage->Path(), monitor->Id() );
@ -120,15 +121,10 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string
path_ptr += snprintf( path_ptr, sizeof(path)-(path_ptr-path), "/%02d", dt_parts[i] );
errno = 0;
// Do we really need to stat it? Perhaps we could do that on error, instead
if ( stat( path, &statbuf ) ) {
if ( errno == ENOENT || errno == ENOTDIR ) {
if ( mkdir( path, 0755 ) ) {
// FIXME This should not be fatal. Should probably move to a different storage area.
Fatal( "Can't mkdir %s: %s", path, strerror(errno));
}
} else {
Warning( "Error stat'ing %s, may be fatal. error is %s", path, strerror(errno));
if ( mkdir( path, 0755 ) ) {
// FIXME This should not be fatal. Should probably move to a different storage area.
if ( errno != EEXIST ) {
Error( "Can't mkdir %s: %s", path, strerror(errno));
}
}
if ( i == 2 )
@ -140,11 +136,26 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string
snprintf( id_file, sizeof(id_file), "%s/.%d", date_path, id );
if ( symlink( time_path, id_file ) < 0 )
Error( "Can't symlink %s -> %s: %s", id_file, path, strerror(errno));
} else if ( storage->Scheme() == Storage::Schemes::MEDIUM ) {
char *path_ptr = path;
path_ptr += snprintf( path_ptr, sizeof(path), "%s/%d/%02d-%02d-%02d",
storage->Path(), monitor->Id(), stime->tm_year-100, stime->tm_mon+1, stime->tm_mday
);
if ( mkdir( path, 0755 ) ) {
// FIXME This should not be fatal. Should probably move to a different storage area.
if ( errno != EEXIST )
Error( "Can't mkdir %s: %s", path, strerror(errno));
}
path_ptr += snprintf( path_ptr, sizeof(path), "/%d", id );
if ( mkdir( path, 0755 ) ) {
// FIXME This should not be fatal. Should probably move to a different storage area.
if ( errno != EEXIST )
Error( "Can't mkdir %s: %s", path, strerror(errno));
}
} else {
snprintf( path, sizeof(path), "%s/%d/%d", storage->Path(), monitor->Id(), id );
if ( stat( path, &statbuf ) && ( errno == ENOENT || errno == ENOTDIR ) ) {
if ( mkdir( path, 0755 ) ) {
if ( mkdir( path, 0755 ) ) {
if ( errno != EEXIST ) {
Error( "Can't mkdir %s: %s", path, strerror(errno));
}
}

26
src/zm_storage.cpp
View File

@ -36,6 +36,8 @@ Storage::Storage() {
} else {
strncpy(path, staticConfig.DIR_EVENTS.c_str(), sizeof(path)-1 );
}
scheme = Schemes::SHALLOW;
scheme_str = "Shallow";
}
Storage::Storage( MYSQL_ROW &dbrow ) {
@ -43,6 +45,15 @@ Storage::Storage( MYSQL_ROW &dbrow ) {
id = atoi( dbrow[index++] );
strncpy( name, dbrow[index++], sizeof(name)-1 );
strncpy( path, dbrow[index++], sizeof(path)-1 );
type_str = std::string(dbrow[index++]);
scheme_str = std::string(dbrow[index++]);
if ( scheme_str == "Deep" ) {
scheme = Schemes::DEEP;
} else if ( scheme_str == "Medium" ) {
scheme = Schemes::MEDIUM;
} else {
scheme = Schemes::SHALLOW;
}
}
/* If a zero or invalid p_id is passed, then the old default path will be assumed. */
@ -51,7 +62,7 @@ Storage::Storage( unsigned int p_id ) {
if ( p_id ) {
char sql[ZM_SQL_SML_BUFSIZ];
snprintf( sql, sizeof(sql), "SELECT Id, Name, Path from Storage WHERE Id=%d", p_id );
snprintf( sql, sizeof(sql), "SELECT Id, Name, Path, Type, Scheme from Storage WHERE Id=%d", p_id );
Debug(2,"Loading Storage for %d using %s", p_id, sql );
zmDbRow dbrow;
if ( ! dbrow.fetch( sql ) ) {
@ -59,8 +70,17 @@ Storage::Storage( unsigned int p_id ) {
} else {
unsigned int index = 0;
id = atoi( dbrow[index++] );
strncpy( name, dbrow[index++], sizeof(name) );
strncpy( path, dbrow[index++], sizeof(path) );
strncpy( name, dbrow[index++], sizeof(name)-1 );
strncpy( path, dbrow[index++], sizeof(path)-1 );
type_str = std::string(dbrow[index++]);
scheme_str = std::string(dbrow[index++]);
if ( scheme_str == "Deep" ) {
scheme = Schemes::DEEP;
} else if ( scheme_str == "Medium" ) {
scheme = Schemes::MEDIUM;
} else {
scheme = Schemes::SHALLOW;
}
Debug( 1, "Loaded Storage area %d '%s'", id, this->Name() );
}
}

16
src/zm_storage.h
View File

@ -24,11 +24,19 @@
class Storage {
public:
typedef enum {
SHALLOW=0,
MEDIUM,
DEEP
} Schemes;
protected:
unsigned int id;
char name[64+1];
char path[64+1];
std::string type_str;
std::string scheme_str;
Schemes scheme;
public:
Storage();
@ -36,9 +44,11 @@ public:
explicit Storage( unsigned int p_id );
~Storage();
unsigned int Id() const { return( id ); }
const char *Name() const { return( name ); }
const char *Path() const { return( path ); }
unsigned int Id() const { return id; }
const char *Name() const { return name; }
const char *Path() const { return path; }
const Schemes Scheme() const { return scheme; }
const std::string SchemeString() const { return scheme_str; }
};
#endif // ZM_STORAGE_H

2
version
View File

@ -1 +1 @@
1.31.16
1.31.17

1
web/ajax/status.php
View File

@ -138,7 +138,6 @@ $statusData = array(
'MaxFrameId' => array( 'sql' => '(SELECT max(Frames.FrameId) FROM Frames WHERE Events.Id = Frames.EventId)' ),
'MinFrameDelta' => array( 'sql' => '(SELECT min(Frames.Delta) FROM Frames WHERE Events.Id = Frames.EventId)' ),
'MaxFrameDelta' => array( 'sql' => '(SELECT max(Frames.Delta) FROM Frames WHERE Events.Id = Frames.EventId)' ),
//'Path' => array( 'postFunc' => 'getEventPath' ),
),
),
'frames' => array(

4
web/api/app/Controller/EventsController.php
View File

@ -123,8 +123,8 @@ class EventsController extends AppController {
$options = array('conditions' => array(array('Event.' . $this->Event->primaryKey => $id), $mon_options));
$event = $this->Event->find('first', $options);
$path = $configs['ZM_DIR_EVENTS'].'/'.$this->Image->getEventPath($event).'/';
$event['Event']['BasePath'] = $path;
//$path = $configs['ZM_DIR_EVENTS'].'/'.$this->Image->getEventPath($event).'/';
//$event['Event']['BasePath'] = $path;
# Get the previous and next events for any monitor
$this->Event->id = $id;

8
web/includes/Event.php
View File

@ -86,8 +86,10 @@ class Event {
public function Relative_Path() {
$event_path = '';
if ( ZM_USE_DEEP_STORAGE ) {
if ( $this->{'Scheme'} == 'Deep' ) {
$event_path = $this->{'MonitorId'} .'/'.strftime( '%y/%m/%d/%H/%M/%S', $this->Time()) ;
} else if ( $this->{'Scheme'} eq 'Medium' ) {
$event_path = $this->{'MonitorId'} .'/'.strftime( '%y-%m-%d', $this->Time() ) . '/'.$this->{'Id'};
} else {
$event_path = $this->{'MonitorId'} .'/'.$this->{'Id'};
}
@ -96,7 +98,7 @@ class Event {
} // end function Relative_Path()
public function Link_Path() {
if ( ZM_USE_DEEP_STORAGE ) {
if ( $this->{'Scheme'} == 'Deep' ) {
return $this->{'MonitorId'} .'/'.strftime( '%y/%m/%d/.', $this->Time()).$this->{'Id'};
}
Error('Calling Link_Path when not using deep storage');
@ -109,7 +111,7 @@ class Event {
if ( !ZM_OPT_FAST_DELETE ) {
dbQuery( 'DELETE FROM Stats WHERE EventId = ?', array($this->{'Id'}) );
dbQuery( 'DELETE FROM Frames WHERE EventId = ?', array($this->{'Id'}) );
if ( ZM_USE_DEEP_STORAGE ) {
if ( $this->{'Scheme'} == 'Deep' ) {
# Assumption: All events have a start time
$start_date = date_parse( $this->{'StartTime'} );

28
web/includes/Frame.php
View File

@ -49,34 +49,6 @@ class Frame {
}
}
public function Path() {
$Storage = $this->Storage();
return $Storage->Path().'/'.$this->Relative_Path();
}
public function Relative_Path() {
$event_path = "";
if ( ZM_USE_DEEP_STORAGE )
{
$event_path =
$this->{'MonitorId'}
.'/'.strftime( "%y/%m/%d/%H/%M/%S",
$this->Time()
)
;
}
else
{
$event_path =
$this->{'MonitorId'}
.'/'.$this->{'Id'}
;
}
return( $event_path );
}
public function getImageSrc( $show='capture' ) {
return $_SERVER['PHP_SELF'].'?view=image&fid='.$this->{'FrameId'}.'&eid='.$this->{'EventId'}.'&show='.$show;

8
web/includes/functions.php
View File

@ -465,14 +465,6 @@ function canEdit( $area, $mid=false ) {
return( $user[$area] == 'Edit' && ( !$mid || visibleMonitor( $mid ) ) );
}
function getEventPath( $event ) {
if ( ZM_USE_DEEP_STORAGE )
$eventPath = $event['MonitorId'].'/'.strftime( '%y/%m/%d/%H/%M/%S', strtotime($event['StartTime']) );
else
$eventPath = $event['MonitorId'].'/'.$event['Id'];
return( $eventPath );
}
function getEventDefaultVideoPath( $event ) {
$Event = new Event( $event );
return $Event->getStreamSrc( array( 'mode'=>'mpeg', 'format'=>'h264' ) );

15
web/skins/classic/views/storage.php
View File

@ -33,9 +33,15 @@ if ( $_REQUEST['id'] ) {
$newStorage['Name'] = translate('NewStorage');
$newStorage['Path'] = '';
$newStorage['Type'] = 'local';
$newStorage['Scheme'] = 'Medium';
}
$type_options = array( 'local' => translate('Local'), 's3fs' => translate('s3fs') );
$scheme_options = array(
'Deep' => translate('Deep'),
'Medium' => translate('Medium'),
'Shallow' => translate('Shallow'),
);
$focusWindow = true;
@ -65,12 +71,15 @@ xhtmlHeaders(__FILE__, translate('Storage')." - ".$newStorage['Name'] );
<th scope="row"><?php echo translate('Type') ?></th>
<td><?php echo htmlSelect( 'newStorage[Type]', $type_options, $newStorage['Type'] ); ?></td>
</tr>
<tr>
<th scope="row"><?php echo translate('StorageScheme') ?></th>
<td><?php echo htmlSelect( 'newStorage[Scheme]', $scheme_options, $newStorage['Scheme'] ); ?></td>
</tr>
</tbody>
</table>
<div id="contentButtons">
<input type="hidden" name="action" value="Save"/>
<input type="submit" value="<?php echo translate('Save') ?>"/>
<input type="button" value="<?php echo translate('Cancel') ?>" onclick="closeWindow();"/>
<button name="action" type="submit" value="Save"><?php echo translate('Save') ?></button>
<button type="button" onclick="closeWindow();"><?php echo translate('Cancel') ?></button>
</div>
</form>
</div>