From d312482a2bacb972003fe58cc4f318c818407d8d Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 18 Dec 2017 12:52:26 -0500 Subject: [PATCH] add StorageScheme to Storage and Events. Deprecate ZM_USE_DEEP_STORAGE --- db/zm_create.sql.in | 1 + db/zm_update-1.31.17.sql | 28 ++++++++++++++ scripts/ZoneMinder/lib/ZoneMinder/Event.pm | 37 +++++++++++++------ scripts/zmaudit.pl.in | 11 +++++- src/zm_event.cpp | 41 +++++++++++++-------- src/zm_storage.cpp | 26 +++++++++++-- src/zm_storage.h | 16 ++++++-- version | 2 +- web/ajax/status.php | 1 - web/api/app/Controller/EventsController.php | 4 +- web/includes/Event.php | 8 ++-- web/includes/Frame.php | 28 -------------- web/includes/functions.php | 8 ---- web/skins/classic/views/storage.php | 15 ++++++-- 14 files changed, 146 insertions(+), 80 deletions(-) create mode 100644 db/zm_update-1.31.17.sql diff --git a/db/zm_create.sql.in b/db/zm_create.sql.in index ec035d227..5d15ca59b 100644 --- a/db/zm_create.sql.in +++ b/db/zm_create.sql.in @@ -639,6 +639,7 @@ CREATE TABLE `Storage` ( `Name` varchar(64) NOT NULL default '', `Type` enum('local','s3fs') NOT NULL default 'local', `DiskSpace` bigint unsigned default NULL, + `Scheme enum('Deep','Medium','Shallow') NOT NULL default 'Medium', PRIMARY KEY (`Id`) ) ENGINE=@ZM_MYSQL_ENGINE@; diff --git a/db/zm_update-1.31.17.sql b/db/zm_update-1.31.17.sql new file mode 100644 index 000000000..3177eaaeb --- /dev/null +++ b/db/zm_update-1.31.17.sql @@ -0,0 +1,28 @@ +alter table Events modify Id int(10) unsigned; +alter table Events DROP Primary key; +alter table Events Add Primary key(Id); +alter table Events modify Id int(10) unsigned auto_incremement; + +SET @s = (SELECT IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = DATABASE() + AND table_name = 'Storage' + AND column_name = 'Scheme' + ) > 0, + "SELECT 'Column Scheme already exists in Storage'", + "ALTER TABLE Storage ADD `Scheme enum('Deep','Medium','Shallow') NOT NULL default 'Medium' AFTER `DiskSpace`" + )); + +PREPARE stmt FROM @s; +EXECUTE stmt; + +SET @s = (SELECT IF( + (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = DATABASE() + AND table_name = 'Events' + AND column_name = 'StorageScheme' + ) > 0, + "SELECT 'Column StorageScheme already exists in Events'", + "ALTER TABLE Events ADD `StorageScheme enum('Deep','Medium','Shallow') NOT NULL default 'Deep' AFTER `DiskSpace`" + )); + +PREPARE stmt FROM @s; +EXECUTE stmt; diff --git a/scripts/ZoneMinder/lib/ZoneMinder/Event.pm b/scripts/ZoneMinder/lib/ZoneMinder/Event.pm index 7f0dc6575..fc8698f15 100644 --- a/scripts/ZoneMinder/lib/ZoneMinder/Event.pm +++ b/scripts/ZoneMinder/lib/ZoneMinder/Event.pm @@ -154,13 +154,14 @@ sub Path { if ( ! $$event{Path} ) { my $Storage = $event->Storage(); + - if ( $Config{ZM_USE_DEEP_STORAGE} ) { + if ( $$events{Scheme} eq 'Deep' ) { if ( $event->Time() ) { $$event{Path} = join('/', $Storage->Path(), $event->{MonitorId}, - strftime( "%y/%m/%d/%H/%M/%S", + strftime( '%y/%m/%d/%H/%M/%S', localtime($event->Time()) ), ); @@ -168,7 +169,18 @@ sub Path { Error("Event $$event{Id} has no value for Time(), unable to determine path"); $$event{Path} = ''; } - } else { + } elsif ( $$events{Scheme} eq 'Medium' ) { + if ( $event->Time() ) { + $$event{Path} = join('/', + $Storage->Path(), + $event->{MonitorId}, + strftime( '%y-%m-%d', localtime($event->Time())), + $event->{Id}, + ); + } else { + Error("Event $$event{Id} has no value for Time(), unable to determine path"); + $$event{Path} = ''; + } else { # Shallow $$event{Path} = join('/', $Storage->Path(), $event->{MonitorId}, @@ -307,24 +319,25 @@ sub delete { } # end sub delete sub delete_files { + my $event = shift; - my $Storage = @_ > 1 ? $_[1] : new ZoneMinder::Storage( $_[0]{StorageId} ); + my $Storage = @_ ? $_[0] : new ZoneMinder::Storage( $$event{StorageId} ); my $storage_path = $Storage->Path(); if ( ! $storage_path ) { - Fatal("Empty storage path when deleting files for event $_[0]{Id} with storage id $_[0]{StorageId} "); + Fatal("Empty storage path when deleting files for event $$event{Id} with storage id $$event{StorageId} "); return; } chdir( $storage_path ); - if ( $Config{ZM_USE_DEEP_STORAGE} ) { - if ( ! $_[0]{MonitorId} ) { - Error("No monitor id assigned to event $_[0]{Id}"); + if ( $$event{Scheme} eq 'Deep' ) { + if ( ! $$event{MonitorId} ) { + Error("No monitor id assigned to event $$event{Id}"); return; } - Debug("Deleting files for Event $_[0]{Id} from $storage_path."); - my $link_path = $_[0]{MonitorId}."/*/*/*/.".$_[0]{Id}; + Debug("Deleting files for Event $$event{Id} from $storage_path."); + my $link_path = $$event{MonitorId}."/*/*/*/.".$$event{Id}; #Debug( "LP1:$link_path" ); my @links = glob($link_path); #Debug( "L:".$links[0].": $!" ); @@ -356,9 +369,9 @@ sub delete_files { my $command = "/bin/rm -rf $storage_path/$delete_path"; ZoneMinder::General::executeShellCommand( $command ); } - } + } # end if links } else { - my $command = "/bin/rm -rf $storage_path/$_[0]{MonitorId}/$_[0]{Id}"; + my $command = "/bin/rm -rf ". $event->Path(); ZoneMinder::General::executeShellCommand( $command ); } } # end sub delete_files diff --git a/scripts/zmaudit.pl.in b/scripts/zmaudit.pl.in index 1593777b4..2de067c3e 100644 --- a/scripts/zmaudit.pl.in +++ b/scripts/zmaudit.pl.in @@ -191,7 +191,7 @@ MAIN: while( $loop ) { # De-taint ( my $monitor_dir ) = ( $monitor =~ /^(.*)$/ ); - if ( $Config{ZM_USE_DEEP_STORAGE} ) { + if ( $$Storage{Scheme} eq 'Deep' ) { foreach my $day_dir ( glob("$monitor_dir/*/*/*") ) { Debug( "Checking day dir $day_dir" ); ( $day_dir ) = ( $day_dir =~ /^(.*)$/ ); # De-taint @@ -235,6 +235,15 @@ MAIN: while( $loop ) { } # end foreach event_link chdir( $Storage->Path() ); } # end foreach day dir + } elsif ( $$Storage{Scheme} eq 'Medium' ) { + foreach my $event_dir ( glob("$monitor_dir/*/*") ) { + next if ! -d $event_dir; + my $Event = $fs_events->{$event} = new ZoneMinder::Event(); + $$Event{Id} = $event; + $$Event{Path} = $event_dir; + $Event->MonitorId( $monitor_dir ); + $Event->StorageId( $Storage->Id() ); + } # end foreach event } else { if ( ! chdir( $monitor_dir ) ) { Error( "Can't chdir directory '$$Storage{Path}/$monitor_dir': $!" ); diff --git a/src/zm_event.cpp b/src/zm_event.cpp index b86c2f18f..6d4a411de 100644 --- a/src/zm_event.cpp +++ b/src/zm_event.cpp @@ -71,7 +71,7 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string static char sql[ZM_SQL_MED_BUFSIZ]; struct tm *stime = localtime( &start_time.tv_sec ); - snprintf( sql, sizeof(sql), "INSERT INTO Events ( MonitorId, StorageId, Name, StartTime, Width, Height, Cause, Notes, StateId, Orientation, Videoed, DefaultVideo, SaveJPEGs ) values ( %d, %d, 'New Event', from_unixtime( %ld ), %d, %d, '%s', '%s', %d, %d, %d, '', %d )", + snprintf( sql, sizeof(sql), "INSERT INTO Events ( MonitorId, StorageId, Name, StartTime, Width, Height, Cause, Notes, StateId, Orientation, Videoed, DefaultVideo, SaveJPEGs, StorageScheme ) values ( %d, %d, 'New Event', from_unixtime( %ld ), %d, %d, '%s', '%s', %d, %d, %d, '', %d, '%s' )", monitor->Id(), storage->Id(), start_time.tv_sec, @@ -82,7 +82,8 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string state_id, monitor->getOrientation(), videoEvent, - monitor->GetOptSaveJPEGs() + monitor->GetOptSaveJPEGs(), + storage->SchemeString() ); if ( mysql_query( &dbconn, sql ) ) { Error( "Can't insert event: %s. sql was (%s)", mysql_error( &dbconn ), sql ); @@ -101,7 +102,7 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string struct stat statbuf; char id_file[PATH_MAX]; - if ( config.use_deep_storage ) { + if ( storage->Scheme() == Storage::Schemes::DEEP ) { char *path_ptr = path; path_ptr += snprintf( path_ptr, sizeof(path), "%s/%d", storage->Path(), monitor->Id() ); @@ -120,15 +121,10 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string path_ptr += snprintf( path_ptr, sizeof(path)-(path_ptr-path), "/%02d", dt_parts[i] ); errno = 0; - // Do we really need to stat it? Perhaps we could do that on error, instead - if ( stat( path, &statbuf ) ) { - if ( errno == ENOENT || errno == ENOTDIR ) { - if ( mkdir( path, 0755 ) ) { - // FIXME This should not be fatal. Should probably move to a different storage area. - Fatal( "Can't mkdir %s: %s", path, strerror(errno)); - } - } else { - Warning( "Error stat'ing %s, may be fatal. error is %s", path, strerror(errno)); + if ( mkdir( path, 0755 ) ) { + // FIXME This should not be fatal. Should probably move to a different storage area. + if ( errno != EEXIST ) { + Error( "Can't mkdir %s: %s", path, strerror(errno)); } } if ( i == 2 ) @@ -140,11 +136,26 @@ Event::Event( Monitor *p_monitor, struct timeval p_start_time, const std::string snprintf( id_file, sizeof(id_file), "%s/.%d", date_path, id ); if ( symlink( time_path, id_file ) < 0 ) Error( "Can't symlink %s -> %s: %s", id_file, path, strerror(errno)); + } else if ( storage->Scheme() == Storage::Schemes::MEDIUM ) { + char *path_ptr = path; + path_ptr += snprintf( path_ptr, sizeof(path), "%s/%d/%02d-%02d-%02d", + storage->Path(), monitor->Id(), stime->tm_year-100, stime->tm_mon+1, stime->tm_mday + ); + if ( mkdir( path, 0755 ) ) { + // FIXME This should not be fatal. Should probably move to a different storage area. + if ( errno != EEXIST ) + Error( "Can't mkdir %s: %s", path, strerror(errno)); + } + path_ptr += snprintf( path_ptr, sizeof(path), "/%d", id ); + if ( mkdir( path, 0755 ) ) { + // FIXME This should not be fatal. Should probably move to a different storage area. + if ( errno != EEXIST ) + Error( "Can't mkdir %s: %s", path, strerror(errno)); + } } else { snprintf( path, sizeof(path), "%s/%d/%d", storage->Path(), monitor->Id(), id ); - - if ( stat( path, &statbuf ) && ( errno == ENOENT || errno == ENOTDIR ) ) { - if ( mkdir( path, 0755 ) ) { + if ( mkdir( path, 0755 ) ) { + if ( errno != EEXIST ) { Error( "Can't mkdir %s: %s", path, strerror(errno)); } } diff --git a/src/zm_storage.cpp b/src/zm_storage.cpp index 0b3857302..042bacc76 100644 --- a/src/zm_storage.cpp +++ b/src/zm_storage.cpp @@ -36,6 +36,8 @@ Storage::Storage() { } else { strncpy(path, staticConfig.DIR_EVENTS.c_str(), sizeof(path)-1 ); } + scheme = Schemes::SHALLOW; + scheme_str = "Shallow"; } Storage::Storage( MYSQL_ROW &dbrow ) { @@ -43,6 +45,15 @@ Storage::Storage( MYSQL_ROW &dbrow ) { id = atoi( dbrow[index++] ); strncpy( name, dbrow[index++], sizeof(name)-1 ); strncpy( path, dbrow[index++], sizeof(path)-1 ); + type_str = std::string(dbrow[index++]); + scheme_str = std::string(dbrow[index++]); + if ( scheme_str == "Deep" ) { + scheme = Schemes::DEEP; + } else if ( scheme_str == "Medium" ) { + scheme = Schemes::MEDIUM; + } else { + scheme = Schemes::SHALLOW; + } } /* If a zero or invalid p_id is passed, then the old default path will be assumed. */ @@ -51,7 +62,7 @@ Storage::Storage( unsigned int p_id ) { if ( p_id ) { char sql[ZM_SQL_SML_BUFSIZ]; - snprintf( sql, sizeof(sql), "SELECT Id, Name, Path from Storage WHERE Id=%d", p_id ); + snprintf( sql, sizeof(sql), "SELECT Id, Name, Path, Type, Scheme from Storage WHERE Id=%d", p_id ); Debug(2,"Loading Storage for %d using %s", p_id, sql ); zmDbRow dbrow; if ( ! dbrow.fetch( sql ) ) { @@ -59,8 +70,17 @@ Storage::Storage( unsigned int p_id ) { } else { unsigned int index = 0; id = atoi( dbrow[index++] ); - strncpy( name, dbrow[index++], sizeof(name) ); - strncpy( path, dbrow[index++], sizeof(path) ); + strncpy( name, dbrow[index++], sizeof(name)-1 ); + strncpy( path, dbrow[index++], sizeof(path)-1 ); + type_str = std::string(dbrow[index++]); + scheme_str = std::string(dbrow[index++]); + if ( scheme_str == "Deep" ) { + scheme = Schemes::DEEP; + } else if ( scheme_str == "Medium" ) { + scheme = Schemes::MEDIUM; + } else { + scheme = Schemes::SHALLOW; + } Debug( 1, "Loaded Storage area %d '%s'", id, this->Name() ); } } diff --git a/src/zm_storage.h b/src/zm_storage.h index 75817a903..f0886c9f3 100644 --- a/src/zm_storage.h +++ b/src/zm_storage.h @@ -24,11 +24,19 @@ class Storage { public: + typedef enum { + SHALLOW=0, + MEDIUM, + DEEP + } Schemes; protected: unsigned int id; char name[64+1]; char path[64+1]; + std::string type_str; + std::string scheme_str; + Schemes scheme; public: Storage(); @@ -36,9 +44,11 @@ public: explicit Storage( unsigned int p_id ); ~Storage(); - unsigned int Id() const { return( id ); } - const char *Name() const { return( name ); } - const char *Path() const { return( path ); } + unsigned int Id() const { return id; } + const char *Name() const { return name; } + const char *Path() const { return path; } + const Schemes Scheme() const { return scheme; } + const std::string SchemeString() const { return scheme_str; } }; #endif // ZM_STORAGE_H diff --git a/version b/version index d1a8f5341..874058d96 100644 --- a/version +++ b/version @@ -1 +1 @@ -1.31.16 +1.31.17 diff --git a/web/ajax/status.php b/web/ajax/status.php index dcd53cc95..2e9977a3d 100644 --- a/web/ajax/status.php +++ b/web/ajax/status.php @@ -138,7 +138,6 @@ $statusData = array( 'MaxFrameId' => array( 'sql' => '(SELECT max(Frames.FrameId) FROM Frames WHERE Events.Id = Frames.EventId)' ), 'MinFrameDelta' => array( 'sql' => '(SELECT min(Frames.Delta) FROM Frames WHERE Events.Id = Frames.EventId)' ), 'MaxFrameDelta' => array( 'sql' => '(SELECT max(Frames.Delta) FROM Frames WHERE Events.Id = Frames.EventId)' ), - //'Path' => array( 'postFunc' => 'getEventPath' ), ), ), 'frames' => array( diff --git a/web/api/app/Controller/EventsController.php b/web/api/app/Controller/EventsController.php index 3f3c3c8b4..446285729 100644 --- a/web/api/app/Controller/EventsController.php +++ b/web/api/app/Controller/EventsController.php @@ -123,8 +123,8 @@ class EventsController extends AppController { $options = array('conditions' => array(array('Event.' . $this->Event->primaryKey => $id), $mon_options)); $event = $this->Event->find('first', $options); - $path = $configs['ZM_DIR_EVENTS'].'/'.$this->Image->getEventPath($event).'/'; - $event['Event']['BasePath'] = $path; + //$path = $configs['ZM_DIR_EVENTS'].'/'.$this->Image->getEventPath($event).'/'; + //$event['Event']['BasePath'] = $path; # Get the previous and next events for any monitor $this->Event->id = $id; diff --git a/web/includes/Event.php b/web/includes/Event.php index 0c9a15b87..c8dffec92 100644 --- a/web/includes/Event.php +++ b/web/includes/Event.php @@ -86,8 +86,10 @@ class Event { public function Relative_Path() { $event_path = ''; - if ( ZM_USE_DEEP_STORAGE ) { + if ( $this->{'Scheme'} == 'Deep' ) { $event_path = $this->{'MonitorId'} .'/'.strftime( '%y/%m/%d/%H/%M/%S', $this->Time()) ; + } else if ( $this->{'Scheme'} eq 'Medium' ) { + $event_path = $this->{'MonitorId'} .'/'.strftime( '%y-%m-%d', $this->Time() ) . '/'.$this->{'Id'}; } else { $event_path = $this->{'MonitorId'} .'/'.$this->{'Id'}; } @@ -96,7 +98,7 @@ class Event { } // end function Relative_Path() public function Link_Path() { - if ( ZM_USE_DEEP_STORAGE ) { + if ( $this->{'Scheme'} == 'Deep' ) { return $this->{'MonitorId'} .'/'.strftime( '%y/%m/%d/.', $this->Time()).$this->{'Id'}; } Error('Calling Link_Path when not using deep storage'); @@ -109,7 +111,7 @@ class Event { if ( !ZM_OPT_FAST_DELETE ) { dbQuery( 'DELETE FROM Stats WHERE EventId = ?', array($this->{'Id'}) ); dbQuery( 'DELETE FROM Frames WHERE EventId = ?', array($this->{'Id'}) ); - if ( ZM_USE_DEEP_STORAGE ) { + if ( $this->{'Scheme'} == 'Deep' ) { # Assumption: All events have a start time $start_date = date_parse( $this->{'StartTime'} ); diff --git a/web/includes/Frame.php b/web/includes/Frame.php index 7575aeed8..95a949365 100644 --- a/web/includes/Frame.php +++ b/web/includes/Frame.php @@ -49,34 +49,6 @@ class Frame { } } - public function Path() { - $Storage = $this->Storage(); - return $Storage->Path().'/'.$this->Relative_Path(); - } - public function Relative_Path() { - $event_path = ""; - - if ( ZM_USE_DEEP_STORAGE ) - { - $event_path = - $this->{'MonitorId'} - .'/'.strftime( "%y/%m/%d/%H/%M/%S", - $this->Time() - ) - ; - } - else - { - $event_path = - $this->{'MonitorId'} - .'/'.$this->{'Id'} - ; - } - - return( $event_path ); - - } - public function getImageSrc( $show='capture' ) { return $_SERVER['PHP_SELF'].'?view=image&fid='.$this->{'FrameId'}.'&eid='.$this->{'EventId'}.'&show='.$show; diff --git a/web/includes/functions.php b/web/includes/functions.php index 9dfedfbad..0b1d3385b 100644 --- a/web/includes/functions.php +++ b/web/includes/functions.php @@ -465,14 +465,6 @@ function canEdit( $area, $mid=false ) { return( $user[$area] == 'Edit' && ( !$mid || visibleMonitor( $mid ) ) ); } -function getEventPath( $event ) { - if ( ZM_USE_DEEP_STORAGE ) - $eventPath = $event['MonitorId'].'/'.strftime( '%y/%m/%d/%H/%M/%S', strtotime($event['StartTime']) ); - else - $eventPath = $event['MonitorId'].'/'.$event['Id']; - return( $eventPath ); -} - function getEventDefaultVideoPath( $event ) { $Event = new Event( $event ); return $Event->getStreamSrc( array( 'mode'=>'mpeg', 'format'=>'h264' ) ); diff --git a/web/skins/classic/views/storage.php b/web/skins/classic/views/storage.php index 067881298..d74ea9079 100644 --- a/web/skins/classic/views/storage.php +++ b/web/skins/classic/views/storage.php @@ -33,9 +33,15 @@ if ( $_REQUEST['id'] ) { $newStorage['Name'] = translate('NewStorage'); $newStorage['Path'] = ''; $newStorage['Type'] = 'local'; + $newStorage['Scheme'] = 'Medium'; } $type_options = array( 'local' => translate('Local'), 's3fs' => translate('s3fs') ); +$scheme_options = array( + 'Deep' => translate('Deep'), + 'Medium' => translate('Medium'), + 'Shallow' => translate('Shallow'), +); $focusWindow = true; @@ -65,12 +71,15 @@ xhtmlHeaders(__FILE__, translate('Storage')." - ".$newStorage['Name'] ); + + + +
- - - + +