Merge branch 'master' into storageareas

This commit is contained in:
Isaac Connor 2019-08-07 15:51:21 -04:00
commit b23a1d0955
3 changed files with 56 additions and 53 deletions

View File

@ -573,55 +573,61 @@ sub CopyTo {
my $moved = 0; my $moved = 0;
if ( $$NewStorage{Type} eq 's3fs' ) { if ( $$NewStorage{Type} eq 's3fs' ) {
my ( $aws_id, $aws_secret, $aws_host, $aws_bucket ) = ( $$NewStorage{Url} =~ /^\s*([^:]+):([^@]+)@([^\/]*)\/(.+)\s*$/ ); if ( $$NewStorage{Url} ) {
eval { my ( $aws_id, $aws_secret, $aws_host, $aws_bucket ) = ( $$NewStorage{Url} =~ /^\s*([^:]+):([^@]+)@([^\/]*)\/(.+)\s*$/ );
require Net::Amazon::S3; if ( $aws_id and $aws_secret and $aws_host and $aws_bucket ) {
require File::Slurp; eval {
my $s3 = Net::Amazon::S3->new( { require Net::Amazon::S3;
aws_access_key_id => $aws_id, require File::Slurp;
aws_secret_access_key => $aws_secret, my $s3 = Net::Amazon::S3->new( {
( $aws_host ? ( host => $aws_host ) : () ), aws_access_key_id => $aws_id,
}); aws_secret_access_key => $aws_secret,
my $bucket = $s3->bucket($aws_bucket); ( $aws_host ? ( host => $aws_host ) : () ),
if ( ! $bucket ) { });
Error("S3 bucket $bucket not found."); my $bucket = $s3->bucket($aws_bucket);
die; if ( !$bucket ) {
Error("S3 bucket $bucket not found.");
die;
}
my $event_path = $self->RelativePath();
Debug("Making directory $event_path/");
if ( ! $bucket->add_key($event_path.'/', '') ) {
die "Unable to add key for $event_path/";
}
my @files = glob("$OldPath/*");
Debug("Files to move @files");
foreach my $file ( @files ) {
next if $file =~ /^\./;
( $file ) = ( $file =~ /^(.*)$/ ); # De-taint
my $starttime = [gettimeofday];
Debug("Moving file $file to $NewPath");
my $size = -s $file;
if ( ! $size ) {
Info('Not moving file with 0 size');
}
my $file_contents = File::Slurp::read_file($file);
if ( ! $file_contents ) {
die 'Loaded empty file, but it had a size. Giving up';
}
my $filename = $event_path.'/'.File::Basename::basename($file);
if ( ! $bucket->add_key($filename, $file_contents) ) {
die "Unable to add key for $filename";
}
my $duration = tv_interval($starttime);
Debug('PUT to S3 ' . Number::Bytes::Human::format_bytes($size) . " in $duration seconds = " . Number::Bytes::Human::format_bytes($duration?$size/$duration:$size) . '/sec');
} # end foreach file.
$moved = 1;
};
Error($@) if $@;
} else {
Error("Unable to parse S3 Url into it's component parts.");
} }
#die $@ if $@;
my $event_path = 'events/'.$self->RelativePath(); } # end if Url
Debug("Making directory $event_path/");
if ( ! $bucket->add_key( $event_path.'/','' ) ) {
die "Unable to add key for $event_path/";
}
my @files = glob("$OldPath/*");
Debug("Files to move @files");
for my $file (@files) {
next if $file =~ /^\./;
( $file ) = ( $file =~ /^(.*)$/ ); # De-taint
my $starttime = [gettimeofday];
Debug("Moving file $file to $NewPath");
my $size = -s $file;
if ( ! $size ) {
Info('Not moving file with 0 size');
}
my $file_contents = File::Slurp::read_file($file);
if ( ! $file_contents ) {
die 'Loaded empty file, but it had a size. Giving up';
}
my $filename = $event_path.'/'.File::Basename::basename($file);
if ( ! $bucket->add_key($filename, $file_contents) ) {
die "Unable to add key for $filename";
}
my $duration = tv_interval($starttime);
Debug('PUT to S3 ' . Number::Bytes::Human::format_bytes($size) . " in $duration seconds = " . Number::Bytes::Human::format_bytes($duration?$size/$duration:$size) . '/sec');
} # end foreach file.
$moved = 1;
};
Error($@) if $@;
die $@ if $@;
} # end if s3 } # end if s3
my $error = ''; my $error = '';

View File

@ -114,7 +114,7 @@ class Filter extends ZM_Object {
} }
public function control($command, $server_id=null) { public function control($command, $server_id=null) {
$Servers = $server_id ? Server::find(array('Id'=>$server_id)) : Server::find(); $Servers = $server_id ? Server::find(array('Id'=>$server_id)) : Server::find(array('Status'=>'Running'));
if ( !count($Servers) and !$server_id ) { if ( !count($Servers) and !$server_id ) {
# This will be the non-multi-server case # This will be the non-multi-server case
$Servers = array(new Server()); $Servers = array(new Server());
@ -124,7 +124,7 @@ class Filter extends ZM_Object {
if ( !defined('ZM_SERVER_ID') or !$Server->Id() or ZM_SERVER_ID==$Server->Id() ) { if ( !defined('ZM_SERVER_ID') or !$Server->Id() or ZM_SERVER_ID==$Server->Id() ) {
# Local # Local
Logger::Debug("Controlling filter locally $command for server ".$Server->Id()); Logger::Debug("Controlling filter locally $command for server ".$Server->Id());
daemonControl($command, 'zmfilter.pl', '--filter_id='.$this->{'Id'}); daemonControl($command, 'zmfilter.pl', '--filter_id='.$this->{'Id'}.' --daemon');
} else { } else {
# Remote case # Remote case

View File

@ -45,9 +45,6 @@ if ( !$filter ) {
$filter = new ZM\Filter(); $filter = new ZM\Filter();
} }
ZM\Logger::Debug("Query: " . $filter->Query_json());
ZM\Logger::Debug("Query: " . print_r($filter->Query(), true));
if ( isset($_REQUEST['filter']) ) { if ( isset($_REQUEST['filter']) ) {
# Update our filter object with whatever changes we have made before saving # Update our filter object with whatever changes we have made before saving
#$filter->set($_REQUEST['filter']); #$filter->set($_REQUEST['filter']);