ZaneYork
/
zoneminder
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'max_queue_size'

This commit is contained in:
Isaac Connor 2021-03-26 17:03:52 -04:00
parent 1275948744 3d13daaf25
commit 1d99bd5cb3
13 changed files with 143 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
db/zm_create.sql.in
View File

@ -497,11 +497,12 @@ CREATE TABLE `Monitors` (
`LabelX` smallint(5) unsigned NOT NULL default '0',
`LabelY` smallint(5) unsigned NOT NULL default '0',
`LabelSize` smallint(5) unsigned NOT NULL DEFAULT '1',
`ImageBufferCount` smallint(5) unsigned NOT NULL default '100',
`WarmupCount` smallint(5) unsigned NOT NULL default '25',
`ImageBufferCount` smallint(5) unsigned NOT NULL default '3',
`MaxImageBufferCount` smallint(5) unsigned NOT NULL default '0',
`WarmupCount` smallint(5) unsigned NOT NULL default '0',
`PreEventCount` smallint(5) unsigned NOT NULL default '10',
`PostEventCount` smallint(5) unsigned NOT NULL default '10',
`StreamReplayBuffer` int(10) unsigned NOT NULL default '1000',
`StreamReplayBuffer` int(10) unsigned NOT NULL default '0',
`AlarmFrameCount` smallint(5) unsigned NOT NULL default '1',
`SectionLength` int(10) unsigned NOT NULL default '600',
`MinSectionLength` int(10) unsigned NOT NULL default '10',

19
db/zm_update-1.35.22.sql Normal file
View File

@ -0,0 +1,19 @@
--
-- Add MaxImageBufferCount, set it to ImageBufferCount if that was large and set ImageBufferCount to 3
--
SET @s = (SELECT IF(
(SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE table_schema = DATABASE()
AND table_name = 'Monitors'
AND column_name = 'MaxImageBufferCount'
) > 0,
"SELECT 'Column MaxImageBufferCount already exists in Monitors'",
"ALTER TABLE `Monitors` ADD `MaxImageBufferCount` smallint(5) unsigned NOT NULL default '0' AFTER `ImageBufferCount`"
));
PREPARE stmt FROM @s;
EXECUTE stmt;
UPDATE `Monitors` SET MaxImageBufferCount=ImageBufferCount WHERE ImageBufferCount >= 20;
UPDATE `Monitors` SET ImageBufferCount = 3;

2
distros/redhat/zoneminder.spec
View File

@ -28,7 +28,7 @@
%global _hardened_build 1
Name: zoneminder
Version: 1.35.21
Version: 1.35.22
Release: 1%{?dist}
Summary: A camera monitoring and analysis tool
Group: System Environment/Daemons

17
src/zm_monitor.cpp
View File

@ -85,7 +85,7 @@ std::string load_monitor_sql =
"`RecordAudio`, "
"`Brightness`, `Contrast`, `Hue`, `Colour`, "
"`EventPrefix`, `LabelFormat`, `LabelX`, `LabelY`, `LabelSize`,"
"`ImageBufferCount`, `WarmupCount`, `PreEventCount`, `PostEventCount`, `StreamReplayBuffer`, `AlarmFrameCount`, "
"`ImageBufferCount`, `MaxImageBufferCount`, `WarmupCount`, `PreEventCount`, `PostEventCount`, `StreamReplayBuffer`, `AlarmFrameCount`, "
"`SectionLength`, `MinSectionLength`, `FrameSkip`, `MotionFrameSkip`, "
"`FPSReportInterval`, `RefBlendPerc`, `AlarmRefBlendPerc`, `TrackMotion`, `Exif`,"
"`RTSPServer`, `RTSPStreamName`,"
@ -327,6 +327,7 @@ Monitor::Monitor()
label_coord(Coord(0,0)),
label_size(0),
image_buffer_count(0),
max_image_buffer_count(0),
warmup_count(0),
pre_event_count(0),
post_event_count(0),
@ -431,7 +432,7 @@ Monitor::Monitor()
"RecordAudio, "
"Brightness, Contrast, Hue, Colour, "
"EventPrefix, LabelFormat, LabelX, LabelY, LabelSize,"
"ImageBufferCount, WarmupCount, PreEventCount, PostEventCount, StreamReplayBuffer, AlarmFrameCount, "
"ImageBufferCount, `MaxImageBufferCount`, WarmupCount, PreEventCount, PostEventCount, StreamReplayBuffer, AlarmFrameCount, "
"SectionLength, MinSectionLength, FrameSkip, MotionFrameSkip, "
"FPSReportInterval, RefBlendPerc, AlarmRefBlendPerc, TrackMotion, Exif,"
"`RTSPServer`,`RTSPStreamName`,
@ -579,10 +580,14 @@ void Monitor::Load(MYSQL_ROW dbrow, bool load_zones=true, Purpose p = QUERY) {
label_size = atoi(dbrow[col]); col++;
image_buffer_count = atoi(dbrow[col]); col++;
max_image_buffer_count = atoi(dbrow[col]); col++;
warmup_count = atoi(dbrow[col]); col++;
pre_event_count = atoi(dbrow[col]); col++;
packetqueue.setMaxVideoPackets(pre_event_count);
packetqueue.setPreEventVideoPackets(pre_event_count);
packetqueue.setMaxVideoPackets(max_image_buffer_count);
packetqueue.setKeepKeyframes(videowriter == PASSTHROUGH);
post_event_count = atoi(dbrow[col]); col++;
stream_replay_buffer = atoi(dbrow[col]); col++;
alarm_frame_count = atoi(dbrow[col]); col++;
@ -590,7 +595,6 @@ void Monitor::Load(MYSQL_ROW dbrow, bool load_zones=true, Purpose p = QUERY) {
alarm_frame_count = 1;
else if ( alarm_frame_count > MAX_PRE_ALARM_FRAMES )
alarm_frame_count = MAX_PRE_ALARM_FRAMES;
pre_event_buffer_count = pre_event_count + alarm_frame_count + warmup_count - 1;
section_length = atoi(dbrow[col]); col++;
min_section_length = atoi(dbrow[col]); col++;
@ -639,9 +643,9 @@ void Monitor::Load(MYSQL_ROW dbrow, bool load_zones=true, Purpose p = QUERY) {
// Should maybe store this for later use
std::string monitor_dir = stringtf("%s/%d", storage->Path(), id);
LoadCamera();
if ( purpose != QUERY ) {
LoadCamera();
Zone **zones = 0;
int n_zones = Zone::Load(this, zones);
this->AddZones(n_zones, zones);
@ -676,7 +680,6 @@ void Monitor::Load(MYSQL_ROW dbrow, bool load_zones=true, Purpose p = QUERY) {
}
} // end if purpose
Debug(1, "Loaded monitor %d(%s), %d zones", id, name, n_zones);
} // Monitor::Load
@ -3125,7 +3128,7 @@ int Monitor::PrimeCapture() {
}
Debug(2, "Video stream id is %d, audio is %d, minimum_packets to keep in buffer %d",
video_stream_id, audio_stream_id, pre_event_buffer_count);
video_stream_id, audio_stream_id, pre_event_count);
if (rtsp_server) {
if (video_stream_id >= 0) {

5
src/zm_monitor.h
View File

@ -292,9 +292,8 @@ protected:
char label_format[64]; // The format of the timestamp on the images
Coord label_coord; // The coordinates of the timestamp on the images
int label_size; // Size of the timestamp on the images
int32_t image_buffer_count; // Size of circular image buffer, at least twice the size of the pre_event_count
int pre_event_buffer_count; // Size of dedicated circular pre event buffer used when analysis is not performed at capturing framerate,
// value is pre_event_count + alarm_frame_count - 1
int32_t image_buffer_count; // Size of circular image buffer, kept in /dev/shm
int32_t max_image_buffer_count; // Max # of video packets to keep in packet queue
int warmup_count; // How many images to process before looking for events
int pre_event_count; // How many images to hold and prepend to an alarm event
int post_event_count; // How many unalarmed images must occur before the alarm state is reset

91
src/zm_packetqueue.cpp
View File

@ -29,6 +29,7 @@
PacketQueue::PacketQueue():
video_stream_id(-1),
max_video_packet_count(-1),
pre_event_video_packet_count(-1),
max_stream_id(-1),
packet_counts(nullptr),
deleting(false),
@ -84,7 +85,50 @@ bool PacketQueue::queuePacket(ZMPacket* add_packet) {
Debug(4, "No video keyframe so no one needs us to queue packets.");
return false;
}
mutex.lock();
{
std::unique_lock<std::mutex> lck(mutex);
if (add_packet->packet.stream_index == video_stream_id) {
if ((max_video_packet_count > 0) and (packet_counts[video_stream_id] > max_video_packet_count)) {
Warning("You have set the video packets in the queue to %d. The queue is full. Either Analysis is not keeping up or your camera's keyframe interval is larger than this setting. We are dropping packets.");
if (add_packet->keyframe) {
// Have a new keyframe, so delete everything
while ((*pktQueue.begin() != add_packet) and (packet_counts[video_stream_id] > max_video_packet_count)) {
ZMPacket *zm_packet = *pktQueue.begin();
ZMLockedPacket *lp = new ZMLockedPacket(zm_packet);
if (!lp->trylock()) {
Debug(1, "Found locked packet when trying to free up video packets. Can't continue");
//delete lp;
break;
}
delete lp;
for (
std::list<packetqueue_iterator *>::iterator iterators_it = iterators.begin();
iterators_it != iterators.end();
++iterators_it
) {
packetqueue_iterator *iterator_it = *iterators_it;
// Have to check each iterator and make sure it doesn't point to the packet we are about to delete
if ( *(*iterator_it) == zm_packet ) {
Debug(1, "Bumping IT because it is at the front that we are deleting");
++(*iterators_it);
}
} // end foreach iterator
pktQueue.pop_front();
packet_counts[zm_packet->packet.stream_index] -= 1;
Debug(1, "Deleting a packet with stream index:%d image_index:%d with keyframe:%d, video frames in queue:%d max: %d, queuesize:%d",
zm_packet->packet.stream_index, zm_packet->image_index, zm_packet->keyframe, packet_counts[video_stream_id], max_video_packet_count, pktQueue.size());
delete zm_packet;
} // end while
}
} // end if too many video packets
if ((max_video_packet_count > 0) and (packet_counts[video_stream_id] > max_video_packet_count)) {
Error("Unable to free up older packets. Not queueing this video packet.");
return false;
}
} // end if this packet is a video packet
pktQueue.push_back(add_packet);
packet_counts[add_packet->packet.stream_index] += 1;
@ -103,7 +147,7 @@ bool PacketQueue::queuePacket(ZMPacket* add_packet) {
--(*iterator_it);
}
} // end foreach iterator
mutex.unlock();
} // end lock scope
// We signal on every packet because someday we may analyze sound
Debug(4, "packetqueue queuepacket, unlocked signalling");
condition.notify_all();
@ -126,13 +170,13 @@ void PacketQueue::clearPackets(ZMPacket *add_packet) {
and
add_packet->keyframe
and
(packet_counts[video_stream_id] > max_video_packet_count)
(packet_counts[video_stream_id] > pre_event_video_packet_count)
and
*(pktQueue.begin()) != add_packet
)
) {
Debug(3, "stream index %d ?= video_stream_id %d, keyframe %d, keep_keyframes %d, counts %d > max %d at begin %d",
add_packet->packet.stream_index, video_stream_id, add_packet->keyframe, keep_keyframes, packet_counts[video_stream_id], max_video_packet_count,
Debug(3, "stream index %d ?= video_stream_id %d, keyframe %d, keep_keyframes %d, counts %d > pre_event_count %d at begin %d",
add_packet->packet.stream_index, video_stream_id, add_packet->keyframe, keep_keyframes, packet_counts[video_stream_id], pre_event_video_packet_count,
( *(pktQueue.begin()) != add_packet )
);
return;
@ -155,7 +199,7 @@ void PacketQueue::clearPackets(ZMPacket *add_packet) {
if (!keep_keyframes) {
// If not doing passthrough, we don't care about starting with a keyframe so logic is simpler
while ((*pktQueue.begin() != add_packet) and (packet_counts[video_stream_id] > max_video_packet_count + tail_count)) {
while ((*pktQueue.begin() != add_packet) and (packet_counts[video_stream_id] > pre_event_video_packet_count + tail_count)) {
ZMPacket *zm_packet = *pktQueue.begin();
ZMLockedPacket *lp = new ZMLockedPacket(zm_packet);
if (!lp->trylock()) break;
@ -164,7 +208,7 @@ void PacketQueue::clearPackets(ZMPacket *add_packet) {
pktQueue.pop_front();
packet_counts[zm_packet->packet.stream_index] -= 1;
Debug(1, "Deleting a packet with stream index:%d image_index:%d with keyframe:%d, video frames in queue:%d max: %d, queuesize:%d",
zm_packet->packet.stream_index, zm_packet->image_index, zm_packet->keyframe, packet_counts[video_stream_id], max_video_packet_count, pktQueue.size());
zm_packet->packet.stream_index, zm_packet->image_index, zm_packet->keyframe, packet_counts[video_stream_id], pre_event_video_packet_count, pktQueue.size());
delete zm_packet;
} // end while
return;
@ -201,7 +245,7 @@ void PacketQueue::clearPackets(ZMPacket *add_packet) {
++video_packets_to_delete;
Debug(4, "Counted %d video packets. Which would leave %d in packetqueue tail count is %d",
video_packets_to_delete, packet_counts[video_stream_id]-video_packets_to_delete, tail_count);
if (packet_counts[video_stream_id] - video_packets_to_delete <= max_video_packet_count + tail_count) {
if (packet_counts[video_stream_id] - video_packets_to_delete <= pre_event_video_packet_count + tail_count) {
break;
}
}
@ -221,7 +265,7 @@ void PacketQueue::clearPackets(ZMPacket *add_packet) {
}
Debug(1, "Deleting a packet with stream index:%d image_index:%d with keyframe:%d, video frames in queue:%d max: %d, queuesize:%d",
zm_packet->packet.stream_index, zm_packet->image_index, zm_packet->keyframe, packet_counts[video_stream_id], max_video_packet_count, pktQueue.size());
zm_packet->packet.stream_index, zm_packet->image_index, zm_packet->keyframe, packet_counts[video_stream_id], pre_event_video_packet_count, pktQueue.size());
pktQueue.pop_front();
packet_counts[zm_packet->packet.stream_index] -= 1;
delete zm_packet;
@ -238,7 +282,7 @@ ZMLockedPacket* PacketQueue::popPacket( ) {
return nullptr;
}
Debug(4, "poPacket Mutex locking");
mutex.lock();
std::unique_lock<std::mutex> lck(mutex);
ZMPacket *zm_packet = pktQueue.front();
for (
@ -260,8 +304,6 @@ ZMLockedPacket* PacketQueue::popPacket( ) {
pktQueue.pop_front();
packet_counts[zm_packet->packet.stream_index] -= 1;
mutex.unlock();
return lp;
} // popPacket
@ -285,7 +327,7 @@ unsigned int PacketQueue::clear(unsigned int frames_to_keep, int stream_id) {
return 0;
}
Debug(5, "Locking in clear");
mutex.lock();
std::unique_lock<std::mutex> lck(mutex);
packetqueue_iterator it = pktQueue.end()--; // point to last element instead of end
ZMPacket *zm_packet = nullptr;
@ -340,13 +382,7 @@ unsigned int PacketQueue::clear(unsigned int frames_to_keep, int stream_id) {
delete_count += 1;
} // while our iterator is not the first packet
zm_packet = nullptr; // tidy up for valgrind
Debug(3, "Deleted %d packets, %d remaining", delete_count, pktQueue.size());
mutex.unlock();
return delete_count;
Debug(3, "Deleted packets, resulting size is %d", pktQueue.size());
mutex.unlock();
return delete_count;
} // end unsigned int PacketQueue::clear( unsigned int frames_to_keep, int stream_id )
@ -389,7 +425,7 @@ unsigned int PacketQueue::clear(struct timeval *duration, int streamId) {
return 0;
}
Debug(4, "Locking in clear");
mutex.lock();
std::unique_lock<std::mutex> lck(mutex);
struct timeval keep_from;
std::list<ZMPacket *>::reverse_iterator it = pktQueue.rbegin();
@ -440,7 +476,6 @@ unsigned int PacketQueue::clear(struct timeval *duration, int streamId) {
}
if ( it == pktQueue.rend() ) {
Debug(1, "Didn't find a keyframe before event starttime. keeping all" );
mutex.unlock();
return 0;
}
@ -462,14 +497,10 @@ unsigned int PacketQueue::clear(struct timeval *duration, int streamId) {
} // end foreach iterator
pktQueue.pop_front();
packet_counts[zm_packet->packet.stream_index] -= 1;
//if ( zm_packet->image_index == -1 )
delete zm_packet;
deleted_frames += 1;
}
zm_packet = nullptr;
Debug(3, "Deleted %d frames", deleted_frames);
mutex.unlock();
return deleted_frames;
}
@ -706,7 +737,13 @@ bool PacketQueue::is_there_an_iterator_pointing_to_packet(ZMPacket *zm_packet) {
void PacketQueue::setMaxVideoPackets(int p) {
max_video_packet_count = p;
Debug(1, "Setting max_video_packet_count to %d", p);
if ( max_video_packet_count < 1 )
max_video_packet_count = 1 ;
if ( max_video_packet_count < 0 )
max_video_packet_count = 0 ;
}
void PacketQueue::setPreEventVideoPackets(int p) {
pre_event_video_packet_count = p;
Debug(1, "Setting pre_event_video_packet_count to %d", p);
if ( pre_event_video_packet_count < 1 )
pre_event_video_packet_count = 1;
// We can simplify a lot of logic in queuePacket if we can assume at least 1 packet in queue
}

3
src/zm_packetqueue.h
View File

@ -35,6 +35,8 @@ class PacketQueue {
int video_stream_id;
int max_video_packet_count; // allow a negative value to someday mean unlimited
// This is now a hard limit on the # of video packets to keep in the queue so that we can limit ram
int pre_event_video_packet_count; // Was max_video_packet_count
int max_stream_id;
int *packet_counts; /* packet count for each stream_id, to keep track of how many video vs audio packets are in the queue */
bool deleting;
@ -52,6 +54,7 @@ class PacketQueue {
int addStream();
void setMaxVideoPackets(int p);
void setPreEventVideoPackets(int p);
void setKeepKeyframes(bool k) { keep_keyframes = k; };
bool queuePacket(ZMPacket* packet);

2
version
View File

@ -1 +1 @@
1.35.21
1.35.22

1
web/includes/Monitor.php
View File

@ -84,6 +84,7 @@ class Monitor extends ZM_Object {
'LabelY' => 0,
'LabelSize' => 1,
'ImageBufferCount' => 3,
'MaxImageBufferCount' => 0,
'WarmupCount' => 0,
'PreEventCount' => 5,
'PostEventCount' => 5,

14
web/lang/en_gb.php
View File

@ -441,6 +441,7 @@ $SLANG = array(
'Idle' => 'Idle',
'Ignore' => 'Ignore',
'ImageBufferSize' => 'Image Buffer Size (frames)',
'MaxImageBufferCount' => 'Maximum Image Buffer Size (frames)',
'Image' => 'Image',
'Images' => 'Images',
'Include' => 'Include',
@ -1137,6 +1138,19 @@ $OLANG = array(
optionally choose to not decode the H264/H265 packets. This will drastically reduce cpu use
but will make live view unavailable for this monitor.'
),
'ImageBufferCount' => array(
'Help' => '
Number of raw images available in /dev/shm. Currently should be set in the 3-5 range. Used for live viewing.'
),
'MaxImageBufferCount' => array(
'Help' => '
Maximum number of video packets that will be held in the packet queue.
The packetqueue will normally manage itself, keeping Pre Event Count frames or all since last keyframe if using
passthrough mode. You can set a maximum to prevent the monitor from consuming too much ram, but your events might
not have all the frames they should if your keyframe interval is larger than this value.
You will get errors in your logs about this. So make sure your keyframe interval is low or you have enough ram.
'
),
// 'LANG_DEFAULT' => array(
// 'Prompt' => "This is a new prompt for this option",

1
web/skins/classic/css/base/views/monitor.css
View File

@ -29,6 +29,7 @@ input[name="newMonitor[Refresh]"],
input[name="newMonitor[LabelX]"],
input[name="newMonitor[LabelY]"],
input[name="newMonitor[ImageBufferCount]"],
input[name="newMonitor[MaxImageBufferCount]"],
input[name="newMonitor[WarmupCount]"],
input[name="newMonitor[PreEventCount]"],
input[name="newMonitor[PostEventCount]"],

13
web/skins/classic/views/js/monitor.js
View File

@ -48,7 +48,6 @@ function updateMonitorDimensions(element) {
form.elements['newMonitor[Height]'].value = dimensions[1];
}
}
update_estimated_ram_use();
return false;
}
@ -140,9 +139,6 @@ function initPage() {
form.submit();
};
});
document.querySelectorAll('input[name="newMonitor[ImageBufferCount]"],input[name="newMonitor[Width]"],input[name="newMonitor[Height]"]').forEach(function(el) {
el.oninput = window['update_estimated_ram_use'].bind(el);
});
document.querySelectorAll('select[name="newMonitor[Function]"]').forEach(function(el) {
el.onchange = function() {
@ -269,15 +265,6 @@ function random_WebColour() {
);
}
function update_estimated_ram_use() {
var buffer_count = document.querySelectorAll('input[name="newMonitor[ImageBufferCount]"]')[0].value;
var width = document.querySelectorAll('input[name="newMonitor[Width]"]')[0].value;
var height = document.querySelectorAll('input[name="newMonitor[Height]"]')[0].value;
var colours = document.querySelectorAll('select[name="newMonitor[Colours]"]')[0].value;
document.getElementById('estimated_ram_use').innerHTML = human_filesize(buffer_count * width * height * colours, 0);
}
function updateLatitudeAndLongitude(latitude, longitude) {
var form = document.getElementById('contentForm');
form.elements['newMonitor[Latitude]'].value = latitude;

10
web/skins/classic/views/monitor.php
View File

@ -1059,9 +1059,13 @@ echo htmlSelect('newMonitor[OutputContainer]', $videowriter_containers, $monitor
{
?>
<tr>
<td class="text-right pr-3"><?php echo translate('ImageBufferSize') ?></td>
<td class="text-right pr-3"><?php echo translate('ImageBufferSize'); echo makeHelpLink('ImageBufferCount'); ?></td>
<td><input type="number" name="newMonitor[ImageBufferCount]" value="<?php echo validHtmlStr($monitor->ImageBufferCount()) ?>" min="1"/></td>
</tr>
<tr>
<td class="text-right pr-3"><?php echo translate('MaxImageBufferCount'); echo makeHelpLink('MaxImageBufferCount'); ?></td>
<td><input type="number" name="newMonitor[MaxImageBufferCount]" value="<?php echo validHtmlStr($monitor->MaxImageBufferCount()) ?>" min="0"/></td>
</tr>
<tr>
<td class="text-right pr-3"><?php echo translate('WarmupFrames') ?></td>
<td><input type="number" name="newMonitor[WarmupCount]" value="<?php echo validHtmlStr($monitor->WarmupCount()) ?>" min="0"/></td>
@ -1082,10 +1086,6 @@ echo htmlSelect('newMonitor[OutputContainer]', $videowriter_containers, $monitor
<td class="text-right pr-3"><?php echo translate('AlarmFrameCount') ?></td>
<td><input type="number" name="newMonitor[AlarmFrameCount]" value="<?php echo validHtmlStr($monitor->AlarmFrameCount()) ?>" min="1"/></td>
</tr>
<tr>
<td class="text-right pr-3"><?php echo translate('Estimated Ram Use') ?></td>
<td id="estimated_ram_use"><?php echo human_filesize($monitor->ImageBufferCount() * $monitor->Width() * $monitor->Height() * $monitor->Colours(), 0) ?></td>
</tr>
<?php
break;
}