From 4deb3a8d845cd407538ed0ef4f46169493810e94 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 23 Nov 2020 19:31:21 -0500 Subject: [PATCH 1/5] escape the word Groups --- db/zm_update-1.31.5.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/zm_update-1.31.5.sql b/db/zm_update-1.31.5.sql index 859487e88..d315c8e84 100644 --- a/db/zm_update-1.31.5.sql +++ b/db/zm_update-1.31.5.sql @@ -10,7 +10,7 @@ SET @s = (SELECT IF( AND column_name = 'ParentId' ) > 0, "SELECT 'Column GroupId exists in Groups'", -"ALTER TABLE Groups ADD `ParentId` int(10) unsigned AFTER `Name`" +"ALTER TABLE `Groups` ADD `ParentId` int(10) unsigned AFTER `Name`" )); PREPARE stmt FROM @s; From be27630a8508f6e0517fb6e3b70832cdc694c5f2 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Mon, 23 Nov 2020 19:33:53 -0500 Subject: [PATCH 2/5] escape the word Groups --- db/zm_update-1.31.7.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/zm_update-1.31.7.sql b/db/zm_update-1.31.7.sql index 1221d9adb..0afd76ce5 100644 --- a/db/zm_update-1.31.7.sql +++ b/db/zm_update-1.31.7.sql @@ -3,7 +3,7 @@ SET @s = (SELECT IF( AND table_name = 'Groups' AND column_name = 'MonitorIds' ) > 0, - "ALTER TABLE Groups MODIFY `MonitorIds` text NOT NULL", + "ALTER TABLE `Groups` MODIFY `MonitorIds` text NOT NULL", "SELECT 'Groups no longer has MonitorIds'" )); From ae7a706526d7b9f7fa7643da47fdfa983a6be891 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Tue, 1 Dec 2020 14:04:51 -0500 Subject: [PATCH 3/5] Handle minTime and maxTime being specified as either a timstamp or a datetime. Remove duplicated query building code. Reduce ram requirements when exporting logs. --- web/ajax/log.php | 127 +++++++++++++++++++---------------------------- 1 file changed, 50 insertions(+), 77 deletions(-) diff --git a/web/ajax/log.php b/web/ajax/log.php index 16f7ac573..5f749df6c 100644 --- a/web/ajax/log.php +++ b/web/ajax/log.php @@ -8,10 +8,6 @@ $filterFields = array('Component', 'ServerId', 'Pid', 'Level', 'File', 'Line'); function buildLogQuery($action) { global $filterFields; - $minTime = isset($_REQUEST['minTime']) ? $_REQUEST['minTime'] : NULL; - $maxTime = isset($_REQUEST['maxTime']) ? $_REQUEST['maxTime'] : NULL; - - $limit = 100; if ( isset($_REQUEST['limit']) ) { if ( ( !is_integer($_REQUEST['limit']) and !ctype_digit($_REQUEST['limit']) ) ) { ZM\Error('Invalid value for limit ' . $_REQUEST['limit']); @@ -33,11 +29,37 @@ function buildLogQuery($action) { $sql = $action.' FROM Logs'; $where = array(); $values = array(); + $limit = 100; + + $minTime = isset($_REQUEST['minTime']) ? $_REQUEST['minTime'] : NULL; + $maxTime = isset($_REQUEST['maxTime']) ? $_REQUEST['maxTime'] : NULL; + if ( !is_null($minTime) && !is_null($maxTime) && ($minTime > $maxTime) ) { + $tempTime = $minTime; + $minTime = $maxTime; + $maxTime = $tempTime; + } + if ( $minTime ) { - $where[] = 'TimeKey > ?'; + if ( preg_match('/^(\d+)(\.\d+)$/', $minTime) or preg_match('/^(\d+)$/', $minTime) ) { + # is a timestamp + } else if ( preg_match('/(.+)(\.\d+)/', $minTime, $matches) ) { + # This handles sub second precision in a date time + $minTime = strtotime($matches[1]).$matches[2]; + } else { + $minTime = strtotime($minTime); + } + $where[] = 'TimeKey >= ?'; $values[] = $minTime; - } elseif ( $maxTime ) { - $where[] = 'TimeKey < ?'; + } + if ( $maxTime ) { + if ( preg_match('/^(\d+)(\.\d+)$/', $maxTime) or preg_match('/^(\d+)$/', $maxTime) ) { + # is a timestamp + } else if ( preg_match('/(.+)(\.\d+)/', $maxTime, $matches) ) { + $maxTime = strtotime($matches[1]).$matches[2]; + } else { + $maxTime = strtotime($maxTime); + } + $where[] = 'TimeKey <= ?'; $values[] = $maxTime; } @@ -161,24 +183,7 @@ switch ( $_REQUEST['task'] ) { if ( !canView('System') ) ajaxError('Insufficient permissions to export logs'); - $minTime = isset($_POST['minTime']) ? $_POST['minTime'] : NULL; - $maxTime = isset($_POST['maxTime']) ? $_POST['maxTime'] : NULL; - if ( !is_null($minTime) && !is_null($maxTime) && ($minTime > $maxTime) ) { - $tempTime = $minTime; - $minTime = $maxTime; - $maxTime = $tempTime; - } - //$limit = isset($_POST['limit'])?$_POST['limit']:1000; - $filter = isset($_POST['filter'])?$_POST['filter']:array(); - $sortField = 'TimeKey'; - if ( isset($_POST['sortField']) ) { - if ( !in_array($_POST['sortField'], $filterFields) and ($_POST['sortField'] != 'TimeKey') ) { - ZM\Error('Invalid sort field '.$_POST['sortField']); - } else { - $sortField = $_POST['sortField']; - } - } - $sortOrder = (isset($_POST['sortOrder']) and $_POST['sortOrder']) == 'asc' ? 'asc' : 'desc'; + $query = buildLogQuery('SELECT *'); global $Servers; if ( !$Servers ) @@ -189,43 +194,6 @@ switch ( $_REQUEST['task'] ) { $servers_by_Id[$server->Id()] = $server; } - $sql = 'SELECT * FROM Logs'; - $where = array(); - $values = array(); - if ( $minTime ) { - if ( preg_match('/(.+)(\.\d+)/', $minTime, $matches) ) { - # This handles sub second precision - $minTime = strtotime($matches[1]).$matches[2]; - } else { - $minTime = strtotime($minTime); - } - $where[] = 'TimeKey >= ?'; - $values[] = $minTime; - } - if ( $maxTime ) { - if ( preg_match('/(.+)(\.\d+)/', $maxTime, $matches) ) { - $maxTime = strtotime($matches[1]).$matches[2]; - } else { - $maxTime = strtotime($maxTime); - } - $where[] = 'TimeKey <= ?'; - $values[] = $maxTime; - } - foreach ( $filter as $field=>$value ) { - if ( $value != '' ) { - if ( $field == 'Level' ) { - $where[] = $field.' <= ?'; - $values[] = $value; - } else { - $where[] = $field.' = ?'; - $values[] = $value; - } - } - } - if ( count($where) ) - $sql.= ' WHERE '.join(' AND ', $where); - $sql .= ' ORDER BY '.$sortField.' '.$sortOrder; - //$sql .= " limit ".dbEscape($limit); $format = isset($_POST['format']) ? $_POST['format'] : 'text'; switch ( $format ) { case 'text' : @@ -245,28 +213,27 @@ switch ( $_REQUEST['task'] ) { } $exportKey = substr(md5(rand()), 0, 8); $exportFile = 'zm-log.'.$exportExt; + $exportPath = ZM_DIR_EXPORTS.'/zm-log-'.$exportKey.'.'.$exportExt; // mkdir will generate a warning if it exists, but that is ok - error_reporting(0); + $old_error_reporting = error_reporting(0); if ( ! ( mkdir(ZM_DIR_EXPORTS) || file_exists(ZM_DIR_EXPORTS) ) ) { ZM\Fatal('Can\'t create exports dir at \''.ZM_DIR_EXPORTS.'\''); } - $exportPath = ZM_DIR_EXPORTS.'/zm-log-'.$exportKey.$exportExt; + error_reporting($old_error_reporting); + ZM\Logger::Debug("Exporting to $exportPath"); if ( !($exportFP = fopen($exportPath, 'w')) ) ZM\Fatal("Unable to open log export file $exportPath"); $logs = array(); - foreach ( dbFetchAll($sql, NULL, $values) as $log ) { - $log['DateTime'] = preg_replace('/^\d+/', strftime('%Y-%m-%d %H:%M:%S', intval($log['TimeKey'])), $log['TimeKey']); - $log['Server'] = ( $log['ServerId'] and isset($servers_by_Id[$log['ServerId']]) ) ? $servers_by_Id[$log['ServerId']]->Name() : ''; - $logs[] = $log; - } - ZM\Logger::Debug(count($logs).' lines being exported by '.$sql.implode(',', $values)); + $result = dbQuery($query['sql'], $query['values'], true); - switch( $format ) { + switch ( $format ) { case 'text' : { - foreach ( $logs as $log ) { + while ( $log = dbFetchNext($result) ) { + $log['DateTime'] = preg_replace('/^\d+/', strftime('%Y-%m-%d %H:%M:%S', intval($log['TimeKey'])), $log['TimeKey']); + $log['Server'] = ( $log['ServerId'] and isset($servers_by_Id[$log['ServerId']]) ) ? $servers_by_Id[$log['ServerId']]->Name() : ''; if ( $log['Line'] ) fprintf($exportFP, "%s %s[%d].%s-%s/%d [%s]\n", $log['DateTime'], $log['Component'], $log['Pid'], $log['Code'], $log['File'], $log['Line'], $log['Message']); @@ -289,7 +256,9 @@ switch ( $_REQUEST['task'] ) { translate('File'), translate('Line') )."\n"); - foreach ( $logs as $log ) { + while ( $log = dbFetchNext($result) ) { + $log['DateTime'] = preg_replace('/^\d+/', strftime('%Y-%m-%d %H:%M:%S', intval($log['TimeKey'])), $log['TimeKey']); + $log['Server'] = ( $log['ServerId'] and isset($servers_by_Id[$log['ServerId']]) ) ? $servers_by_Id[$log['ServerId']]->Name() : ''; fprintf($exportFP, "%s\t%s\t%s\t%d\t%s\t%s\t%s\t%s\n", $log['DateTime'], $log['Component'], $log['Server'], $log['Pid'], $log['Code'], $log['Message'], $log['File'], $log['Line']); } @@ -343,7 +312,9 @@ switch ( $_REQUEST['task'] ) { '.translate('DateTime').''.translate('Component').''.translate('Server').''.translate('Pid').''.translate('Level').''.translate('Message').''.translate('File').''.translate('Line').' '); - foreach ( $logs as $log ) { + while ( $log = dbFetchNext($result) ) { + $log['DateTime'] = preg_replace('/^\d+/', strftime('%Y-%m-%d %H:%M:%S', intval($log['TimeKey'])), $log['TimeKey']); + $log['Server'] = ( $log['ServerId'] and isset($servers_by_Id[$log['ServerId']]) ) ? $servers_by_Id[$log['ServerId']]->Name() : ''; $classLevel = $log['Level']; if ( $classLevel < ZM\Logger::FATAL ) { $classLevel = ZM\Logger::FATAL; @@ -387,7 +358,9 @@ switch ( $_REQUEST['task'] ) { ' ); - foreach ( $logs as $log ) { + while ( $log = dbFetchNext($result) ) { + $log['DateTime'] = preg_replace('/^\d+/', strftime('%Y-%m-%d %H:%M:%S', intval($log['TimeKey'])), $log['TimeKey']); + $log['Server'] = ( $log['ServerId'] and isset($servers_by_Id[$log['ServerId']]) ) ? $servers_by_Id[$log['ServerId']]->Name() : ''; fprintf( $exportFP, ' %s @@ -409,7 +382,7 @@ switch ( $_REQUEST['task'] ) { $exportExt = 'xml'; break; } - fclose( $exportFP ); + fclose($exportFP); ajaxResponse( array( 'key' => $exportKey, 'format' => $format, @@ -446,7 +419,7 @@ switch ( $_REQUEST['task'] ) { } $exportFile = 'zm-log.'.$exportExt; - $exportPath = ZM_DIR_EXPORTS.'/zm-log-'.$exportKey.$exportExt; + $exportPath = ZM_DIR_EXPORTS.'/zm-log-'.$exportKey.'.'.$exportExt; header('Pragma: public'); header('Expires: 0'); From baeb1dbd5b65cc2e5378554ba9239ba4a904d0e5 Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Tue, 1 Dec 2020 14:07:23 -0500 Subject: [PATCH 4/5] Take an optional debug param in dbQuery --- web/includes/database.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/web/includes/database.php b/web/includes/database.php index 2fb2db108..637b20fa0 100644 --- a/web/includes/database.php +++ b/web/includes/database.php @@ -125,7 +125,7 @@ function dbEscape( $string ) { return $dbConn->quote($string); } -function dbQuery($sql, $params=NULL) { +function dbQuery($sql, $params=NULL, $debug = false) { global $dbConn; if ( dbLog($sql, true) ) return; @@ -142,7 +142,7 @@ function dbQuery($sql, $params=NULL) { return NULL; } } else { - if ( defined('ZM_DB_DEBUG') ) { + if ( defined('ZM_DB_DEBUG') or $debug ) { ZM\Logger::Debug("SQL: $sql values:" . ($params?implode(',',$params):'')); } $result = $dbConn->query($sql); @@ -151,7 +151,7 @@ function dbQuery($sql, $params=NULL) { return NULL; } } - if ( defined('ZM_DB_DEBUG') ) { + if ( defined('ZM_DB_DEBUG') or $debug ) { if ( $params ) ZM\Logger::Debug("SQL: $sql " . implode(',',$params) . ' rows: '.$result->rowCount()); else From 48775e2230603841fb205be44dfd3fd3a881162d Mon Sep 17 00:00:00 2001 From: Isaac Connor Date: Tue, 1 Dec 2020 14:24:41 -0500 Subject: [PATCH 5/5] Fix Clear Log by using minTime and maxTime as timestamps. Fix export including unselected filters. Can't just serialize the form asit may contain fields in teh querystring so you get an array of values instead of a single value --- web/skins/classic/views/js/log.js | 50 ++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/web/skins/classic/views/js/log.js b/web/skins/classic/views/js/log.js index f0aee7c00..2ad01e8fc 100644 --- a/web/skins/classic/views/js/log.js +++ b/web/skins/classic/views/js/log.js @@ -1,5 +1,12 @@ var logParms = 'view=request&request=log&task=query'; -var logReq = new Request.JSON( {url: thisUrl, method: 'post', timeout: AJAX_TIMEOUT, link: 'cancel', onSuccess: logResponse} ); +var logReq = new Request.JSON( { + url: thisUrl, + method: 'post', + timeout: AJAX_TIMEOUT, + link: 'cancel', + onSuccess: logResponse +} ); + var logTimer = undefined; var logTable = undefined; @@ -171,18 +178,24 @@ function clearError() { function clearLog() { logReq.cancel(); + var clearReq = new Request.JSON({ + url: thisUrl, + method: 'post', + timeout: AJAX_TIMEOUT, + link: 'cancel', + onSuccess: clearResponse + }); var clearParms = 'view=request&request=log&task=delete'; - var clearReq = new Request.JSON({url: thisUrl, method: 'post', timeout: AJAX_TIMEOUT, link: 'cancel', onSuccess: clearResponse}); - var tbody = $(logTable).getElement('tbody'); - var rows = tbody.getElements('tr'); - if ( rows && rows.length ) { - var minTime = rows[0].getElement('td').get('text'); - clearParms += "&minTime="+encodeURIComponent(minTime); - var maxTime = rows[rows.length-1].getElement('td').get('text'); - clearParms += "&maxTime="+encodeURIComponent(maxTime); - } - var form = $('logForm'); - clearReq.send(clearParms+"&"+form.toQueryString()); + clearParms += "&minTime="+minLogTime; + clearParms += "&maxTime="+maxLogTime; + var filters =['Component', 'ServerId', 'Pid', 'Level', 'File', 'Line']; + filters.forEach(function(filter) { + var f = $j('#filter\\['+filter+'\\]'); + if ( f.val() ) { + clearParms += '&'+encodeURIComponent('filter[' + filter + ']')+'='+encodeURIComponent(f.val()); + } + }); + clearReq.send(clearParms); } function filterLog() { @@ -236,13 +249,21 @@ function exportRequest() { $('exportErrorText').set('text', ''); $('exportError').hide(); if ( form.validate() ) { + var exportReq = new Request.JSON({ + url: thisUrl, + method: 'post', + link: 'cancel', + onSuccess: exportResponse, + onFailure: exportFail + }); var exportParms = "view=request&request=log&task=export"; - var exportReq = new Request.JSON({url: thisUrl, method: 'post', link: 'cancel', onSuccess: exportResponse, onFailure: exportFail}); var selection = form.getElement('input[name=selector]:checked').get('value'); if ( selection == 'filter' || selection == 'current' ) { $$('#filters select').each( function( select ) { - exportParms += "&"+select.get('id')+"="+select.get('value'); + if ( select.get('value') ) { + exportParms += "&"+select.get('id')+"="+select.get('value'); + } } ); } @@ -250,6 +271,7 @@ function exportRequest() { var tbody = $(logTable).getElement( 'tbody' ); var rows = tbody.getElements( 'tr' ); if ( rows ) { + // Need to convert this to TimeKey var minTime = rows[0].getElement('td').get('text'); exportParms += "&minTime="+encodeURIComponent(minTime); var maxTime = rows[rows.length-1].getElement('td').get('text');