From 7ed1fbf0aac5a3b777e3ed5b8f104bd27ce4bf0c Mon Sep 17 00:00:00 2001 From: Jesse Plamondon-Willard Date: Sun, 29 Oct 2017 15:28:07 -0400 Subject: [PATCH] defer log compression to backend and significantly improve compression (#358) --- .../Controllers/LogParserController.cs | 84 ++++++++++++++++++- src/SMAPI.Web/Views/LogParser/Index.cshtml | 1 - .../wwwroot/Content/js/log-parser.js | 13 +-- 3 files changed, 87 insertions(+), 11 deletions(-) diff --git a/src/SMAPI.Web/Controllers/LogParserController.cs b/src/SMAPI.Web/Controllers/LogParserController.cs index 8e986196..f143bc5c 100644 --- a/src/SMAPI.Web/Controllers/LogParserController.cs +++ b/src/SMAPI.Web/Controllers/LogParserController.cs @@ -1,3 +1,7 @@ +using System; +using System.IO; +using System.IO.Compression; +using System.Text; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Options; @@ -20,6 +24,10 @@ namespace StardewModdingAPI.Web.Controllers /// The underlying Pastebin client. private readonly PastebinClient PastebinClient; + /// The first bytes in a valid zip file. + /// See . + private const uint GzipLeadBytes = 0x8b1f; + /********* ** Public methods @@ -60,7 +68,9 @@ namespace StardewModdingAPI.Web.Controllers [Route("log/fetch/{id}")] public async Task GetAsync(string id) { - return await this.PastebinClient.GetAsync(id); + GetPasteResponse response = await this.PastebinClient.GetAsync(id); + response.Content = this.DecompressString(response.Content); + return response; } /// Save raw log data. @@ -69,7 +79,79 @@ namespace StardewModdingAPI.Web.Controllers [Route("log/save")] public async Task PostAsync([FromBody] string content) { + content = this.CompressString(content); return await this.PastebinClient.PostAsync(content); } + + + /********* + ** Private methods + *********/ + /// Compress a string. + /// The text to compress. + /// Derived from . + private string CompressString(string text) + { + // get raw bytes + byte[] buffer = Encoding.UTF8.GetBytes(text); + + // compressed + byte[] compressedData; + using (MemoryStream stream = new MemoryStream()) + { + using (GZipStream zipStream = new GZipStream(stream, CompressionLevel.Optimal, leaveOpen: true)) + zipStream.Write(buffer, 0, buffer.Length); + + stream.Position = 0; + compressedData = new byte[stream.Length]; + stream.Read(compressedData, 0, compressedData.Length); + } + + // prefix length + var zipBuffer = new byte[compressedData.Length + 4]; + Buffer.BlockCopy(compressedData, 0, zipBuffer, 4, compressedData.Length); + Buffer.BlockCopy(BitConverter.GetBytes(buffer.Length), 0, zipBuffer, 0, 4); + + // return string representation + return Convert.ToBase64String(zipBuffer); + } + + /// Decompress a string. + /// The compressed text. + /// Derived from . + private string DecompressString(string rawText) + { + // get raw bytes + byte[] zipBuffer; + try + { + zipBuffer = Convert.FromBase64String(rawText); + } + catch + { + return rawText; // not valid base64, wasn't compressed by the log parser + } + + // skip if not gzip + if (BitConverter.ToUInt16(zipBuffer, 4) != LogParserController.GzipLeadBytes) + return rawText; + + // decompress + using (MemoryStream memoryStream = new MemoryStream()) + { + // read length prefix + int dataLength = BitConverter.ToInt32(zipBuffer, 0); + memoryStream.Write(zipBuffer, 4, zipBuffer.Length - 4); + + // read data + var buffer = new byte[dataLength]; + memoryStream.Position = 0; + using (GZipStream gZipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) + gZipStream.Read(buffer, 0, buffer.Length); + + // return original string + return Encoding.UTF8.GetString(buffer); + } + } } } diff --git a/src/SMAPI.Web/Views/LogParser/Index.cshtml b/src/SMAPI.Web/Views/LogParser/Index.cshtml index a84866c8..49688d78 100644 --- a/src/SMAPI.Web/Views/LogParser/Index.cshtml +++ b/src/SMAPI.Web/Views/LogParser/Index.cshtml @@ -5,7 +5,6 @@ @section Head { -