StardewValley
/
SMAPI
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

migrate update check caching to MongoDB (#651)

This commit is contained in:
Jesse Plamondon-Willard 2019-07-24 18:33:26 -04:00
parent 03a082297a
commit 17c6ae7ed9
No known key found for this signature in database
GPG Key ID: CF8B1456B3E29F49
8 changed files with 249 additions and 49 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

68
src/SMAPI.Web/Controllers/ModsApiController.cs
View File

@ -2,17 +2,17 @@ using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Options;
using StardewModdingAPI.Toolkit;
using StardewModdingAPI.Toolkit.Framework.Clients.WebApi;
using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
using StardewModdingAPI.Toolkit.Framework.ModData;
using StardewModdingAPI.Toolkit.Framework.UpdateData;
using StardewModdingAPI.Web.Framework.Caching.Mods;
using StardewModdingAPI.Web.Framework.Caching.Wiki;
using StardewModdingAPI.Web.Framework.Clients.Chucklefish;
using StardewModdingAPI.Web.Framework.Clients.GitHub;
using StardewModdingAPI.Web.Framework.Clients.ModDrop;
@ -33,8 +33,11 @@ namespace StardewModdingAPI.Web.Controllers
/// <summary>The mod repositories which provide mod metadata.</summary>
private readonly IDictionary<ModRepositoryKey, IModRepository> Repositories;
/// <summary>The cache in which to store mod metadata.</summary>
private readonly IMemoryCache Cache;
/// <summary>The cache in which to store wiki data.</summary>
private readonly IWikiCacheRepository WikiCache;
/// <summary>The cache in which to store mod data.</summary>
private readonly IModCacheRepository ModCache;
/// <summary>The number of minutes successful update checks should be cached before refetching them.</summary>
private readonly int SuccessCacheMinutes;
@ -42,9 +45,6 @@ namespace StardewModdingAPI.Web.Controllers
/// <summary>The number of minutes failed update checks should be cached before refetching them.</summary>
private readonly int ErrorCacheMinutes;
/// <summary>A regex which matches SMAPI-style semantic version.</summary>
private readonly string VersionRegex;
/// <summary>The internal mod metadata list.</summary>
private readonly ModDatabase ModDatabase;
@ -57,22 +57,23 @@ namespace StardewModdingAPI.Web.Controllers
*********/
/// <summary>Construct an instance.</summary>
/// <param name="environment">The web hosting environment.</param>
/// <param name="cache">The cache in which to store mod metadata.</param>
/// <param name="wikiCache">The cache in which to store wiki data.</param>
/// <param name="modCache">The cache in which to store mod metadata.</param>
/// <param name="configProvider">The config settings for mod update checks.</param>
/// <param name="chucklefish">The Chucklefish API client.</param>
/// <param name="github">The GitHub API client.</param>
/// <param name="modDrop">The ModDrop API client.</param>
/// <param name="nexus">The Nexus API client.</param>
public ModsApiController(IHostingEnvironment environment, IMemoryCache cache, IOptions<ModUpdateCheckConfig> configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus)
public ModsApiController(IHostingEnvironment environment, IWikiCacheRepository wikiCache, IModCacheRepository modCache, IOptions<ModUpdateCheckConfig> configProvider, IChucklefishClient chucklefish, IGitHubClient github, IModDropClient modDrop, INexusClient nexus)
{
this.ModDatabase = new ModToolkit().GetModDatabase(Path.Combine(environment.WebRootPath, "SMAPI.metadata.json"));
ModUpdateCheckConfig config = configProvider.Value;
this.CompatibilityPageUrl = config.CompatibilityPageUrl;
this.Cache = cache;
this.WikiCache = wikiCache;
this.ModCache = modCache;
this.SuccessCacheMinutes = config.SuccessCacheMinutes;
this.ErrorCacheMinutes = config.ErrorCacheMinutes;
this.VersionRegex = config.SemanticVersionRegex;
this.Repositories =
new IModRepository[]
{
@ -93,7 +94,7 @@ namespace StardewModdingAPI.Web.Controllers
return new ModEntryModel[0];
// fetch wiki data
WikiModEntry[] wikiData = await this.GetWikiDataAsync();
WikiModEntry[] wikiData = this.WikiCache.GetWikiMods().Select(p => p.GetModel()).ToArray();
IDictionary<string, ModEntryModel> mods = new Dictionary<string, ModEntryModel>(StringComparer.CurrentCultureIgnoreCase);
foreach (ModSearchEntryModel mod in model.Mods)
{
@ -218,26 +219,6 @@ namespace StardewModdingAPI.Web.Controllers
return current != null && (other == null || other.IsOlderThan(current));
}
/// <summary>Get mod data from the wiki compatibility list.</summary>
private async Task<WikiModEntry[]> GetWikiDataAsync()
{
ModToolkit toolkit = new ModToolkit();
return await this.Cache.GetOrCreateAsync("_wiki", async entry =>
{
try
{
WikiModEntry[] entries = (await toolkit.GetWikiCompatibilityListAsync()).Mods;
entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.SuccessCacheMinutes);
return entries;
}
catch
{
entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.ErrorCacheMinutes);
return new WikiModEntry[0];
}
});
}
/// <summary>Get the mod info for an update key.</summary>
/// <param name="updateKey">The namespaced update key.</param>
private async Task<ModInfoModel> GetInfoForUpdateKeyAsync(string updateKey)
@ -247,24 +228,27 @@ namespace StardewModdingAPI.Web.Controllers
if (!parsed.LooksValid)
return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"The update key '{updateKey}' isn't in a valid format. It should contain the site key and mod ID like 'Nexus:541'.");
// get matching repository
if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository))
return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}].");
// fetch mod info
return await this.Cache.GetOrCreateAsync($"{repository.VendorKey}:{parsed.ID}".ToLower(), async entry =>
// get mod
if (!this.ModCache.TryGetMod(parsed.Repository, parsed.ID, out CachedMod mod) || this.ModCache.IsStale(mod.LastUpdated, mod.FetchStatus == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes))
{
// get site
if (!this.Repositories.TryGetValue(parsed.Repository, out IModRepository repository))
return new ModInfoModel().WithError(RemoteModStatus.DoesNotExist, $"There's no mod site with key '{parsed.Repository}'. Expected one of [{string.Join(", ", this.Repositories.Keys)}].");
// fetch mod
ModInfoModel result = await repository.GetModInfoAsync(parsed.ID);
if (result.Error == null)
{
if (result.Version == null)
result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with no version number.");
else if (!Regex.IsMatch(result.Version, this.VersionRegex, RegexOptions.CultureInvariant | RegexOptions.IgnoreCase))
else if (!SemanticVersion.TryParse(result.Version, out _))
result.WithError(RemoteModStatus.InvalidData, $"The update key '{updateKey}' matches a mod with invalid semantic version '{result.Version}'.");
}
entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(result.Status == RemoteModStatus.TemporaryError ? this.ErrorCacheMinutes : this.SuccessCacheMinutes);
return result;
});
// cache mod
this.ModCache.SaveMod(repository.VendorKey, parsed.ID, result, out mod);
}
return mod.GetModel();
}
/// <summary>Get update keys based on the available mod metadata, while maintaining the precedence order.</summary>

97
src/SMAPI.Web/Framework/Caching/Mods/CachedMod.cs Normal file
View File

@ -0,0 +1,97 @@
using System;
using System.Diagnostics.CodeAnalysis;
using MongoDB.Bson;
using MongoDB.Bson.Serialization.Attributes;
using StardewModdingAPI.Toolkit.Framework.UpdateData;
using StardewModdingAPI.Web.Framework.ModRepositories;
namespace StardewModdingAPI.Web.Framework.Caching.Mods
{
/// <summary>The model for cached mod data.</summary>
internal class CachedMod
{
/*********
** Accessors
*********/
/****
** Tracking
****/
/// <summary>The internal MongoDB ID.</summary>
[SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Named per MongoDB conventions.")]
[BsonIgnoreIfDefault]
public ObjectId _id { get; set; }
/// <summary>When the data was last updated.</summary>
public DateTimeOffset LastUpdated { get; set; }
/// <summary>When the data was last requested through the web API.</summary>
public DateTimeOffset LastRequested { get; set; }
/****
** Metadata
****/
/// <summary>The mod site on which the mod is found.</summary>
public ModRepositoryKey Site { get; set; }
/// <summary>The mod's unique ID within the <see cref="Site"/>.</summary>
public string ID { get; set; }
/// <summary>The mod availability status on the remote site.</summary>
public RemoteModStatus FetchStatus { get; set; }
/// <summary>The error message providing more info for the <see cref="FetchStatus"/>, if applicable.</summary>
public string FetchError { get; set; }
/****
** Mod info
****/
/// <summary>The mod's display name.</summary>
public string Name { get; set; }
/// <summary>The mod's latest version.</summary>
public string MainVersion { get; set; }
/// <summary>The mod's latest optional or prerelease version, if newer than <see cref="MainVersion"/>.</summary>
public string PreviewVersion { get; set; }
/// <summary>The URL for the mod page.</summary>
public string Url { get; set; }
/*********
** Accessors
*********/
/// <summary>Construct an instance.</summary>
public CachedMod() { }
/// <summary>Construct an instance.</summary>
/// <param name="site">The mod site on which the mod is found.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The mod data.</param>
public CachedMod(ModRepositoryKey site, string id, ModInfoModel mod)
{
// tracking
this.LastUpdated = DateTimeOffset.UtcNow;
this.LastRequested = DateTimeOffset.UtcNow;
// metadata
this.Site = site;
this.ID = id;
this.FetchStatus = mod.Status;
this.FetchError = mod.Error;
// mod info
this.Name = mod.Name;
this.MainVersion = mod.Version;
this.PreviewVersion = mod.PreviewVersion;
this.Url = mod.Url;
}
/// <summary>Get the API model for the cached data.</summary>
public ModInfoModel GetModel()
{
return new ModInfoModel(name: this.Name, version: this.MainVersion, previewVersion: this.PreviewVersion, url: this.Url).WithError(this.FetchStatus, this.FetchError);
}
}
}

26
src/SMAPI.Web/Framework/Caching/Mods/IModCacheRepository.cs Normal file
View File

@ -0,0 +1,26 @@
using StardewModdingAPI.Toolkit.Framework.UpdateData;
using StardewModdingAPI.Web.Framework.ModRepositories;
namespace StardewModdingAPI.Web.Framework.Caching.Mods
{
/// <summary>Encapsulates logic for accessing the mod data cache.</summary>
internal interface IModCacheRepository : ICacheRepository
{
/*********
** Methods
*********/
/// <summary>Get the cached mod data.</summary>
/// <param name="site">The mod site to search.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The fetched mod.</param>
/// <param name="markRequested">Whether to update the mod's 'last requested' date.</param>
bool TryGetMod(ModRepositoryKey site, string id, out CachedMod mod, bool markRequested = true);
/// <summary>Save data fetched for a mod.</summary>
/// <param name="site">The mod site on which the mod is found.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The mod data.</param>
/// <param name="cachedMod">The stored mod record.</param>
void SaveMod(ModRepositoryKey site, string id, ModInfoModel mod, out CachedMod cachedMod);
}
}

96
src/SMAPI.Web/Framework/Caching/Mods/ModCacheRepository.cs Normal file
View File

@ -0,0 +1,96 @@
using System;
using MongoDB.Driver;
using StardewModdingAPI.Toolkit.Framework.UpdateData;
using StardewModdingAPI.Web.Framework.ModRepositories;
namespace StardewModdingAPI.Web.Framework.Caching.Mods
{
/// <summary>Encapsulates logic for accessing the mod data cache.</summary>
internal class ModCacheRepository : BaseCacheRepository, IModCacheRepository
{
/*********
** Fields
*********/
/// <summary>The collection for cached mod data.</summary>
private readonly IMongoCollection<CachedMod> Mods;
/*********
** Public methods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="database">The authenticated MongoDB database.</param>
public ModCacheRepository(IMongoDatabase database)
{
// get collections
this.Mods = database.GetCollection<CachedMod>("mods");
// add indexes if needed
this.Mods.Indexes.CreateOne(new CreateIndexModel<CachedMod>(Builders<CachedMod>.IndexKeys.Ascending(p => p.ID).Ascending(p => p.Site)));
}
/*********
** Public methods
*********/
/// <summary>Get the cached mod data.</summary>
/// <param name="site">The mod site to search.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The fetched mod.</param>
/// <param name="markRequested">Whether to update the mod's 'last requested' date.</param>
public bool TryGetMod(ModRepositoryKey site, string id, out CachedMod mod, bool markRequested = true)
{
// get mod
id = this.NormaliseId(id);
mod = this.Mods.Find(entry => entry.ID == id && entry.Site == site).FirstOrDefault();
if (mod == null)
return false;
// bump 'last requested'
if (markRequested)
{
mod.LastRequested = DateTimeOffset.UtcNow;
mod = this.SaveMod(mod);
}
return true;
}
/// <summary>Save data fetched for a mod.</summary>
/// <param name="site">The mod site on which the mod is found.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The mod data.</param>
/// <param name="cachedMod">The stored mod record.</param>
public void SaveMod(ModRepositoryKey site, string id, ModInfoModel mod, out CachedMod cachedMod)
{
id = this.NormaliseId(id);
cachedMod = this.SaveMod(new CachedMod(site, id, mod));
}
/*********
** Private methods
*********/
/// <summary>Save data fetched for a mod.</summary>
/// <param name="mod">The mod data.</param>
public CachedMod SaveMod(CachedMod mod)
{
string id = this.NormaliseId(mod.ID);
this.Mods.ReplaceOne(
entry => entry.ID == id && entry.Site == mod.Site,
mod,
new UpdateOptions { IsUpsert = true }
);
return mod;
}
/// <summary>Normalise a mod ID for case-insensitive search.</summary>
/// <param name="id">The mod ID.</param>
public string NormaliseId(string id)
{
return id.Trim().ToLower();
}
}
}

4
src/SMAPI.Web/Framework/ConfigModels/ModUpdateCheckConfig.cs
View File

@ -12,10 +12,6 @@ namespace StardewModdingAPI.Web.Framework.ConfigModels
/// <summary>The number of minutes failed update checks should be cached before refetching them.</summary>
public int ErrorCacheMinutes { get; set; }
/// <summary>A regex which matches SMAPI-style semantic version.</summary>
/// <remarks>Derived from SMAPI's SemanticVersion implementation.</remarks>
public string SemanticVersionRegex { get; set; }
/// <summary>The web URL for the wiki compatibility list.</summary>
public string CompatibilityPageUrl { get; set; }
}

4
src/SMAPI.Web/Framework/ModRepositories/ModInfoModel.cs
View File

@ -18,7 +18,7 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories
/// <summary>The mod's web URL.</summary>
public string Url { get; set; }
/// <summary>The mod availability status.</summary>
/// <summary>The mod availability status on the remote site.</summary>
public RemoteModStatus Status { get; set; } = RemoteModStatus.Ok;
/// <summary>The error message indicating why the mod is invalid (if applicable).</summary>
@ -45,7 +45,7 @@ namespace StardewModdingAPI.Web.Framework.ModRepositories
}
/// <summary>Set a mod error.</summary>
/// <param name="status">The mod availability status.</param>
/// <param name="status">The mod availability status on the remote site.</param>
/// <param name="error">The error message indicating why the mod is invalid (if applicable).</param>
public ModInfoModel WithError(RemoteModStatus status, string error)
{

2
src/SMAPI.Web/Startup.cs
View File

@ -13,6 +13,7 @@ using Newtonsoft.Json;
using StardewModdingAPI.Toolkit.Serialisation;
using StardewModdingAPI.Web.Framework;
using StardewModdingAPI.Web.Framework.Caching;
using StardewModdingAPI.Web.Framework.Caching.Mods;
using StardewModdingAPI.Web.Framework.Caching.Wiki;
using StardewModdingAPI.Web.Framework.Clients.Chucklefish;
using StardewModdingAPI.Web.Framework.Clients.GitHub;
@ -88,6 +89,7 @@ namespace StardewModdingAPI.Web
BsonSerializer.RegisterSerializer(new UtcDateTimeOffsetSerializer());
return new MongoClient(mongoConfig.GetConnectionString()).GetDatabase(mongoConfig.Database);
});
services.AddSingleton<IModCacheRepository>(serv => new ModCacheRepository(serv.GetRequiredService<IMongoDatabase>()));
services.AddSingleton<IWikiCacheRepository>(serv => new WikiCacheRepository(serv.GetRequiredService<IMongoDatabase>()));
// init Hangfire

1
src/SMAPI.Web/appsettings.json
View File

@ -66,7 +66,6 @@
"ModUpdateCheck": {
"SuccessCacheMinutes": 60,
"ErrorCacheMinutes": 5,
"SemanticVersionRegex": "^(?>(?<major>0|[1-9]\\d*))\\.(?>(?<minor>0|[1-9]\\d*))(?>(?:\\.(?<patch>0|[1-9]\\d*))?)(?:-(?<prerelease>(?>[a-z0-9]+[\\-\\.]?)+))?$",
"CompatibilityPageUrl": "https://mods.smapi.io"
}
}