diff --git a/docs/technical/web.md b/docs/technical/web.md index 978114ef..db54a87a 100644 --- a/docs/technical/web.md +++ b/docs/technical/web.md @@ -82,25 +82,49 @@ For example: ``` ## For SMAPI developers -### Local development -`SMAPI.Web` is a regular ASP.NET MVC Core app, so you can just launch it from within -Visual Studio to run a local version. +### Local environment +A local environment lets you run a complete copy of the web project (including cache database) on +your machine, with no external dependencies aside from the actual mod sites. -There are two differences when it's run locally: all endpoints use HTTP instead of HTTPS, and the -subdomain portion becomes a route (e.g. `log.smapi.io` → `localhost:59482/log`). +Initial setup: -Before running it locally, you need to enter your credentials in the `appsettings.Development.json` -file. See the next section for a description of each setting. This file is listed in `.gitignore` -to prevent accidentally committing credentials. +1. [Install MongoDB](https://docs.mongodb.com/manual/administration/install-community/) and add its + `bin` folder to the system PATH. +2. Create a local folder for the MongoDB data (e.g. `C:\dev\smapi-cache`). +3. Enter your credentials in the `appsettings.Development.json` file. You can leave the MongoDB + credentials as-is to use the default local instance; see the next section for the other settings. -### Deploying to Amazon Beanstalk -The app can be deployed to a standard Amazon Beanstalk IIS environment. When creating the -environment, make sure to specify the following environment properties: +To launch the environment: +1. Launch MongoDB from a terminal (change the data path if applicable): + ```sh + mongod --dbpath C:\dev\smapi-cache + ``` +2. Launch `SMAPI.Web` from Visual Studio to run a local version of the site. + (Local URLs will use HTTP instead of HTTPS, and subdomains will become routes, like + `log.smapi.io` → `localhost:59482/log`.) -property name | description -------------------------------- | ----------------- -`LogParser:PastebinDevKey` | The [Pastebin developer key](https://pastebin.com/api#1) used to authenticate with the Pastebin API. -`LogParser:PastebinUserKey` | The [Pastebin user key](https://pastebin.com/api#8) used to authenticate with the Pastebin API. Can be left blank to post anonymously. -`LogParser:SectionUrl` | The root URL of the log page, like `https://log.smapi.io/`. -`ModUpdateCheck:GitHubPassword` | The password with which to authenticate to GitHub when fetching release info. -`ModUpdateCheck:GitHubUsername` | The username with which to authenticate to GitHub when fetching release info. +### Production environment +A production environment includes the web servers and cache database hosted online for public +access. This section assumes you're creating a new production environment from scratch (not using +the official live environment). + +Initial setup: + +1. Launch an empty MongoDB server (e.g. using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas)). +2. Create an AWS Beanstalk .NET environment with these environment properties: + + property name | description + ------------------------------- | ----------------- + `LogParser:PastebinDevKey` | The [Pastebin developer key](https://pastebin.com/api#1) used to authenticate with the Pastebin API. + `LogParser:PastebinUserKey` | The [Pastebin user key](https://pastebin.com/api#8) used to authenticate with the Pastebin API. Can be left blank to post anonymously. + `LogParser:SectionUrl` | The root URL of the log page, like `https://log.smapi.io/`. + `ModUpdateCheck:GitHubPassword` | The password with which to authenticate to GitHub when fetching release info. + `ModUpdateCheck:GitHubUsername` | The username with which to authenticate to GitHub when fetching release info. + `MongoDB:Host` | The hostname for the MongoDB instance. + `MongoDB:Username` | The login username for the MongoDB instance. + `MongoDB:Password` | The login password for the MongoDB instance. + +To deploy updates: +1. Deploy the web project using [AWS Toolkit for Visual Studio](https://aws.amazon.com/visualstudio/). +2. If the MongoDB schema changed, delete the collections in the MongoDB database. (They'll be + recreated automatically.) diff --git a/src/SMAPI.Web/Controllers/ModsController.cs b/src/SMAPI.Web/Controllers/ModsController.cs index ca866a8d..b6040e06 100644 --- a/src/SMAPI.Web/Controllers/ModsController.cs +++ b/src/SMAPI.Web/Controllers/ModsController.cs @@ -1,12 +1,10 @@ -using System; using System.Linq; using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.AspNetCore.Mvc; -using Microsoft.Extensions.Caching.Memory; using Microsoft.Extensions.Options; using StardewModdingAPI.Toolkit; -using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; +using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.ConfigModels; using StardewModdingAPI.Web.ViewModels; @@ -19,7 +17,7 @@ namespace StardewModdingAPI.Web.Controllers ** Fields *********/ /// The cache in which to store mod metadata. - private readonly IMemoryCache Cache; + private readonly IWikiCacheRepository Cache; /// The number of minutes successful update checks should be cached before refetching them. private readonly int CacheMinutes; @@ -31,7 +29,7 @@ namespace StardewModdingAPI.Web.Controllers /// Construct an instance. /// The cache in which to store mod metadata. /// The config settings for mod update checks. - public ModsController(IMemoryCache cache, IOptions configProvider) + public ModsController(IWikiCacheRepository cache, IOptions configProvider) { ModCompatibilityListConfig config = configProvider.Value; @@ -54,21 +52,24 @@ namespace StardewModdingAPI.Web.Controllers /// Asynchronously fetch mod metadata from the wiki. public async Task FetchDataAsync() { - return await this.Cache.GetOrCreateAsync($"{nameof(ModsController)}_mod_list", async entry => + // refresh cache + CachedWikiMod[] mods; + if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata) || this.Cache.IsStale(metadata.LastUpdated, this.CacheMinutes)) { - WikiModList data = await new ModToolkit().GetWikiCompatibilityListAsync(); - ModListModel model = new ModListModel( - stableVersion: data.StableVersion, - betaVersion: data.BetaVersion, - mods: data - .Mods - .Select(mod => new ModModel(mod)) - .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting - ); + var wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync(); + this.Cache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods, out metadata, out mods); + } + else + mods = this.Cache.GetWikiMods().ToArray(); - entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.CacheMinutes); - return model; - }); + // build model + return new ModListModel( + stableVersion: metadata.StableVersion, + betaVersion: metadata.BetaVersion, + mods: mods + .Select(mod => new ModModel(mod.GetModel())) + .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting + ); } } } diff --git a/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs new file mode 100644 index 00000000..904455c5 --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs @@ -0,0 +1,19 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.Caching +{ + /// The base logic for a cache repository. + internal abstract class BaseCacheRepository + { + /********* + ** Public methods + *********/ + /// Whether cached data is stale. + /// The date when the data was updated. + /// The age in minutes before data is considered stale. + public bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes) + { + return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-cacheMinutes); + } + } +} diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs new file mode 100644 index 00000000..de1ea9db --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs @@ -0,0 +1,43 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using MongoDB.Bson; + +namespace StardewModdingAPI.Web.Framework.Caching.Wiki +{ + /// The model for cached wiki metadata. + public class CachedWikiMetadata + { + /********* + ** Accessors + *********/ + /// The internal MongoDB ID. + [SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Named per MongoDB conventions.")] + public ObjectId _id { get; set; } + + /// When the data was last updated. + public DateTimeOffset LastUpdated { get; set; } + + /// The current stable Stardew Valley version. + public string StableVersion { get; set; } + + /// The current beta Stardew Valley version. + public string BetaVersion { get; set; } + + + /********* + ** Public methods + *********/ + /// Construct an instance. + public CachedWikiMetadata() { } + + /// Construct an instance. + /// The current stable Stardew Valley version. + /// The current beta Stardew Valley version. + public CachedWikiMetadata(string stableVersion, string betaVersion) + { + this.StableVersion = stableVersion; + this.BetaVersion = betaVersion; + this.LastUpdated = DateTimeOffset.UtcNow;; + } + } +} diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMod.cs b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMod.cs new file mode 100644 index 00000000..f4331f8d --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMod.cs @@ -0,0 +1,188 @@ +using System; +using System.Diagnostics.CodeAnalysis; +using MongoDB.Bson; +using StardewModdingAPI.Toolkit; +using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; + +namespace StardewModdingAPI.Web.Framework.Caching.Wiki +{ + /// The model for cached wiki mods. + public class CachedWikiMod + { + /********* + ** Accessors + *********/ + /**** + ** Tracking + ****/ + /// The internal MongoDB ID. + [SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Named per MongoDB conventions.")] + public ObjectId _id { get; set; } + + /// When the data was last updated. + public DateTimeOffset LastUpdated { get; set; } + + /**** + ** Mod info + ****/ + /// The mod's unique ID. If the mod has alternate/old IDs, they're listed in latest to newest order. + public string[] ID { get; set; } + + /// The mod's display name. If the mod has multiple names, the first one is the most canonical name. + public string[] Name { get; set; } + + /// The mod's author name. If the author has multiple names, the first one is the most canonical name. + public string[] Author { get; set; } + + /// The mod ID on Nexus. + public int? NexusID { get; set; } + + /// The mod ID in the Chucklefish mod repo. + public int? ChucklefishID { get; set; } + + /// The mod ID in the ModDrop mod repo. + public int? ModDropID { get; set; } + + /// The GitHub repository in the form 'owner/repo'. + public string GitHubRepo { get; set; } + + /// The URL to a non-GitHub source repo. + public string CustomSourceUrl { get; set; } + + /// The custom mod page URL (if applicable). + public string CustomUrl { get; set; } + + /// The name of the mod which loads this content pack, if applicable. + public string ContentPackFor { get; set; } + + /// The human-readable warnings for players about this mod. + public string[] Warnings { get; set; } + + /// The link anchor for the mod entry in the wiki compatibility list. + public string Anchor { get; set; } + + /**** + ** Stable compatibility + ****/ + /// The compatibility status. + public WikiCompatibilityStatus MainStatus { get; set; } + + /// The human-readable summary of the compatibility status or workaround, without HTML formatting. + public string MainSummary { get; set; } + + /// The game or SMAPI version which broke this mod (if applicable). + public string MainBrokeIn { get; set; } + + /// The version of the latest unofficial update, if applicable. + public string MainUnofficialVersion { get; set; } + + /// The URL to the latest unofficial update, if applicable. + public string MainUnofficialUrl { get; set; } + + /**** + ** Beta compatibility + ****/ + /// The compatibility status. + public WikiCompatibilityStatus? BetaStatus { get; set; } + + /// The human-readable summary of the compatibility status or workaround, without HTML formatting. + public string BetaSummary { get; set; } + + /// The game or SMAPI version which broke this mod (if applicable). + public string BetaBrokeIn { get; set; } + + /// The version of the latest unofficial update, if applicable. + public string BetaUnofficialVersion { get; set; } + + /// The URL to the latest unofficial update, if applicable. + public string BetaUnofficialUrl { get; set; } + + + /********* + ** Accessors + *********/ + /// Construct an instance. + public CachedWikiMod() { } + + /// Construct an instance. + /// The mod data. + public CachedWikiMod(WikiModEntry mod) + { + // tracking + this.LastUpdated = DateTimeOffset.UtcNow; + + // mod info + this.ID = mod.ID; + this.Name = mod.Name; + this.Author = mod.Author; + this.NexusID = mod.NexusID; + this.ChucklefishID = mod.ChucklefishID; + this.ModDropID = mod.ModDropID; + this.GitHubRepo = mod.GitHubRepo; + this.CustomSourceUrl = mod.CustomSourceUrl; + this.CustomUrl = mod.CustomUrl; + this.ContentPackFor = mod.ContentPackFor; + this.Warnings = mod.Warnings; + this.Anchor = mod.Anchor; + + // stable compatibility + this.MainStatus = mod.Compatibility.Status; + this.MainSummary = mod.Compatibility.Summary; + this.MainBrokeIn = mod.Compatibility.BrokeIn; + this.MainUnofficialVersion = mod.Compatibility.UnofficialVersion?.ToString(); + this.MainUnofficialUrl = mod.Compatibility.UnofficialUrl; + + // beta compatibility + this.BetaStatus = mod.BetaCompatibility?.Status; + this.BetaSummary = mod.BetaCompatibility?.Summary; + this.BetaBrokeIn = mod.BetaCompatibility?.BrokeIn; + this.BetaUnofficialVersion = mod.BetaCompatibility?.UnofficialVersion?.ToString(); + this.BetaUnofficialUrl = mod.BetaCompatibility?.UnofficialUrl; + } + + /// Reconstruct the original model. + public WikiModEntry GetModel() + { + var mod = new WikiModEntry + { + ID = this.ID, + Name = this.Name, + Author = this.Author, + NexusID = this.NexusID, + ChucklefishID = this.ChucklefishID, + ModDropID = this.ModDropID, + GitHubRepo = this.GitHubRepo, + CustomSourceUrl = this.CustomSourceUrl, + CustomUrl = this.CustomUrl, + ContentPackFor = this.ContentPackFor, + Warnings = this.Warnings, + Anchor = this.Anchor, + + // stable compatibility + Compatibility = new WikiCompatibilityInfo + { + Status = this.MainStatus, + Summary = this.MainSummary, + BrokeIn = this.MainBrokeIn, + UnofficialVersion = this.MainUnofficialVersion != null ? new SemanticVersion(this.MainUnofficialVersion) : null, + UnofficialUrl = this.MainUnofficialUrl + } + }; + + // beta compatibility + if (this.BetaStatus != null) + { + mod.BetaCompatibility = new WikiCompatibilityInfo + { + Status = this.BetaStatus.Value, + Summary = this.BetaSummary, + BrokeIn = this.BetaBrokeIn, + UnofficialVersion = this.BetaUnofficialVersion != null ? new SemanticVersion(this.BetaUnofficialVersion) : null, + UnofficialUrl = this.BetaUnofficialUrl + }; + } + + return mod; + } + } +} diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs new file mode 100644 index 00000000..d319db69 --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq.Expressions; +using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; + +namespace StardewModdingAPI.Web.Framework.Caching.Wiki +{ + /// Encapsulates logic for accessing the mod data cache. + internal interface IWikiCacheRepository + { + /********* + ** Methods + *********/ + /// Get the cached wiki metadata. + /// The fetched metadata. + bool TryGetWikiMetadata(out CachedWikiMetadata metadata); + + /// Whether cached data is stale. + /// The date when the data was updated. + /// The age in minutes before data is considered stale. + bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes); + + /// Get the cached wiki mods. + /// A filter to apply, if any. + IEnumerable GetWikiMods(Expression> filter = null); + + /// Save data fetched from the wiki compatibility list. + /// The current stable Stardew Valley version. + /// The current beta Stardew Valley version. + /// The mod data. + /// The stored metadata record. + /// The stored mod records. + void SaveWikiData(string stableVersion, string betaVersion, IEnumerable mods, out CachedWikiMetadata cachedMetadata, out CachedWikiMod[] cachedMods); + } +} diff --git a/src/SMAPI.Web/Framework/Caching/Wiki/WikiCacheRepository.cs b/src/SMAPI.Web/Framework/Caching/Wiki/WikiCacheRepository.cs new file mode 100644 index 00000000..5b907462 --- /dev/null +++ b/src/SMAPI.Web/Framework/Caching/Wiki/WikiCacheRepository.cs @@ -0,0 +1,73 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Linq.Expressions; +using MongoDB.Driver; +using StardewModdingAPI.Toolkit.Framework.Clients.Wiki; + +namespace StardewModdingAPI.Web.Framework.Caching.Wiki +{ + /// Encapsulates logic for accessing the wiki data cache. + internal class WikiCacheRepository : BaseCacheRepository, IWikiCacheRepository + { + /********* + ** Fields + *********/ + /// The collection for wiki metadata. + private readonly IMongoCollection WikiMetadata; + + /// The collection for wiki mod data. + private readonly IMongoCollection WikiMods; + + + /********* + ** Public methods + *********/ + /// Construct an instance. + /// The authenticated MongoDB database. + public WikiCacheRepository(IMongoDatabase database) + { + // get collections + this.WikiMetadata = database.GetCollection("wiki-metadata"); + this.WikiMods = database.GetCollection("wiki-mods"); + + // add indexes if needed + this.WikiMods.Indexes.CreateOne(new CreateIndexModel(Builders.IndexKeys.Text(p => p.ID))); + } + + /// Get the cached wiki metadata. + /// The fetched metadata. + public bool TryGetWikiMetadata(out CachedWikiMetadata metadata) + { + metadata = this.WikiMetadata.Find("{}").FirstOrDefault(); + return metadata != null; + } + + /// Get the cached wiki mods. + /// A filter to apply, if any. + public IEnumerable GetWikiMods(Expression> filter = null) + { + return filter != null + ? this.WikiMods.Find(filter).ToList() + : this.WikiMods.Find("{}").ToList(); + } + + /// Save data fetched from the wiki compatibility list. + /// The current stable Stardew Valley version. + /// The current beta Stardew Valley version. + /// The mod data. + /// The stored metadata record. + /// The stored mod records. + public void SaveWikiData(string stableVersion, string betaVersion, IEnumerable mods, out CachedWikiMetadata cachedMetadata, out CachedWikiMod[] cachedMods) + { + cachedMetadata = new CachedWikiMetadata(stableVersion, betaVersion); + cachedMods = mods.Select(mod => new CachedWikiMod(mod)).ToArray(); + + this.WikiMods.DeleteMany("{}"); + this.WikiMods.InsertMany(cachedMods); + + this.WikiMetadata.DeleteMany("{}"); + this.WikiMetadata.InsertOne(cachedMetadata); + } + } +} diff --git a/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs new file mode 100644 index 00000000..352eb960 --- /dev/null +++ b/src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs @@ -0,0 +1,36 @@ +using System; + +namespace StardewModdingAPI.Web.Framework.ConfigModels +{ + /// The config settings for mod compatibility list. + internal class MongoDbConfig + { + /********* + ** Accessors + *********/ + /// The MongoDB hostname. + public string Host { get; set; } + + /// The MongoDB username (if any). + public string Username { get; set; } + + /// The MongoDB password (if any). + public string Password { get; set; } + + + /********* + ** Public method + *********/ + /// Get the MongoDB connection string. + /// The initial database for which to authenticate. + public string GetConnectionString(string authDatabase) + { + bool isLocal = this.Host == "localhost"; + bool hasLogin = !string.IsNullOrWhiteSpace(this.Username) && !string.IsNullOrWhiteSpace(this.Password); + + return $"mongodb{(isLocal ? "" : "+srv")}://" + + (hasLogin ? $"{Uri.EscapeDataString(this.Username)}:{Uri.EscapeDataString(this.Password)}@" : "") + + $"{this.Host}/{authDatabase}retryWrites=true&w=majority"; + } + } +} diff --git a/src/SMAPI.Web/SMAPI.Web.csproj b/src/SMAPI.Web/SMAPI.Web.csproj index b0aaa8a9..90641611 100644 --- a/src/SMAPI.Web/SMAPI.Web.csproj +++ b/src/SMAPI.Web/SMAPI.Web.csproj @@ -23,12 +23,16 @@ + + + + diff --git a/src/SMAPI.Web/Startup.cs b/src/SMAPI.Web/Startup.cs index b409a81d..0a8d23a8 100644 --- a/src/SMAPI.Web/Startup.cs +++ b/src/SMAPI.Web/Startup.cs @@ -6,9 +6,11 @@ using Microsoft.AspNetCore.Routing; using Microsoft.Extensions.Configuration; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; +using MongoDB.Driver; using Newtonsoft.Json; using StardewModdingAPI.Toolkit.Serialisation; using StardewModdingAPI.Web.Framework; +using StardewModdingAPI.Web.Framework.Caching.Wiki; using StardewModdingAPI.Web.Framework.Clients.Chucklefish; using StardewModdingAPI.Web.Framework.Clients.GitHub; using StardewModdingAPI.Web.Framework.Clients.ModDrop; @@ -53,6 +55,7 @@ namespace StardewModdingAPI.Web .Configure(this.Configuration.GetSection("ModCompatibilityList")) .Configure(this.Configuration.GetSection("ModUpdateCheck")) .Configure(this.Configuration.GetSection("Site")) + .Configure(this.Configuration.GetSection("MongoDB")) .Configure(options => options.ConstraintMap.Add("semanticVersion", typeof(VersionConstraint))) .AddMemoryCache() .AddMvc() @@ -108,6 +111,15 @@ namespace StardewModdingAPI.Web devKey: api.PastebinDevKey )); } + + // init MongoDB + { + MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get(); + string connectionString = mongoConfig.GetConnectionString("smapi"); + + services.AddSingleton(serv => new MongoClient(connectionString).GetDatabase("smapi")); + services.AddSingleton(serv => new WikiCacheRepository(serv.GetService())); + } } /// The method called by the runtime to configure the HTTP request pipeline. diff --git a/src/SMAPI.Web/appsettings.Development.json b/src/SMAPI.Web/appsettings.Development.json index 49234a3b..9b0ec535 100644 --- a/src/SMAPI.Web/appsettings.Development.json +++ b/src/SMAPI.Web/appsettings.Development.json @@ -31,5 +31,11 @@ "PastebinUserKey": null, "PastebinDevKey": null + }, + + "MongoDB": { + "Host": "localhost", + "Username": null, + "Password": null } } diff --git a/src/SMAPI.Web/appsettings.json b/src/SMAPI.Web/appsettings.json index 9e15aa97..65ccea75 100644 --- a/src/SMAPI.Web/appsettings.json +++ b/src/SMAPI.Web/appsettings.json @@ -47,6 +47,12 @@ "PastebinDevKey": null // see top note }, + "MongoDB": { + "Host": null, // see top note + "Username": null, // see top note + "Password": null // see top note + }, + "ModCompatibilityList": { "CacheMinutes": 10 },