migrate compatibility list's wiki data to MongoDB cache (#651)

docs/technical/web.md

@@ -82,25 +82,49 @@ For example:
 ```
 
 ## For SMAPI developers
-### Local development
+### Local environment
-`SMAPI.Web` is a regular ASP.NET MVC Core app, so you can just launch it from within
+A local environment lets you run a complete copy of the web project (including cache database) on
-Visual Studio to run a local version.
+your machine, with no external dependencies aside from the actual mod sites.
 
-There are two differences when it's run locally: all endpoints use HTTP instead of HTTPS, and the
+Initial setup:
-subdomain portion becomes a route (e.g. `log.smapi.io` → `localhost:59482/log`).
 
-Before running it locally, you need to enter your credentials in the `appsettings.Development.json`
+1. [Install MongoDB](https://docs.mongodb.com/manual/administration/install-community/) and add its
-file. See the next section for a description of each setting. This file is listed in `.gitignore`
+   `bin` folder to the system PATH.
-to prevent accidentally committing credentials.
+2. Create a local folder for the MongoDB data (e.g. `C:\dev\smapi-cache`).
+3. Enter your credentials in the `appsettings.Development.json` file. You can leave the MongoDB
+   credentials as-is to use the default local instance; see the next section for the other settings.
 
-### Deploying to Amazon Beanstalk
+To launch the environment:
-The app can be deployed to a standard Amazon Beanstalk IIS environment. When creating the
+1. Launch MongoDB from a terminal (change the data path if applicable):
-environment, make sure to specify the following environment properties:
+    ```sh
+    mongod --dbpath C:\dev\smapi-cache
+    ```
+2. Launch `SMAPI.Web` from Visual Studio to run a local version of the site.  
+    <small>(Local URLs will use HTTP instead of HTTPS, and subdomains will become routes, like
+    `log.smapi.io` &rarr; `localhost:59482/log`.)</small>
 
-property name                   | description
+### Production environment
-------------------------------- | -----------------
+A production environment includes the web servers and cache database hosted online for public
-`LogParser:PastebinDevKey`      | The [Pastebin developer key](https://pastebin.com/api#1) used to authenticate with the Pastebin API.
+access. This section assumes you're creating a new production environment from scratch (not using
-`LogParser:PastebinUserKey`     | The [Pastebin user key](https://pastebin.com/api#8) used to authenticate with the Pastebin API. Can be left blank to post anonymously.
+the official live environment).
-`LogParser:SectionUrl`          | The root URL of the log page, like `https://log.smapi.io/`.
+
-`ModUpdateCheck:GitHubPassword` | The password with which to authenticate to GitHub when fetching release info.
+Initial setup:
-`ModUpdateCheck:GitHubUsername` | The username with which to authenticate to GitHub when fetching release info.
+
+1. Launch an empty MongoDB server (e.g. using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas)).
+2. Create an AWS Beanstalk .NET environment with these environment properties:
+
+   property name                   | description
+   ------------------------------- | -----------------
+   `LogParser:PastebinDevKey`      | The [Pastebin developer key](https://pastebin.com/api#1) used to authenticate with the Pastebin API.
+   `LogParser:PastebinUserKey`     | The [Pastebin user key](https://pastebin.com/api#8) used to authenticate with the Pastebin API. Can be left blank to post anonymously.
+   `LogParser:SectionUrl`          | The root URL of the log page, like `https://log.smapi.io/`.
+   `ModUpdateCheck:GitHubPassword` | The password with which to authenticate to GitHub when fetching release info.
+   `ModUpdateCheck:GitHubUsername` | The username with which to authenticate to GitHub when fetching release info.
+   `MongoDB:Host`                  | The hostname for the MongoDB instance.
+   `MongoDB:Username`              | The login username for the MongoDB instance.
+   `MongoDB:Password`              | The login password for the MongoDB instance.
+
+To deploy updates:
+1. Deploy the web project using [AWS Toolkit for Visual Studio](https://aws.amazon.com/visualstudio/).
+2. If the MongoDB schema changed, delete the collections in the MongoDB database. (They'll be
+   recreated automatically.)

src/SMAPI.Web/Controllers/ModsController.cs

@@ -1,12 +1,10 @@
-using System;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.Extensions.Caching.Memory;
 using Microsoft.Extensions.Options;
 using StardewModdingAPI.Toolkit;
-using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
+using StardewModdingAPI.Web.Framework.Caching.Wiki;
 using StardewModdingAPI.Web.Framework.ConfigModels;
 using StardewModdingAPI.Web.ViewModels;
 
@@ -19,7 +17,7 @@ namespace StardewModdingAPI.Web.Controllers
         ** Fields
         *********/
         /// <summary>The cache in which to store mod metadata.</summary>
-        private readonly IMemoryCache Cache;
+        private readonly IWikiCacheRepository Cache;
 
         /// <summary>The number of minutes successful update checks should be cached before refetching them.</summary>
         private readonly int CacheMinutes;
@@ -31,7 +29,7 @@ namespace StardewModdingAPI.Web.Controllers
         /// <summary>Construct an instance.</summary>
         /// <param name="cache">The cache in which to store mod metadata.</param>
         /// <param name="configProvider">The config settings for mod update checks.</param>
-        public ModsController(IMemoryCache cache, IOptions<ModCompatibilityListConfig> configProvider)
+        public ModsController(IWikiCacheRepository cache, IOptions<ModCompatibilityListConfig> configProvider)
         {
             ModCompatibilityListConfig config = configProvider.Value;
 
@@ -54,21 +52,24 @@ namespace StardewModdingAPI.Web.Controllers
         /// <summary>Asynchronously fetch mod metadata from the wiki.</summary>
         public async Task<ModListModel> FetchDataAsync()
         {
-            return await this.Cache.GetOrCreateAsync($"{nameof(ModsController)}_mod_list", async entry =>
+            // refresh cache
+            CachedWikiMod[] mods;
+            if (!this.Cache.TryGetWikiMetadata(out CachedWikiMetadata metadata) || this.Cache.IsStale(metadata.LastUpdated, this.CacheMinutes))
             {
-                WikiModList data = await new ModToolkit().GetWikiCompatibilityListAsync();
+                var wikiCompatList = await new ModToolkit().GetWikiCompatibilityListAsync();
-                ModListModel model = new ModListModel(
+                this.Cache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods, out metadata, out mods);
-                    stableVersion: data.StableVersion,
+            }
-                    betaVersion: data.BetaVersion,
+            else
-                    mods: data
+                mods = this.Cache.GetWikiMods().ToArray();
-                        .Mods
-                        .Select(mod => new ModModel(mod))
-                        .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting
-                );
 
-                entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.CacheMinutes);
+            // build model
-                return model;
+            return new ModListModel(
-            });
+                stableVersion: metadata.StableVersion,
+                betaVersion: metadata.BetaVersion,
+                mods: mods
+                    .Select(mod => new ModModel(mod.GetModel()))
+                    .OrderBy(p => Regex.Replace(p.Name.ToLower(), "[^a-z0-9]", "")) // ignore case, spaces, and special characters when sorting
+            );
         }
     }
 }

src/SMAPI.Web/Framework/Caching/BaseCacheRepository.cs

@@ -0,0 +1,19 @@
+using System;
+
+namespace StardewModdingAPI.Web.Framework.Caching
+{
+    /// <summary>The base logic for a cache repository.</summary>
+    internal abstract class BaseCacheRepository
+    {
+        /*********
+        ** Public methods
+        *********/
+        /// <summary>Whether cached data is stale.</summary>
+        /// <param name="lastUpdated">The date when the data was updated.</param>
+        /// <param name="cacheMinutes">The age in minutes before data is considered stale.</param>
+        public bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes)
+        {
+            return lastUpdated < DateTimeOffset.UtcNow.AddMinutes(-cacheMinutes);
+        }
+    }
+}

src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMetadata.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using MongoDB.Bson;
+
+namespace StardewModdingAPI.Web.Framework.Caching.Wiki
+{
+    /// <summary>The model for cached wiki metadata.</summary>
+    public class CachedWikiMetadata
+    {
+        /*********
+        ** Accessors
+        *********/
+        /// <summary>The internal MongoDB ID.</summary>
+        [SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Named per MongoDB conventions.")]
+        public ObjectId _id { get; set; }
+
+        /// <summary>When the data was last updated.</summary>
+        public DateTimeOffset LastUpdated { get; set; }
+
+        /// <summary>The current stable Stardew Valley version.</summary>
+        public string StableVersion { get; set; }
+
+        /// <summary>The current beta Stardew Valley version.</summary>
+        public string BetaVersion { get; set; }
+
+
+        /*********
+        ** Public methods
+        *********/
+        /// <summary>Construct an instance.</summary>
+        public CachedWikiMetadata() { }
+
+        /// <summary>Construct an instance.</summary>
+        /// <param name="stableVersion">The current stable Stardew Valley version.</param>
+        /// <param name="betaVersion">The current beta Stardew Valley version.</param>
+        public CachedWikiMetadata(string stableVersion, string betaVersion)
+        {
+            this.StableVersion = stableVersion;
+            this.BetaVersion = betaVersion;
+            this.LastUpdated = DateTimeOffset.UtcNow;;
+        }
+    }
+}

src/SMAPI.Web/Framework/Caching/Wiki/CachedWikiMod.cs

@@ -0,0 +1,188 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using MongoDB.Bson;
+using StardewModdingAPI.Toolkit;
+using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
+
+namespace StardewModdingAPI.Web.Framework.Caching.Wiki
+{
+    /// <summary>The model for cached wiki mods.</summary>
+    public class CachedWikiMod
+    {
+        /*********
+        ** Accessors
+        *********/
+        /****
+        ** Tracking
+        ****/
+        /// <summary>The internal MongoDB ID.</summary>
+        [SuppressMessage("ReSharper", "InconsistentNaming", Justification = "Named per MongoDB conventions.")]
+        public ObjectId _id { get; set; }
+
+        /// <summary>When the data was last updated.</summary>
+        public DateTimeOffset LastUpdated { get; set; }
+
+        /****
+        ** Mod info
+        ****/
+        /// <summary>The mod's unique ID. If the mod has alternate/old IDs, they're listed in latest to newest order.</summary>
+        public string[] ID { get; set; }
+
+        /// <summary>The mod's display name. If the mod has multiple names, the first one is the most canonical name.</summary>
+        public string[] Name { get; set; }
+
+        /// <summary>The mod's author name.  If the author has multiple names, the first one is the most canonical name.</summary>
+        public string[] Author { get; set; }
+
+        /// <summary>The mod ID on Nexus.</summary>
+        public int? NexusID { get; set; }
+
+        /// <summary>The mod ID in the Chucklefish mod repo.</summary>
+        public int? ChucklefishID { get; set; }
+
+        /// <summary>The mod ID in the ModDrop mod repo.</summary>
+        public int? ModDropID { get; set; }
+
+        /// <summary>The GitHub repository in the form 'owner/repo'.</summary>
+        public string GitHubRepo { get; set; }
+
+        /// <summary>The URL to a non-GitHub source repo.</summary>
+        public string CustomSourceUrl { get; set; }
+
+        /// <summary>The custom mod page URL (if applicable).</summary>
+        public string CustomUrl { get; set; }
+
+        /// <summary>The name of the mod which loads this content pack, if applicable.</summary>
+        public string ContentPackFor { get; set; }
+
+        /// <summary>The human-readable warnings for players about this mod.</summary>
+        public string[] Warnings { get; set; }
+
+        /// <summary>The link anchor for the mod entry in the wiki compatibility list.</summary>
+        public string Anchor { get; set; }
+
+        /****
+        ** Stable compatibility
+        ****/
+        /// <summary>The compatibility status.</summary>
+        public WikiCompatibilityStatus MainStatus { get; set; }
+
+        /// <summary>The human-readable summary of the compatibility status or workaround, without HTML formatting.</summary>
+        public string MainSummary { get; set; }
+
+        /// <summary>The game or SMAPI version which broke this mod (if applicable).</summary>
+        public string MainBrokeIn { get; set; }
+
+        /// <summary>The version of the latest unofficial update, if applicable.</summary>
+        public string MainUnofficialVersion { get; set; }
+
+        /// <summary>The URL to the latest unofficial update, if applicable.</summary>
+        public string MainUnofficialUrl { get; set; }
+
+        /****
+        ** Beta compatibility
+        ****/
+        /// <summary>The compatibility status.</summary>
+        public WikiCompatibilityStatus? BetaStatus { get; set; }
+
+        /// <summary>The human-readable summary of the compatibility status or workaround, without HTML formatting.</summary>
+        public string BetaSummary { get; set; }
+
+        /// <summary>The game or SMAPI version which broke this mod (if applicable).</summary>
+        public string BetaBrokeIn { get; set; }
+
+        /// <summary>The version of the latest unofficial update, if applicable.</summary>
+        public string BetaUnofficialVersion { get; set; }
+
+        /// <summary>The URL to the latest unofficial update, if applicable.</summary>
+        public string BetaUnofficialUrl { get; set; }
+
+
+        /*********
+        ** Accessors
+        *********/
+        /// <summary>Construct an instance.</summary>
+        public CachedWikiMod() { }
+
+        /// <summary>Construct an instance.</summary>
+        /// <param name="mod">The mod data.</param>
+        public CachedWikiMod(WikiModEntry mod)
+        {
+            // tracking
+            this.LastUpdated = DateTimeOffset.UtcNow;
+
+            // mod info
+            this.ID = mod.ID;
+            this.Name = mod.Name;
+            this.Author = mod.Author;
+            this.NexusID = mod.NexusID;
+            this.ChucklefishID = mod.ChucklefishID;
+            this.ModDropID = mod.ModDropID;
+            this.GitHubRepo = mod.GitHubRepo;
+            this.CustomSourceUrl = mod.CustomSourceUrl;
+            this.CustomUrl = mod.CustomUrl;
+            this.ContentPackFor = mod.ContentPackFor;
+            this.Warnings = mod.Warnings;
+            this.Anchor = mod.Anchor;
+
+            // stable compatibility
+            this.MainStatus = mod.Compatibility.Status;
+            this.MainSummary = mod.Compatibility.Summary;
+            this.MainBrokeIn = mod.Compatibility.BrokeIn;
+            this.MainUnofficialVersion = mod.Compatibility.UnofficialVersion?.ToString();
+            this.MainUnofficialUrl = mod.Compatibility.UnofficialUrl;
+
+            // beta compatibility
+            this.BetaStatus = mod.BetaCompatibility?.Status;
+            this.BetaSummary = mod.BetaCompatibility?.Summary;
+            this.BetaBrokeIn = mod.BetaCompatibility?.BrokeIn;
+            this.BetaUnofficialVersion = mod.BetaCompatibility?.UnofficialVersion?.ToString();
+            this.BetaUnofficialUrl = mod.BetaCompatibility?.UnofficialUrl;
+        }
+
+        /// <summary>Reconstruct the original model.</summary>
+        public WikiModEntry GetModel()
+        {
+            var mod = new WikiModEntry
+            {
+                ID = this.ID,
+                Name = this.Name,
+                Author = this.Author,
+                NexusID = this.NexusID,
+                ChucklefishID = this.ChucklefishID,
+                ModDropID = this.ModDropID,
+                GitHubRepo = this.GitHubRepo,
+                CustomSourceUrl = this.CustomSourceUrl,
+                CustomUrl = this.CustomUrl,
+                ContentPackFor = this.ContentPackFor,
+                Warnings = this.Warnings,
+                Anchor = this.Anchor,
+
+                // stable compatibility
+                Compatibility = new WikiCompatibilityInfo
+                {
+                    Status = this.MainStatus,
+                    Summary = this.MainSummary,
+                    BrokeIn = this.MainBrokeIn,
+                    UnofficialVersion = this.MainUnofficialVersion != null ? new SemanticVersion(this.MainUnofficialVersion) : null,
+                    UnofficialUrl = this.MainUnofficialUrl
+                }
+            };
+
+            // beta compatibility
+            if (this.BetaStatus != null)
+            {
+                mod.BetaCompatibility = new WikiCompatibilityInfo
+                {
+                    Status = this.BetaStatus.Value,
+                    Summary = this.BetaSummary,
+                    BrokeIn = this.BetaBrokeIn,
+                    UnofficialVersion = this.BetaUnofficialVersion != null ? new SemanticVersion(this.BetaUnofficialVersion) : null,
+                    UnofficialUrl = this.BetaUnofficialUrl
+                };
+            }
+
+            return mod;
+        }
+    }
+}

src/SMAPI.Web/Framework/Caching/Wiki/IWikiCacheRepository.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
+
+namespace StardewModdingAPI.Web.Framework.Caching.Wiki
+{
+    /// <summary>Encapsulates logic for accessing the mod data cache.</summary>
+    internal interface IWikiCacheRepository
+    {
+        /*********
+        ** Methods
+        *********/
+        /// <summary>Get the cached wiki metadata.</summary>
+        /// <param name="metadata">The fetched metadata.</param>
+        bool TryGetWikiMetadata(out CachedWikiMetadata metadata);
+
+        /// <summary>Whether cached data is stale.</summary>
+        /// <param name="lastUpdated">The date when the data was updated.</param>
+        /// <param name="cacheMinutes">The age in minutes before data is considered stale.</param>
+        bool IsStale(DateTimeOffset lastUpdated, int cacheMinutes);
+
+        /// <summary>Get the cached wiki mods.</summary>
+        /// <param name="filter">A filter to apply, if any.</param>
+        IEnumerable<CachedWikiMod> GetWikiMods(Expression<Func<CachedWikiMod, bool>> filter = null);
+
+        /// <summary>Save data fetched from the wiki compatibility list.</summary>
+        /// <param name="stableVersion">The current stable Stardew Valley version.</param>
+        /// <param name="betaVersion">The current beta Stardew Valley version.</param>
+        /// <param name="mods">The mod data.</param>
+        /// <param name="cachedMetadata">The stored metadata record.</param>
+        /// <param name="cachedMods">The stored mod records.</param>
+        void SaveWikiData(string stableVersion, string betaVersion, IEnumerable<WikiModEntry> mods, out CachedWikiMetadata cachedMetadata, out CachedWikiMod[] cachedMods);
+    }
+}

src/SMAPI.Web/Framework/Caching/Wiki/WikiCacheRepository.cs

@@ -0,0 +1,73 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Linq.Expressions;
+using MongoDB.Driver;
+using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
+
+namespace StardewModdingAPI.Web.Framework.Caching.Wiki
+{
+    /// <summary>Encapsulates logic for accessing the wiki data cache.</summary>
+    internal class WikiCacheRepository : BaseCacheRepository, IWikiCacheRepository
+    {
+        /*********
+        ** Fields
+        *********/
+        /// <summary>The collection for wiki metadata.</summary>
+        private readonly IMongoCollection<CachedWikiMetadata> WikiMetadata;
+
+        /// <summary>The collection for wiki mod data.</summary>
+        private readonly IMongoCollection<CachedWikiMod> WikiMods;
+
+
+        /*********
+        ** Public methods
+        *********/
+        /// <summary>Construct an instance.</summary>
+        /// <param name="database">The authenticated MongoDB database.</param>
+        public WikiCacheRepository(IMongoDatabase database)
+        {
+            // get collections
+            this.WikiMetadata = database.GetCollection<CachedWikiMetadata>("wiki-metadata");
+            this.WikiMods = database.GetCollection<CachedWikiMod>("wiki-mods");
+
+            // add indexes if needed
+            this.WikiMods.Indexes.CreateOne(new CreateIndexModel<CachedWikiMod>(Builders<CachedWikiMod>.IndexKeys.Text(p => p.ID)));
+        }
+
+        /// <summary>Get the cached wiki metadata.</summary>
+        /// <param name="metadata">The fetched metadata.</param>
+        public bool TryGetWikiMetadata(out CachedWikiMetadata metadata)
+        {
+            metadata = this.WikiMetadata.Find("{}").FirstOrDefault();
+            return metadata != null;
+        }
+
+        /// <summary>Get the cached wiki mods.</summary>
+        /// <param name="filter">A filter to apply, if any.</param>
+        public IEnumerable<CachedWikiMod> GetWikiMods(Expression<Func<CachedWikiMod, bool>> filter = null)
+        {
+            return filter != null
+                ? this.WikiMods.Find(filter).ToList()
+                : this.WikiMods.Find("{}").ToList();
+        }
+
+        /// <summary>Save data fetched from the wiki compatibility list.</summary>
+        /// <param name="stableVersion">The current stable Stardew Valley version.</param>
+        /// <param name="betaVersion">The current beta Stardew Valley version.</param>
+        /// <param name="mods">The mod data.</param>
+        /// <param name="cachedMetadata">The stored metadata record.</param>
+        /// <param name="cachedMods">The stored mod records.</param>
+        public void SaveWikiData(string stableVersion, string betaVersion, IEnumerable<WikiModEntry> mods, out CachedWikiMetadata cachedMetadata, out CachedWikiMod[] cachedMods)
+        {
+            cachedMetadata = new CachedWikiMetadata(stableVersion, betaVersion);
+            cachedMods = mods.Select(mod => new CachedWikiMod(mod)).ToArray();
+
+            this.WikiMods.DeleteMany("{}");
+            this.WikiMods.InsertMany(cachedMods);
+
+            this.WikiMetadata.DeleteMany("{}");
+            this.WikiMetadata.InsertOne(cachedMetadata);
+        }
+    }
+}

src/SMAPI.Web/Framework/ConfigModels/MongoDbConfig.cs

@@ -0,0 +1,36 @@
+using System;
+
+namespace StardewModdingAPI.Web.Framework.ConfigModels
+{
+    /// <summary>The config settings for mod compatibility list.</summary>
+    internal class MongoDbConfig
+    {
+        /*********
+        ** Accessors
+        *********/
+        /// <summary>The MongoDB hostname.</summary>
+        public string Host { get; set; }
+
+        /// <summary>The MongoDB username (if any).</summary>
+        public string Username { get; set; }
+
+        /// <summary>The MongoDB password (if any).</summary>
+        public string Password { get; set; }
+
+
+        /*********
+        ** Public method
+        *********/
+        /// <summary>Get the MongoDB connection string.</summary>
+        /// <param name="authDatabase">The initial database for which to authenticate.</param>
+        public string GetConnectionString(string authDatabase)
+        {
+            bool isLocal = this.Host == "localhost";
+            bool hasLogin = !string.IsNullOrWhiteSpace(this.Username) && !string.IsNullOrWhiteSpace(this.Password);
+
+            return $"mongodb{(isLocal ? "" : "+srv")}://"
+                + (hasLogin ? $"{Uri.EscapeDataString(this.Username)}:{Uri.EscapeDataString(this.Password)}@" : "")
+                + $"{this.Host}/{authDatabase}retryWrites=true&w=majority";
+        }
+    }
+}

src/SMAPI.Web/SMAPI.Web.csproj

@@ -23,12 +23,16 @@
     <PackageReference Include="Microsoft.AspNetCore.Rewrite" Version="2.1.1" />
     <PackageReference Include="Microsoft.AspNetCore.StaticFiles" Version="2.1.1" />
     <PackageReference Include="Microsoft.Extensions.Logging.Debug" Version="2.1.1" />
+    <PackageReference Include="MongoDB.Driver" Version="2.8.1" />
     <PackageReference Include="Pathoschild.Http.FluentClient" Version="3.2.0" />
   </ItemGroup>
   <ItemGroup>
     <DotNetCliToolReference Include="Microsoft.VisualStudio.Web.CodeGeneration.Tools" Version="1.0.1" />
   </ItemGroup>
   <Import Project="..\SMAPI.Internal\SMAPI.Internal.projitems" Label="Shared" />
+  <ItemGroup>
+    <None Include="..\..\docs\technical\web.md" Link="web.md" />
+  </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\SMAPI.Toolkit\SMAPI.Toolkit.csproj" />
   </ItemGroup>

src/SMAPI.Web/Startup.cs

@@ -6,9 +6,11 @@ using Microsoft.AspNetCore.Routing;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
+using MongoDB.Driver;
 using Newtonsoft.Json;
 using StardewModdingAPI.Toolkit.Serialisation;
 using StardewModdingAPI.Web.Framework;
+using StardewModdingAPI.Web.Framework.Caching.Wiki;
 using StardewModdingAPI.Web.Framework.Clients.Chucklefish;
 using StardewModdingAPI.Web.Framework.Clients.GitHub;
 using StardewModdingAPI.Web.Framework.Clients.ModDrop;
@@ -53,6 +55,7 @@ namespace StardewModdingAPI.Web
                 .Configure<ModCompatibilityListConfig>(this.Configuration.GetSection("ModCompatibilityList"))
                 .Configure<ModUpdateCheckConfig>(this.Configuration.GetSection("ModUpdateCheck"))
                 .Configure<SiteConfig>(this.Configuration.GetSection("Site"))
+                .Configure<MongoDbConfig>(this.Configuration.GetSection("MongoDB"))
                 .Configure<RouteOptions>(options => options.ConstraintMap.Add("semanticVersion", typeof(VersionConstraint)))
                 .AddMemoryCache()
                 .AddMvc()
@@ -108,6 +111,15 @@ namespace StardewModdingAPI.Web
                     devKey: api.PastebinDevKey
                 ));
             }
+
+            // init MongoDB
+            {
+                MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get<MongoDbConfig>();
+                string connectionString = mongoConfig.GetConnectionString("smapi");
+
+                services.AddSingleton<IMongoDatabase>(serv => new MongoClient(connectionString).GetDatabase("smapi"));
+                services.AddSingleton<IWikiCacheRepository>(serv => new WikiCacheRepository(serv.GetService<IMongoDatabase>()));
+            }
         }
 
         /// <summary>The method called by the runtime to configure the HTTP request pipeline.</summary>

src/SMAPI.Web/appsettings.Development.json

@@ -31,5 +31,11 @@
 
         "PastebinUserKey": null,
         "PastebinDevKey": null
+    },
+
+    "MongoDB": {
+        "Host": "localhost",
+        "Username": null,
+        "Password": null
     }
 }

src/SMAPI.Web/appsettings.json

@@ -47,6 +47,12 @@
         "PastebinDevKey": null // see top note
     },
 
+    "MongoDB": {
+        "Host": null, // see top note
+        "Username": null, // see top note
+        "Password": null // see top note
+    },
+
     "ModCompatibilityList": {
         "CacheMinutes": 10
     },