simplify single-instance deployment and make MongoDB server optional

This commit is contained in:
Jesse Plamondon-Willard 2020-05-16 14:30:07 -04:00
parent a2cfb71d89
commit 5e6f1640dc
No known key found for this signature in database
GPG Key ID: CF8B1456B3E29F49
14 changed files with 308 additions and 88 deletions

View File

@ -8,12 +8,16 @@
* For the web UI:
* Updated web framework to improve site performance and reliability.
* Added GitHub licenses to mod compatibility list.
* Internal changes to improve performance and reliability.
* For modders:
* Added `Multiplayer.PeerConnected` event.
* Simplified paranoid warnings in the log and reduced their log level.
* Fixed asset propagation for Gil's portraits.
* For SMAPI developers:
* When deploying web services to a single-instance app, the MongoDB server can now be replaced with in-memory storage.
## 3.5
Released 27 April 2020 for Stardew Valley 1.4.1 or later.

View File

@ -340,9 +340,20 @@ short url | → | target page
A local environment lets you run a complete copy of the web project (including cache database) on
your machine, with no external dependencies aside from the actual mod sites.
1. Enter the Nexus credentials in `appsettings.Development.json` . You can leave the other
credentials empty to default to fetching data anonymously, and storing data in-memory and
on disk.
1. Edit `appsettings.Development.json` and set these options:
property name | description
------------- | -----------
`NexusApiKey` | [Your Nexus API key](https://www.nexusmods.com/users/myaccount?tab=api#personal_key).
Optional settings:
property name | description
--------------------------- | -----------
`AzureBlobConnectionString` | The connection string for the Azure Blob storage account. Defaults to using the system's temporary file folder if not specified.
`GitHubUsername`<br />`GitHubPassword` | The GitHub credentials with which to query GitHub release info. Defaults to anonymous requests if not specified.
`Storage` | How to storage cached wiki/mod data. `InMemory` is recommended in most cases, or `MongoInMemory` to test the MongoDB storage code. See [production environment](#production-environment) for more info on `Mongo`.
2. Launch `SMAPI.Web` from Visual Studio to run a local version of the site.
### Production environment
@ -355,19 +366,15 @@ accordingly.
Initial setup:
1. Launch an empty MongoDB server (e.g. using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas))
for mod data.
2. Create an Azure Blob storage account for uploaded files.
3. Create an Azure App Services environment running the latest .NET Core on Linux or Windows.
4. Add these application settings in the new App Services environment:
1. Create an Azure Blob storage account for uploaded files.
2. Create an Azure App Services environment running the latest .NET Core on Linux or Windows.
3. Add these application settings in the new App Services environment:
property name | description
------------------------------- | -----------------
`ApiClients.AzureBlobConnectionString` | The connection string for the Azure Blob storage account created in step 2.
`ApiClients.GitHubUsername`<br />`ApiClients.GitHubPassword` | The login credentials for the GitHub account with which to fetch release info. If these are omitted, GitHub will impose much stricter rate limits.
`ApiClients:NexusApiKey` | The [Nexus API authentication key](https://github.com/Pathoschild/FluentNexus#init-a-client).
`MongoDB:ConnectionString` | The connection string for the MongoDB instance.
`MongoDB:Database` | The MongoDB database name (e.g. `smapi` in production or `smapi-edge` in testing environments).
Optional settings:
@ -378,6 +385,23 @@ Initial setup:
`Site:BetaBlurb` | If `Site:BetaEnabled` is true and there's a beta version of SMAPI in its GitHub releases, this is shown on the beta download button as explanatory subtext.
`Site:SupporterList` | A list of Patreon supports to credit on the download page.
To enable distributed servers:
1. Launch an empty MongoDB server (e.g. using [MongoDB Atlas](https://www.mongodb.com/cloud/atlas))
for mod data.
2. Add these application settings in the App Services environment:
property name | description
------------------------------- | -----------------
`Storage:Mode` | Set to `Mongo`.
`Storage:ConnectionString` | Set to the connection string for the MongoDB instance.
Optional settings:
property name | description
------------------------------- | -----------------
`Storage:Database` | Set to the MongoDB database name (defaults to `smapi`).
To deploy updates:
1. [Deploy the web project from Visual Studio](https://docs.microsoft.com/en-us/visualstudio/deployment/quickstart-deploy-to-azure).
2. If the MongoDB schema changed, delete the MongoDB database. (It'll be recreated automatically.)

View File

@ -4,7 +4,7 @@ using StardewModdingAPI.Web.Framework.ModRepositories;
namespace StardewModdingAPI.Web.Framework.Caching.Mods
{
/// <summary>Encapsulates logic for accessing the mod data cache.</summary>
/// <summary>Manages cached mod data.</summary>
internal interface IModCacheRepository : ICacheRepository
{
/*********

View File

@ -0,0 +1,89 @@
using System;
using System.Collections.Generic;
using System.Linq;
using StardewModdingAPI.Toolkit.Framework.UpdateData;
using StardewModdingAPI.Web.Framework.ModRepositories;
namespace StardewModdingAPI.Web.Framework.Caching.Mods
{
/// <summary>Manages cached mod data in-memory.</summary>
internal class ModCacheMemoryRepository : BaseCacheRepository, IModCacheRepository
{
/*********
** Fields
*********/
/// <summary>The cached mod data indexed by <c>{site key}:{ID}</c>.</summary>
private readonly IDictionary<string, CachedMod> Mods = new Dictionary<string, CachedMod>(StringComparer.InvariantCultureIgnoreCase);
/*********
** Public methods
*********/
/// <summary>Get the cached mod data.</summary>
/// <param name="site">The mod site to search.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The fetched mod.</param>
/// <param name="markRequested">Whether to update the mod's 'last requested' date.</param>
public bool TryGetMod(ModRepositoryKey site, string id, out CachedMod mod, bool markRequested = true)
{
// get mod
if (!this.Mods.TryGetValue(this.GetKey(site, id), out mod))
return false;
// bump 'last requested'
if (markRequested)
{
mod.LastRequested = DateTimeOffset.UtcNow;
mod = this.SaveMod(mod);
}
return true;
}
/// <summary>Save data fetched for a mod.</summary>
/// <param name="site">The mod site on which the mod is found.</param>
/// <param name="id">The mod's unique ID within the <paramref name="site"/>.</param>
/// <param name="mod">The mod data.</param>
/// <param name="cachedMod">The stored mod record.</param>
public void SaveMod(ModRepositoryKey site, string id, ModInfoModel mod, out CachedMod cachedMod)
{
string key = this.GetKey(site, id);
cachedMod = this.SaveMod(new CachedMod(site, id, mod));
}
/// <summary>Delete data for mods which haven't been requested within a given time limit.</summary>
/// <param name="age">The minimum age for which to remove mods.</param>
public void RemoveStaleMods(TimeSpan age)
{
DateTimeOffset minDate = DateTimeOffset.UtcNow.Subtract(age);
string[] staleKeys = this.Mods
.Where(p => p.Value.LastRequested < minDate)
.Select(p => p.Key)
.ToArray();
foreach (string key in staleKeys)
this.Mods.Remove(key);
}
/// <summary>Save data fetched for a mod.</summary>
/// <param name="mod">The mod data.</param>
public CachedMod SaveMod(CachedMod mod)
{
string key = this.GetKey(mod.Site, mod.ID);
return this.Mods[key] = mod;
}
/*********
** Private methods
*********/
/// <summary>Get a cache key.</summary>
/// <param name="site">The mod site.</param>
/// <param name="id">The mod ID.</param>
public string GetKey(ModRepositoryKey site, string id)
{
return $"{site}:{id.Trim()}".ToLower();
}
}
}

View File

@ -5,8 +5,8 @@ using StardewModdingAPI.Web.Framework.ModRepositories;
namespace StardewModdingAPI.Web.Framework.Caching.Mods
{
/// <summary>Encapsulates logic for accessing the mod data cache.</summary>
internal class ModCacheRepository : BaseCacheRepository, IModCacheRepository
/// <summary>Manages cached mod data in MongoDB.</summary>
internal class ModCacheMongoRepository : BaseCacheRepository, IModCacheRepository
{
/*********
** Fields
@ -20,7 +20,7 @@ namespace StardewModdingAPI.Web.Framework.Caching.Mods
*********/
/// <summary>Construct an instance.</summary>
/// <param name="database">The authenticated MongoDB database.</param>
public ModCacheRepository(IMongoDatabase database)
public ModCacheMongoRepository(IMongoDatabase database)
{
// get collections
this.Mods = database.GetCollection<CachedMod>("mods");
@ -29,6 +29,7 @@ namespace StardewModdingAPI.Web.Framework.Caching.Mods
this.Mods.Indexes.CreateOne(new CreateIndexModel<CachedMod>(Builders<CachedMod>.IndexKeys.Ascending(p => p.ID).Ascending(p => p.Site)));
}
/*********
** Public methods
*********/
@ -75,10 +76,6 @@ namespace StardewModdingAPI.Web.Framework.Caching.Mods
this.Mods.DeleteMany(p => p.LastRequested < minDate);
}
/*********
** Private methods
*********/
/// <summary>Save data fetched for a mod.</summary>
/// <param name="mod">The mod data.</param>
public CachedMod SaveMod(CachedMod mod)
@ -94,6 +91,10 @@ namespace StardewModdingAPI.Web.Framework.Caching.Mods
return mod;
}
/*********
** Private methods
*********/
/// <summary>Normalize a mod ID for case-insensitive search.</summary>
/// <param name="id">The mod ID.</param>
public string NormalizeId(string id)

View File

@ -5,7 +5,7 @@ using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
namespace StardewModdingAPI.Web.Framework.Caching.Wiki
{
/// <summary>Encapsulates logic for accessing the wiki data cache.</summary>
/// <summary>Manages cached wiki data.</summary>
internal interface IWikiCacheRepository : ICacheRepository
{
/*********

View File

@ -0,0 +1,54 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Linq.Expressions;
using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
namespace StardewModdingAPI.Web.Framework.Caching.Wiki
{
/// <summary>Manages cached wiki data in-memory.</summary>
internal class WikiCacheMemoryRepository : BaseCacheRepository, IWikiCacheRepository
{
/*********
** Fields
*********/
/// <summary>The saved wiki metadata.</summary>
private CachedWikiMetadata Metadata;
/// <summary>The cached wiki data.</summary>
private CachedWikiMod[] Mods = new CachedWikiMod[0];
/*********
** Public methods
*********/
/// <summary>Get the cached wiki metadata.</summary>
/// <param name="metadata">The fetched metadata.</param>
public bool TryGetWikiMetadata(out CachedWikiMetadata metadata)
{
metadata = this.Metadata;
return metadata != null;
}
/// <summary>Get the cached wiki mods.</summary>
/// <param name="filter">A filter to apply, if any.</param>
public IEnumerable<CachedWikiMod> GetWikiMods(Expression<Func<CachedWikiMod, bool>> filter = null)
{
return filter != null
? this.Mods.Where(filter.Compile())
: this.Mods.ToArray();
}
/// <summary>Save data fetched from the wiki compatibility list.</summary>
/// <param name="stableVersion">The current stable Stardew Valley version.</param>
/// <param name="betaVersion">The current beta Stardew Valley version.</param>
/// <param name="mods">The mod data.</param>
/// <param name="cachedMetadata">The stored metadata record.</param>
/// <param name="cachedMods">The stored mod records.</param>
public void SaveWikiData(string stableVersion, string betaVersion, IEnumerable<WikiModEntry> mods, out CachedWikiMetadata cachedMetadata, out CachedWikiMod[] cachedMods)
{
this.Metadata = cachedMetadata = new CachedWikiMetadata(stableVersion, betaVersion);
this.Mods = cachedMods = mods.Select(mod => new CachedWikiMod(mod)).ToArray();
}
}
}

View File

@ -7,17 +7,17 @@ using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
namespace StardewModdingAPI.Web.Framework.Caching.Wiki
{
/// <summary>Encapsulates logic for accessing the wiki data cache.</summary>
internal class WikiCacheRepository : BaseCacheRepository, IWikiCacheRepository
/// <summary>Manages cached wiki data in MongoDB.</summary>
internal class WikiCacheMongoRepository : BaseCacheRepository, IWikiCacheRepository
{
/*********
** Fields
*********/
/// <summary>The collection for wiki metadata.</summary>
private readonly IMongoCollection<CachedWikiMetadata> WikiMetadata;
private readonly IMongoCollection<CachedWikiMetadata> Metadata;
/// <summary>The collection for wiki mod data.</summary>
private readonly IMongoCollection<CachedWikiMod> WikiMods;
private readonly IMongoCollection<CachedWikiMod> Mods;
/*********
@ -25,21 +25,21 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki
*********/
/// <summary>Construct an instance.</summary>
/// <param name="database">The authenticated MongoDB database.</param>
public WikiCacheRepository(IMongoDatabase database)
public WikiCacheMongoRepository(IMongoDatabase database)
{
// get collections
this.WikiMetadata = database.GetCollection<CachedWikiMetadata>("wiki-metadata");
this.WikiMods = database.GetCollection<CachedWikiMod>("wiki-mods");
this.Metadata = database.GetCollection<CachedWikiMetadata>("wiki-metadata");
this.Mods = database.GetCollection<CachedWikiMod>("wiki-mods");
// add indexes if needed
this.WikiMods.Indexes.CreateOne(new CreateIndexModel<CachedWikiMod>(Builders<CachedWikiMod>.IndexKeys.Ascending(p => p.ID)));
this.Mods.Indexes.CreateOne(new CreateIndexModel<CachedWikiMod>(Builders<CachedWikiMod>.IndexKeys.Ascending(p => p.ID)));
}
/// <summary>Get the cached wiki metadata.</summary>
/// <param name="metadata">The fetched metadata.</param>
public bool TryGetWikiMetadata(out CachedWikiMetadata metadata)
{
metadata = this.WikiMetadata.Find("{}").FirstOrDefault();
metadata = this.Metadata.Find("{}").FirstOrDefault();
return metadata != null;
}
@ -48,8 +48,8 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki
public IEnumerable<CachedWikiMod> GetWikiMods(Expression<Func<CachedWikiMod, bool>> filter = null)
{
return filter != null
? this.WikiMods.Find(filter).ToList()
: this.WikiMods.Find("{}").ToList();
? this.Mods.Find(filter).ToList()
: this.Mods.Find("{}").ToList();
}
/// <summary>Save data fetched from the wiki compatibility list.</summary>
@ -63,11 +63,11 @@ namespace StardewModdingAPI.Web.Framework.Caching.Wiki
cachedMetadata = new CachedWikiMetadata(stableVersion, betaVersion);
cachedMods = mods.Select(mod => new CachedWikiMod(mod)).ToArray();
this.WikiMods.DeleteMany("{}");
this.WikiMods.InsertMany(cachedMods);
this.Mods.DeleteMany("{}");
this.Mods.InsertMany(cachedMods);
this.WikiMetadata.DeleteMany("{}");
this.WikiMetadata.InsertOne(cachedMetadata);
this.Metadata.DeleteMany("{}");
this.Metadata.InsertOne(cachedMetadata);
}
}
}

View File

@ -1,25 +0,0 @@
namespace StardewModdingAPI.Web.Framework.ConfigModels
{
/// <summary>The config settings for mod compatibility list.</summary>
internal class MongoDbConfig
{
/*********
** Accessors
*********/
/// <summary>The MongoDB connection string.</summary>
public string ConnectionString { get; set; }
/// <summary>The database name.</summary>
public string Database { get; set; }
/*********
** Public method
*********/
/// <summary>Get whether a MongoDB instance is configured.</summary>
public bool IsConfigured()
{
return !string.IsNullOrWhiteSpace(this.ConnectionString);
}
}
}

View File

@ -0,0 +1,18 @@
namespace StardewModdingAPI.Web.Framework.ConfigModels
{
/// <summary>The config settings for cache storage.</summary>
internal class StorageConfig
{
/*********
** Accessors
*********/
/// <summary>The storage mechanism to use.</summary>
public StorageMode Mode { get; set; }
/// <summary>The connection string for the storage mechanism, if applicable.</summary>
public string ConnectionString { get; set; }
/// <summary>The database name for the storage mechanism, if applicable.</summary>
public string Database { get; set; }
}
}

View File

@ -0,0 +1,15 @@
namespace StardewModdingAPI.Web.Framework.ConfigModels
{
/// <summary>Indicates a storage mechanism to use.</summary>
internal enum StorageMode
{
/// <summary>Store data in a hosted MongoDB instance.</summary>
Mongo,
/// <summary>Store data in an in-memory MongoDB instance. This is useful for testing MongoDB storage locally, but will likely fail when deployed since it needs permission to open a local port.</summary>
MongoInMemory,
/// <summary>Store data in-memory. This is suitable for local testing or single-instance servers, but will cause issues when distributed across multiple servers.</summary>
InMemory
}
}

View File

@ -67,12 +67,13 @@ namespace StardewModdingAPI.Web
.Configure<BackgroundServicesConfig>(this.Configuration.GetSection("BackgroundServices"))
.Configure<ModCompatibilityListConfig>(this.Configuration.GetSection("ModCompatibilityList"))
.Configure<ModUpdateCheckConfig>(this.Configuration.GetSection("ModUpdateCheck"))
.Configure<MongoDbConfig>(this.Configuration.GetSection("MongoDB"))
.Configure<StorageConfig>(this.Configuration.GetSection("Storage"))
.Configure<SiteConfig>(this.Configuration.GetSection("Site"))
.Configure<RouteOptions>(options => options.ConstraintMap.Add("semanticVersion", typeof(VersionConstraint)))
.AddLogging()
.AddMemoryCache();
MongoDbConfig mongoConfig = this.Configuration.GetSection("MongoDB").Get<MongoDbConfig>();
StorageConfig storageConfig = this.Configuration.GetSection("Storage").Get<StorageConfig>();
StorageMode storageMode = storageConfig.Mode;
// init MVC
services
@ -82,44 +83,66 @@ namespace StardewModdingAPI.Web
services
.AddRazorPages();
// init MongoDB
services.AddSingleton<MongoDbRunner>(_ => !mongoConfig.IsConfigured()
? MongoDbRunner.Start()
: throw new InvalidOperationException("The MongoDB connection is configured, so the local development version should not be used.")
);
services.AddSingleton<IMongoDatabase>(serv =>
// init storage
switch (storageMode)
{
// get connection string
string connectionString = mongoConfig.IsConfigured()
? mongoConfig.ConnectionString
: serv.GetRequiredService<MongoDbRunner>().ConnectionString;
case StorageMode.InMemory:
services.AddSingleton<IModCacheRepository>(new ModCacheMemoryRepository());
services.AddSingleton<IWikiCacheRepository>(new WikiCacheMemoryRepository());
break;
// get client
BsonSerializer.RegisterSerializer(new UtcDateTimeOffsetSerializer());
return new MongoClient(connectionString).GetDatabase(mongoConfig.Database);
});
services.AddSingleton<IModCacheRepository>(serv => new ModCacheRepository(serv.GetRequiredService<IMongoDatabase>()));
services.AddSingleton<IWikiCacheRepository>(serv => new WikiCacheRepository(serv.GetRequiredService<IMongoDatabase>()));
case StorageMode.Mongo:
case StorageMode.MongoInMemory:
{
// local MongoDB instance
services.AddSingleton<MongoDbRunner>(_ => storageMode == StorageMode.MongoInMemory
? MongoDbRunner.Start()
: throw new NotSupportedException($"The in-memory MongoDB runner isn't available in storage mode {storageMode}.")
);
// MongoDB
services.AddSingleton<IMongoDatabase>(serv =>
{
BsonSerializer.RegisterSerializer(new UtcDateTimeOffsetSerializer());
return new MongoClient(this.GetMongoDbConnectionString(serv, storageConfig))
.GetDatabase(storageConfig.Database);
});
// repositories
services.AddSingleton<IModCacheRepository>(serv => new ModCacheMongoRepository(serv.GetRequiredService<IMongoDatabase>()));
services.AddSingleton<IWikiCacheRepository>(serv => new WikiCacheMongoRepository(serv.GetRequiredService<IMongoDatabase>()));
}
break;
default:
throw new NotSupportedException($"Unhandled storage mode '{storageMode}'.");
}
// init Hangfire
services
.AddHangfire(config =>
.AddHangfire((serv, config) =>
{
config
.SetDataCompatibilityLevel(CompatibilityLevel.Version_170)
.UseSimpleAssemblyNameTypeSerializer()
.UseRecommendedSerializerSettings();
if (mongoConfig.IsConfigured())
switch (storageMode)
{
config.UseMongoStorage(MongoClientSettings.FromConnectionString(mongoConfig.ConnectionString), $"{mongoConfig.Database}-hangfire", new MongoStorageOptions
{
MigrationOptions = new MongoMigrationOptions(MongoMigrationStrategy.Drop),
CheckConnection = false // error on startup takes down entire process
});
case StorageMode.InMemory:
config.UseMemoryStorage();
break;
case StorageMode.MongoInMemory:
case StorageMode.Mongo:
string connectionString = this.GetMongoDbConnectionString(serv, storageConfig);
config.UseMongoStorage(MongoClientSettings.FromConnectionString(connectionString), $"{storageConfig.Database}-hangfire", new MongoStorageOptions
{
MigrationOptions = new MongoMigrationOptions(MongoMigrationStrategy.Drop),
CheckConnection = false // error on startup takes down entire process
});
break;
}
else
config.UseMemoryStorage();
});
// init background service
@ -140,6 +163,7 @@ namespace StardewModdingAPI.Web
baseUrl: api.ChucklefishBaseUrl,
modPageUrlFormat: api.ChucklefishModPageUrlFormat
));
services.AddSingleton<ICurseForgeClient>(new CurseForgeClient(
userAgent: userAgent,
apiUrl: api.CurseForgeBaseUrl
@ -229,6 +253,20 @@ namespace StardewModdingAPI.Web
settings.NullValueHandling = NullValueHandling.Ignore;
}
/// <summary>Get the MongoDB connection string for the given storage configuration.</summary>
/// <param name="services">The service provider.</param>
/// <param name="storageConfig">The storage configuration</param>
/// <exception cref="NotSupportedException">There's no MongoDB instance in the given storage mode.</exception>
private string GetMongoDbConnectionString(IServiceProvider services, StorageConfig storageConfig)
{
return storageConfig.Mode switch
{
StorageMode.Mongo => storageConfig.ConnectionString,
StorageMode.MongoInMemory => services.GetRequiredService<MongoDbRunner>().ConnectionString,
_ => throw new NotSupportedException($"There's no MongoDB instance in storage mode {storageConfig.Mode}.")
};
}
/// <summary>Get the redirect rules to apply.</summary>
private RewriteOptions GetRedirectRules()
{

View File

@ -17,7 +17,8 @@
"NexusApiKey": null
},
"MongoDB": {
"Storage": {
"Mode": "MongoInMemory",
"ConnectionString": null,
"Database": "smapi-edge"
},

View File

@ -49,7 +49,8 @@
"PastebinBaseUrl": "https://pastebin.com/"
},
"MongoDB": {
"Storage": {
"Mode": "InMemory",
"ConnectionString": null,
"Database": "smapi"
},