expand metadata fetched from the wiki (#597)

This commit is contained in:
Jesse Plamondon-Willard 2018-10-20 14:55:13 -04:00
parent e09499f628
commit f09befe240
7 changed files with 162 additions and 134 deletions

View File

@ -90,7 +90,7 @@ namespace StardewModdingAPI.Web.Controllers
return new ModEntryModel[0];
// fetch wiki data
WikiCompatibilityEntry[] wikiData = await this.GetWikiDataAsync();
WikiModEntry[] wikiData = await this.GetWikiDataAsync();
IDictionary<string, ModEntryModel> mods = new Dictionary<string, ModEntryModel>(StringComparer.CurrentCultureIgnoreCase);
foreach (ModSearchEntryModel mod in model.Mods)
{
@ -114,11 +114,11 @@ namespace StardewModdingAPI.Web.Controllers
/// <param name="wikiData">The wiki data.</param>
/// <param name="includeExtendedMetadata">Whether to include extended metadata for each mod.</param>
/// <returns>Returns the mod data if found, else <c>null</c>.</returns>
private async Task<ModEntryModel> GetModData(ModSearchEntryModel search, WikiCompatibilityEntry[] wikiData, bool includeExtendedMetadata)
private async Task<ModEntryModel> GetModData(ModSearchEntryModel search, WikiModEntry[] wikiData, bool includeExtendedMetadata)
{
// crossreference data
ModDataRecord record = this.ModDatabase.Get(search.ID);
WikiCompatibilityEntry wikiEntry = wikiData.FirstOrDefault(entry => entry.ID.Contains(search.ID.Trim(), StringComparer.InvariantCultureIgnoreCase));
WikiModEntry wikiEntry = wikiData.FirstOrDefault(entry => entry.ID.Contains(search.ID.Trim(), StringComparer.InvariantCultureIgnoreCase));
string[] updateKeys = this.GetUpdateKeys(search.UpdateKeys, record, wikiEntry).ToArray();
// get latest versions
@ -162,19 +162,19 @@ namespace StardewModdingAPI.Web.Controllers
}
// get unofficial version
if (wikiEntry?.UnofficialVersion != null && this.IsNewer(wikiEntry.UnofficialVersion, result.Main?.Version) && this.IsNewer(wikiEntry.UnofficialVersion, result.Optional?.Version))
result.Unofficial = new ModEntryVersionModel(wikiEntry.UnofficialVersion, this.WikiCompatibilityPageUrl);
if (wikiEntry?.Compatibility.UnofficialVersion != null && this.IsNewer(wikiEntry.Compatibility.UnofficialVersion, result.Main?.Version) && this.IsNewer(wikiEntry.Compatibility.UnofficialVersion, result.Optional?.Version))
result.Unofficial = new ModEntryVersionModel(wikiEntry.Compatibility.UnofficialVersion, this.WikiCompatibilityPageUrl);
// get unofficial version for beta
if (wikiEntry?.HasBetaInfo == true)
{
result.HasBetaInfo = true;
if (wikiEntry.BetaStatus == WikiCompatibilityStatus.Unofficial)
if (wikiEntry.BetaCompatibility.Status == WikiCompatibilityStatus.Unofficial)
{
if (wikiEntry.BetaUnofficialVersion != null)
if (wikiEntry.BetaCompatibility.UnofficialVersion != null)
{
result.UnofficialForBeta = (wikiEntry.BetaUnofficialVersion != null && this.IsNewer(wikiEntry.BetaUnofficialVersion, result.Main?.Version) && this.IsNewer(wikiEntry.BetaUnofficialVersion, result.Optional?.Version))
? new ModEntryVersionModel(wikiEntry.BetaUnofficialVersion, this.WikiCompatibilityPageUrl)
result.UnofficialForBeta = (wikiEntry.BetaCompatibility.UnofficialVersion != null && this.IsNewer(wikiEntry.BetaCompatibility.UnofficialVersion, result.Main?.Version) && this.IsNewer(wikiEntry.BetaCompatibility.UnofficialVersion, result.Optional?.Version))
? new ModEntryVersionModel(wikiEntry.BetaCompatibility.UnofficialVersion, this.WikiCompatibilityPageUrl)
: null;
}
else
@ -216,21 +216,21 @@ namespace StardewModdingAPI.Web.Controllers
}
/// <summary>Get mod data from the wiki compatibility list.</summary>
private async Task<WikiCompatibilityEntry[]> GetWikiDataAsync()
private async Task<WikiModEntry[]> GetWikiDataAsync()
{
ModToolkit toolkit = new ModToolkit();
return await this.Cache.GetOrCreateAsync("_wiki", async entry =>
{
try
{
WikiCompatibilityEntry[] entries = await toolkit.GetWikiCompatibilityListAsync();
WikiModEntry[] entries = await toolkit.GetWikiCompatibilityListAsync();
entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.SuccessCacheMinutes);
return entries;
}
catch
{
entry.AbsoluteExpiration = DateTimeOffset.UtcNow.AddMinutes(this.ErrorCacheMinutes);
return new WikiCompatibilityEntry[0];
return new WikiModEntry[0];
}
});
}
@ -268,7 +268,7 @@ namespace StardewModdingAPI.Web.Controllers
/// <param name="specifiedKeys">The specified update keys.</param>
/// <param name="record">The mod's entry in SMAPI's internal database.</param>
/// <param name="entry">The mod's entry in the wiki list.</param>
public IEnumerable<string> GetUpdateKeys(string[] specifiedKeys, ModDataRecord record, WikiCompatibilityEntry entry)
public IEnumerable<string> GetUpdateKeys(string[] specifiedKeys, ModDataRecord record, WikiModEntry entry)
{
IEnumerable<string> GetRaw()
{

View File

@ -68,7 +68,7 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.WebApi
/// <summary>Construct an instance.</summary>
/// <param name="wiki">The mod metadata from the wiki (if available).</param>
/// <param name="db">The mod metadata from SMAPI's internal DB (if available).</param>
public ModExtendedMetadataModel(WikiCompatibilityEntry wiki, ModDataRecord db)
public ModExtendedMetadataModel(WikiModEntry wiki, ModDataRecord db)
{
// wiki data
if (wiki != null)
@ -81,11 +81,11 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.WebApi
this.CustomSourceUrl = wiki.CustomSourceUrl;
this.CustomUrl = wiki.CustomUrl;
this.CompatibilityStatus = wiki.Status;
this.CompatibilitySummary = wiki.Summary;
this.CompatibilityStatus = wiki.Compatibility.Status;
this.CompatibilitySummary = wiki.Compatibility.Summary;
this.BetaCompatibilityStatus = wiki.BetaStatus;
this.BetaCompatibilitySummary = wiki.BetaSummary;
this.BetaCompatibilityStatus = wiki.BetaCompatibility?.Status;
this.BetaCompatibilitySummary = wiki.BetaCompatibility?.Summary;
}
// internal DB data

View File

@ -30,7 +30,7 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki
}
/// <summary>Fetch mod compatibility entries.</summary>
public async Task<WikiCompatibilityEntry[]> FetchAsync()
public async Task<WikiModEntry[]> FetchAsync()
{
// fetch HTML
ResponseModel response = await this.Client
@ -69,100 +69,113 @@ namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki
*********/
/// <summary>Parse valid mod compatibility entries.</summary>
/// <param name="nodes">The HTML compatibility entries.</param>
private IEnumerable<WikiCompatibilityEntry> ParseEntries(IEnumerable<HtmlNode> nodes)
private IEnumerable<WikiModEntry> ParseEntries(IEnumerable<HtmlNode> nodes)
{
foreach (HtmlNode node in nodes)
{
// parse mod info
string name = node.Descendants("td").FirstOrDefault()?.Descendants("a")?.FirstOrDefault()?.InnerText?.Trim();
string[] ids = this.GetAttribute(node, "data-id")?.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToArray() ?? new string[0];
int? nexusID = this.GetNullableIntAttribute(node, "data-nexus-id");
int? chucklefishID = this.GetNullableIntAttribute(node, "data-chucklefish-id");
string githubRepo = this.GetAttribute(node, "data-github");
string customSourceUrl = this.GetAttribute(node, "data-custom-source");
string customUrl = this.GetAttribute(node, "data-custom-url");
// extract fields
string name = this.GetMetadataField(node, "mod-name");
string alternateNames = this.GetMetadataField(node, "mod-name2");
string author = this.GetMetadataField(node, "mod-author");
string alternateAuthors = this.GetMetadataField(node, "mod-author2");
string[] ids = this.GetMetadataField(node, "mod-id")?.Split(new[] { ',' }, StringSplitOptions.RemoveEmptyEntries).Select(p => p.Trim()).ToArray() ?? new string[0];
int? nexusID = this.GetNullableIntField(node, "mod-nexus-id");
int? chucklefishID = this.GetNullableIntField(node, "mod-chucklefish-id");
string githubRepo = this.GetMetadataField(node, "mod-github");
string customSourceUrl = this.GetMetadataField(node, "mod-custom-source");
string customUrl = this.GetMetadataField(node, "mod-url");
string brokeIn = this.GetMetadataField(node, "mod-broke-in");
string anchor = this.GetMetadataField(node, "mod-anchor");
// parse stable compatibility
WikiCompatibilityStatus status = this.GetStatusAttribute(node, "data-status") ?? WikiCompatibilityStatus.Ok;
ISemanticVersion unofficialVersion = this.GetSemanticVersionAttribute(node, "data-unofficial-version");
string summary = node.Descendants().FirstOrDefault(p => p.HasClass("data-summary"))?.InnerText.Trim();
WikiCompatibilityInfo compatibility = new WikiCompatibilityInfo
{
Status = this.GetStatusField(node, "mod-status") ?? WikiCompatibilityStatus.Ok,
UnofficialVersion = this.GetSemanticVersionField(node, "mod-unofficial-version"),
UnofficialUrl = this.GetMetadataField(node, "mod-unofficial-url"),
Summary = this.GetMetadataField(node, "mod-summary")?.Trim()
};
// parse beta compatibility
WikiCompatibilityStatus? betaStatus = this.GetStatusAttribute(node, "data-beta-status");
ISemanticVersion betaUnofficialVersion = betaStatus.HasValue ? this.GetSemanticVersionAttribute(node, "data-beta-unofficial-version") : null;
string betaSummary = betaStatus.HasValue ? node.Descendants().FirstOrDefault(p => p.HasClass("data-beta-summary"))?.InnerText.Trim() : null;
WikiCompatibilityInfo betaCompatibility = null;
{
WikiCompatibilityStatus? betaStatus = this.GetStatusField(node, "mod-beta-status");
if (betaStatus.HasValue)
{
betaCompatibility = new WikiCompatibilityInfo
{
Status = betaStatus.Value,
UnofficialVersion = this.GetSemanticVersionField(node, "mod-beta-unofficial-version"),
UnofficialUrl = this.GetMetadataField(node, "mod-beta-unofficial-url"),
Summary = this.GetMetadataField(node, "mod-beta-summary")
};
}
}
// yield model
yield return new WikiCompatibilityEntry
yield return new WikiModEntry
{
// mod info
ID = ids,
Name = name,
AlternateNames = alternateNames,
Author = author,
AlternateAuthors = alternateAuthors,
NexusID = nexusID,
ChucklefishID = chucklefishID,
GitHubRepo = githubRepo,
CustomSourceUrl = customSourceUrl,
CustomUrl = customUrl,
// stable compatibility
Status = status,
Summary = summary,
UnofficialVersion = unofficialVersion,
// beta compatibility
BetaStatus = betaStatus,
BetaSummary = betaSummary,
BetaUnofficialVersion = betaUnofficialVersion
BrokeIn = brokeIn,
Compatibility = compatibility,
BetaCompatibility = betaCompatibility,
Anchor = anchor
};
}
}
/// <summary>Get a compatibility status attribute value.</summary>
/// <param name="node">The HTML node.</param>
/// <param name="attributeName">The attribute name.</param>
private WikiCompatibilityStatus? GetStatusAttribute(HtmlNode node, string attributeName)
/// <summary>Get the value of a metadata field.</summary>
/// <param name="container">The metadata container.</param>
/// <param name="name">The field name.</param>
private string GetMetadataField(HtmlNode container, string name)
{
string raw = node.GetAttributeValue(attributeName, null);
return container.Descendants().FirstOrDefault(p => p.HasClass(name))?.InnerHtml;
}
/// <summary>Get the value of a metadata field as a compatibility status.</summary>
/// <param name="container">The metadata container.</param>
/// <param name="name">The field name.</param>
private WikiCompatibilityStatus? GetStatusField(HtmlNode container, string name)
{
string raw = this.GetMetadataField(container, name);
if (raw == null)
return null; // not a mod node?
return null;
if (!Enum.TryParse(raw, true, out WikiCompatibilityStatus status))
throw new InvalidOperationException($"Unknown status '{raw}' when parsing compatibility list.");
return status;
}
/// <summary>Get a semantic version attribute value.</summary>
/// <param name="node">The HTML node.</param>
/// <param name="attributeName">The attribute name.</param>
private ISemanticVersion GetSemanticVersionAttribute(HtmlNode node, string attributeName)
/// <summary>Get the value of a metadata field as a semantic version.</summary>
/// <param name="container">The metadata container.</param>
/// <param name="name">The field name.</param>
private ISemanticVersion GetSemanticVersionField(HtmlNode container, string name)
{
string raw = node.GetAttributeValue(attributeName, null);
string raw = this.GetMetadataField(container, name);
return SemanticVersion.TryParse(raw, out ISemanticVersion version)
? version
: null;
}
/// <summary>Get a nullable integer attribute value.</summary>
/// <param name="node">The HTML node.</param>
/// <param name="attributeName">The attribute name.</param>
private int? GetNullableIntAttribute(HtmlNode node, string attributeName)
/// <summary>Get the value of a metadata field as a nullable integer.</summary>
/// <param name="container">The metadata container.</param>
/// <param name="name">The field name.</param>
private int? GetNullableIntField(HtmlNode container, string name)
{
string raw = this.GetAttribute(node, attributeName);
string raw = this.GetMetadataField(container, name);
if (raw != null && int.TryParse(raw, out int value))
return value;
return null;
}
/// <summary>Get a strings attribute value.</summary>
/// <param name="node">The HTML node.</param>
/// <param name="attributeName">The attribute name.</param>
private string GetAttribute(HtmlNode node, string attributeName)
{
string raw = node.GetAttributeValue(attributeName, null);
if (raw != null)
raw = HtmlEntity.DeEntitize(raw);
return raw;
}
/// <summary>The response model for the MediaWiki parse API.</summary>
[SuppressMessage("ReSharper", "ClassNeverInstantiated.Local")]
[SuppressMessage("ReSharper", "UnusedAutoPropertyAccessor.Local")]

View File

@ -1,60 +0,0 @@
namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki
{
/// <summary>An entry in the mod compatibility list.</summary>
public class WikiCompatibilityEntry
{
/*********
** Accessors
*********/
/****
** Mod info
****/
/// <summary>The mod's unique ID. A mod may have multiple current IDs in rare cases (e.g. due to parallel releases or unofficial updates).</summary>
public string[] ID { get; set; }
/// <summary>The mod's display name.</summary>
public string Name { get; set; }
/// <summary>The mod ID on Nexus.</summary>
public int? NexusID { get; set; }
/// <summary>The mod ID in the Chucklefish mod repo.</summary>
public int? ChucklefishID { get; set; }
/// <summary>The GitHub repository in the form 'owner/repo'.</summary>
public string GitHubRepo { get; set; }
/// <summary>The URL to a non-GitHub source repo.</summary>
public string CustomSourceUrl { get; set; }
/// <summary>The custom mod page URL (if applicable).</summary>
public string CustomUrl { get; set; }
/****
** Stable compatibility
****/
/// <summary>The compatibility status.</summary>
public WikiCompatibilityStatus Status { get; set; }
/// <summary>The human-readable summary of the compatibility status or workaround, without HTML formatting.</summary>
public string Summary { get; set; }
/// <summary>The version of the latest unofficial update, if applicable.</summary>
public ISemanticVersion UnofficialVersion { get; set; }
/****
** Beta compatibility
****/
/// <summary>Whether a Stardew Valley or SMAPI beta which affects mod compatibility is in progress. If this is true, <see cref="BetaUnofficialVersion"/> should be used for beta versions of SMAPI instead of <see cref="UnofficialVersion"/>.</summary>
public bool HasBetaInfo => this.BetaStatus != null;
/// <summary>The compatibility status for the Stardew Valley beta, if a beta is in progress.</summary>
public WikiCompatibilityStatus? BetaStatus { get; set; }
/// <summary>The human-readable summary of the compatibility status or workaround for the Stardew Valley beta (if any), without HTML formatting.</summary>
public string BetaSummary { get; set; }
/// <summary>The version of the latest unofficial update for the Stardew Valley beta (if any), if applicable.</summary>
public ISemanticVersion BetaUnofficialVersion { get; set; }
}
}

View File

@ -0,0 +1,21 @@
namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki
{
/// <summary>Compatibility info for a mod.</summary>
public class WikiCompatibilityInfo
{
/*********
** Accessors
*********/
/// <summary>The compatibility status.</summary>
public WikiCompatibilityStatus Status { get; set; }
/// <summary>The human-readable summary of the compatibility status or workaround, without HTML formatting.</summary>
public string Summary { get; set; }
/// <summary>The version of the latest unofficial update, if applicable.</summary>
public ISemanticVersion UnofficialVersion { get; set; }
/// <summary>The URL to the latest unofficial update, if applicable.</summary>
public string UnofficialUrl { get; set; }
}
}

View File

@ -0,0 +1,54 @@
namespace StardewModdingAPI.Toolkit.Framework.Clients.Wiki
{
/// <summary>A mod entry in the wiki list.</summary>
public class WikiModEntry
{
/*********
** Accessors
*********/
/// <summary>The mod's unique ID. A mod may have multiple current IDs in rare cases (e.g. due to parallel releases or unofficial updates).</summary>
public string[] ID { get; set; }
/// <summary>The mod's display name.</summary>
public string Name { get; set; }
/// <summary>The mod's alternative names, if any.</summary>
public string AlternateNames { get; set; }
/// <summary>The mod's author name.</summary>
public string Author { get; set; }
/// <summary>The mod's alternative author names, if any.</summary>
public string AlternateAuthors { get; set; }
/// <summary>The mod ID on Nexus.</summary>
public int? NexusID { get; set; }
/// <summary>The mod ID in the Chucklefish mod repo.</summary>
public int? ChucklefishID { get; set; }
/// <summary>The GitHub repository in the form 'owner/repo'.</summary>
public string GitHubRepo { get; set; }
/// <summary>The URL to a non-GitHub source repo.</summary>
public string CustomSourceUrl { get; set; }
/// <summary>The custom mod page URL (if applicable).</summary>
public string CustomUrl { get; set; }
/// <summary>The game or SMAPI version which broke this mod (if applicable).</summary>
public string BrokeIn { get; set; }
/// <summary>The mod's compatibility with the latest stable version of the game.</summary>
public WikiCompatibilityInfo Compatibility { get; set; }
/// <summary>The mod's compatibility with the latest beta version of the game (if any).</summary>
public WikiCompatibilityInfo BetaCompatibility { get; set; }
/// <summary>Whether a Stardew Valley or SMAPI beta which affects mod compatibility is in progress. If this is true, <see cref="BetaCompatibility"/> should be used for beta versions of SMAPI instead of <see cref="Compatibility"/>.</summary>
public bool HasBetaInfo => this.BetaCompatibility != null;
/// <summary>The link anchor for the mod entry in the wiki compatibility list.</summary>
public string Anchor { get; set; }
}
}

View File

@ -47,7 +47,7 @@ namespace StardewModdingAPI.Toolkit
}
/// <summary>Extract mod metadata from the wiki compatibility list.</summary>
public async Task<WikiCompatibilityEntry[]> GetWikiCompatibilityListAsync()
public async Task<WikiModEntry[]> GetWikiCompatibilityListAsync()
{
var client = new WikiCompatibilityClient(this.UserAgent);
return await client.FetchAsync();