diff --git a/src/BotTerminator.cs b/src/BotTerminator.cs index c13c894..63dbc0f 100644 --- a/src/BotTerminator.cs +++ b/src/BotTerminator.cs @@ -76,7 +76,7 @@ public async Task StartAsync() return; } UserLookup = new WikiBotDatabase(await RedditInstance.GetSubredditAsync(SubredditName, false)); - await UserLookup.CheckUserAsync(CacheFreshenerUserName); + await UserLookup.CheckUserAsync(CacheFreshenerUserName, String.Empty); await UpdateSubredditCacheAsync(); this.Modules = new List() @@ -92,7 +92,14 @@ public async Task StartAsync() public async Task CacheSubredditAsync(String subredditName) { - SubredditLookup[subredditName] = new CachedSubreddit(await RedditInstance.GetSubredditAsync(subredditName, false), configurationLoader); + Subreddit subreddit = null; + // we have to do it this way, otherwise ModPermissions won't be set + await RedditInstance.User.GetModeratorSubreddits(-1).ForEachAsync(moderatedSubreddit => + { + if (moderatedSubreddit.DisplayName == SubredditName) subreddit = moderatedSubreddit; + }); + SubredditLookup[subredditName] = new CachedSubreddit(subreddit, configurationLoader); + await SubredditLookup[subredditName].ReloadOptionsAsync(this); } public async Task UpdateSubredditCacheAsync() @@ -119,11 +126,11 @@ await RedditInstance.User.GetModeratorSubreddits(-1).ForEachAsync(subreddit => } } - } + }/* else { - SubredditLookup[subreddit.DisplayName] = new CachedSubreddit(subreddit, configurationLoader); - } + await SubredditLookup[subreddit.DisplayName].ReloadOptionsAsync(this); + }*/ } await Task.WhenAll(moderatedSubreddits.Select(subreddit => SubredditLookup[subreddit.DisplayName].ReloadOptionsAsync(this))); } @@ -146,7 +153,7 @@ public bool IsConfigurable(Subreddit subreddit) /// A value determining whether the user is not bannable on Reddit. public static Boolean IsUnbannable(ModeratableThing thing) => String.IsNullOrWhiteSpace(thing?.AuthorName) || thing?.AuthorName == DeletedUserName; - internal async Task CheckShouldBanAsync(ModeratableThing thing) + internal async Task CheckShouldBanAsync(ModeratableThing thing, IEnumerable bannedGroups) { if (IsUnbannable(thing)) return false; @@ -159,7 +166,14 @@ internal async Task CheckShouldBanAsync(ModeratableThing thing) return false; } } - return await UserLookup.CheckUserAsync(thing.AuthorName); + IEnumerable groupNames = (await UserLookup.GetGroupsForUserAsync(thing.AuthorName)).Select(group => group.Name); + return groupNames.Any(groupName => bannedGroups.Contains(groupName)); + } + + internal async Task> GetBannedGroupsAsync(AbstractSubredditOptionSet options) + { + IReadOnlyCollection actioned = options.ActionedUserTypes.ToHashSet(); + return actioned.Count == 0 ? await UserLookup.GetDefaultBannedGroupsAsync() : (await UserLookup.GetAllGroupsAsync()).Values.Where(group => actioned.Contains(group.Name)).ToArray(); } internal async Task QuarantineOptInAsync(String subredditName) diff --git a/src/Configuration/AbstractSubredditOptionSet.cs b/src/Configuration/AbstractSubredditOptionSet.cs index e95cf36..cfa5b19 100644 --- a/src/Configuration/AbstractSubredditOptionSet.cs +++ b/src/Configuration/AbstractSubredditOptionSet.cs @@ -32,6 +32,9 @@ public abstract class AbstractSubredditOptionSet [JsonProperty("banDuration", Required = Required.DisallowNull)] public abstract Int32 BanDuration { get; set; } + [JsonProperty("actionedUserTypes")] + public abstract IEnumerable ActionedUserTypes { get; set; } + [JsonProperty("ignoredUsers")] public abstract IEnumerable IgnoredUsers { get; set; } diff --git a/src/Configuration/BanListConfig.cs b/src/Configuration/BanListConfig.cs index 926be3b..400a54c 100644 --- a/src/Configuration/BanListConfig.cs +++ b/src/Configuration/BanListConfig.cs @@ -1,13 +1,60 @@ -using Newtonsoft.Json; +using BotTerminator.Models; +using Newtonsoft.Json; +using RedditSharp.Things; using System; using System.Collections.Generic; +using System.Linq; namespace BotTerminator.Configuration { [JsonObject] public class BanListConfig : BotConfig { - [JsonProperty("bannedUsers")] - public ISet Items { get; set; } = new HashSet(); + private const Int32 CurrentConfigVersion = 2; + + public BanListConfig(int version = CurrentConfigVersion) + { + this.Version = version; + } + + [JsonProperty("nonGroupFlairCssClasses", DefaultValueHandling = DefaultValueHandling.Populate)] + public IReadOnlyCollection NonGroupFlairCssClasses { get; set; } = Array.Empty(); + + [JsonProperty("groups")] + public Dictionary GroupLookup { get; set; } = new Dictionary(); + + [JsonIgnore] + public Int32 Count => GroupLookup.Values.Sum(group => group.Members.Count); + + public IEnumerable GetAllNames() + { + return NonGroupFlairCssClasses.Concat(GroupLookup.Keys); + } + + public IReadOnlyCollection GetDefaultActionedOnGroups() + { + return GroupLookup.Values.Where(group => group.ActionByDefault).ToArray(); + } + + public IReadOnlyCollection GetGroupsByUser(String username) + { + return GroupLookup.Values.Where(group => group.Members.Contains(username)).ToArray(); + } + + public bool IsInGroup(String groupCssClass, String username) + { + return GroupLookup.ContainsKey(groupCssClass) && GroupLookup[groupCssClass].Members.Contains(username); + } + + public bool IsInAnyGroup(String username) + { + return GroupLookup.Values.Any(group => group.Members.Contains(username)); + } + + public bool ShouldHide(Post post) + { + String cssClass = post.LinkFlairCssClass; + return IsInAnyGroup(cssClass) || NonGroupFlairCssClasses.Contains(cssClass); + } } } diff --git a/src/Configuration/ShadedOptionSet.cs b/src/Configuration/ShadedOptionSet.cs index e452d31..7c67c73 100644 --- a/src/Configuration/ShadedOptionSet.cs +++ b/src/Configuration/ShadedOptionSet.cs @@ -99,5 +99,15 @@ public override RemovalType RemovalType } set => throw new NotSupportedException(operationNotSupportedMessage); } + + public override IEnumerable ActionedUserTypes + { + get + { + return optionSets.First(optionSet => optionSet.ActionedUserTypes != null).ActionedUserTypes; + // TODO: somehow respect enumerablesAdditive + } + set => throw new NotSupportedException(operationNotSupportedMessage); + } } } diff --git a/src/Configuration/SubredditOptionSet.cs b/src/Configuration/SubredditOptionSet.cs index 677819c..32bd8b5 100644 --- a/src/Configuration/SubredditOptionSet.cs +++ b/src/Configuration/SubredditOptionSet.cs @@ -40,8 +40,12 @@ public SubredditOptionSet(AbstractSubredditOptionSet toCopyFrom) public override Int32 BanDuration { get; set; } = 0; + // TODO: can we use a set here? + public override IEnumerable IgnoredUsers { get; set; } = new List(0); + public override IEnumerable ActionedUserTypes { get; set; } + public override RemovalType RemovalType { get; set; } = RemovalType.Spam; } } diff --git a/src/Data/IBotDatabase.cs b/src/Data/IBotDatabase.cs index 9aa067f..6b2dfde 100644 --- a/src/Data/IBotDatabase.cs +++ b/src/Data/IBotDatabase.cs @@ -1,4 +1,6 @@ -using System; +using BotTerminator.Configuration; +using BotTerminator.Models; +using System; using System.Collections.Generic; using System.Linq; using System.Text; @@ -8,8 +10,16 @@ namespace BotTerminator.Data { public interface IBotDatabase { - Task CheckUserAsync(String name); + Task GetConfigAsync(); - Task UpdateUserAsync(String name, Boolean value, Boolean force = false); + Task> GetAllGroupsAsync(); + + Task> GetGroupsForUserAsync(String name); + + Task> GetDefaultBannedGroupsAsync(); + + Task CheckUserAsync(String username, String groupName); + + Task UpdateUserAsync(String username, String groupName, Boolean value, Boolean force = false); } } diff --git a/src/Data/MemoryBotDatabase.cs b/src/Data/MemoryBotDatabase.cs index 2448df7..f774fcd 100644 --- a/src/Data/MemoryBotDatabase.cs +++ b/src/Data/MemoryBotDatabase.cs @@ -3,24 +3,34 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using BotTerminator.Configuration; +using BotTerminator.Models; namespace BotTerminator.Data { public class MemoryBotDatabase : IBotDatabase { - private HashSet Users { get; set; } = new HashSet(); + private BanListConfig Users { get; set; } = new BanListConfig(); - public Task CheckUserAsync(String name) => Task.FromResult(Users.Contains(name)); + public Task GetConfigAsync() => Task.FromResult(Users); - public Task UpdateUserAsync(String name, Boolean value, Boolean force) + public Task> GetAllGroupsAsync() => Task.FromResult>(Users.GroupLookup); + + public Task CheckUserAsync(String username, String groupName) => Task.FromResult(Users.IsInGroup(groupName, username)); + + public Task> GetDefaultBannedGroupsAsync() => Task.FromResult(Users.GroupLookup.Where(group => group.Value.ActionByDefault).ToList() as IReadOnlyCollection); + + public Task> GetGroupsForUserAsync(String username) => Task.FromResult(Users.GetGroupsByUser(username)); + + public Task UpdateUserAsync(String name, String groupName, Boolean value, Boolean force = false) { if (value) { - Users.Add(name); + Users.GroupLookup[groupName].Members.Add(name); } else { - Users.Remove(name); + Users.GroupLookup[groupName].Members.Remove(name); } return Task.CompletedTask; } diff --git a/src/Data/NullBotDatabase.cs b/src/Data/NullBotDatabase.cs index c453f07..a47a4c0 100644 --- a/src/Data/NullBotDatabase.cs +++ b/src/Data/NullBotDatabase.cs @@ -3,13 +3,23 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using BotTerminator.Configuration; +using BotTerminator.Models; namespace BotTerminator.Data { public class NullBotDatabase : IBotDatabase { - public Task CheckUserAsync(String name) => Task.FromResult(false); + public Task CheckUserAsync(String username, String groupName) => Task.FromResult(false); - public Task UpdateUserAsync(String name, Boolean value, Boolean force) => Task.CompletedTask; + public Task GetConfigAsync() => Task.FromResult(new BanListConfig()); + + public Task> GetAllGroupsAsync() => Task.FromResult>(new Dictionary()); + + public Task> GetDefaultBannedGroupsAsync() => Task.FromResult(Array.Empty() as IReadOnlyCollection); + + public Task> GetGroupsForUserAsync(String name) => Task.FromResult(Array.Empty() as IReadOnlyCollection); + + public Task UpdateUserAsync(String username, String groupName, Boolean value, Boolean force = false) => Task.CompletedTask; } } diff --git a/src/Data/SQLiteBotDatabase.cs b/src/Data/SQLiteBotDatabase.cs index d7cff43..4a56fa2 100644 --- a/src/Data/SQLiteBotDatabase.cs +++ b/src/Data/SQLiteBotDatabase.cs @@ -3,6 +3,8 @@ using System.Linq; using System.Text; using System.Threading.Tasks; +using BotTerminator.Configuration; +using BotTerminator.Models; namespace BotTerminator.Data { @@ -13,9 +15,39 @@ public Task CheckUserAsync(String name) throw new NotImplementedException(); } + public Task CheckUserAsync(String username, String groupName) + { + throw new NotImplementedException(); + } + + public Task> GetAllGroupsAsync() + { + throw new NotImplementedException(); + } + + public Task GetConfigAsync() + { + throw new NotImplementedException(); + } + + public Task> GetDefaultBannedGroupsAsync() + { + throw new NotImplementedException(); + } + + public Task> GetGroupsForUserAsync(String name) + { + throw new NotImplementedException(); + } + public Task UpdateUserAsync(String name, Boolean value, Boolean force = false) { throw new NotImplementedException(); } + + public Task UpdateUserAsync(String username, String groupName, Boolean value, Boolean force = false) + { + throw new NotImplementedException(); + } } } diff --git a/src/Data/WikiBotDatabase.cs b/src/Data/WikiBotDatabase.cs index af929e0..46e778b 100644 --- a/src/Data/WikiBotDatabase.cs +++ b/src/Data/WikiBotDatabase.cs @@ -1,8 +1,10 @@ using BotTerminator.Configuration; +using BotTerminator.Models; using Newtonsoft.Json; using RedditSharp; using RedditSharp.Things; using System; +using System.Collections.Generic; using System.Threading.Tasks; namespace BotTerminator.Data @@ -18,40 +20,88 @@ public class WikiBotDatabase : IBotDatabase private DateTimeOffset LastUpdatedAtUtc { get; set; } = DateTimeOffset.MinValue; private static readonly TimeSpan staleTimeSpan = new TimeSpan(0, 10, 0); - public Boolean IsStale => Cache.Items.Count == 0 || DateTimeOffset.UtcNow - LastUpdatedAtUtc > staleTimeSpan; + public Boolean IsStale => Cache.Count == 0 || DateTimeOffset.UtcNow - LastUpdatedAtUtc > staleTimeSpan; public WikiBotDatabase(Subreddit sr) { this.SrWiki = sr.GetWiki; } - public async Task CheckUserAsync(String name) + public async Task GetConfigAsync() { - if (IsStale) - { - await GetUpdatedListFromWikiAsync(); - LastUpdatedAtUtc = DateTimeOffset.UtcNow; - } - return Cache.Items.Contains(name); + await UpdateIfStaleAsync(); + return Cache; + } + + public async Task> GetAllGroupsAsync() + { + await UpdateIfStaleAsync(); + return Cache.GroupLookup; + } + + public async Task> GetGroupsForUserAsync(String username) + { + await UpdateIfStaleAsync(); + return Cache.GetGroupsByUser(username); + } + + public async Task CheckUserAsync(String name, String group) + { + await UpdateIfStaleAsync(); + return Cache.IsInGroup(group, name); } - public async Task UpdateUserAsync(String name, Boolean value, Boolean force) + public async Task UpdateUserAsync(String name, String group, Boolean value, Boolean force) { - if (value) + if (Cache.GroupLookup.ContainsKey(group)) { - Cache.Items.Add(name); + if (value) + { + Cache.GroupLookup[group].Members.Add(name); + } + else + { + Cache.GroupLookup[group].Members.Remove(name); + } } if (force || IsStale) { - await SrWiki.EditPageAsync(pageName, JsonConvert.SerializeObject(Cache)); + await SrWiki.EditPageAsync(pageName, JsonConvert.SerializeObject(Cache, Formatting.Indented)); LastUpdatedAtUtc = DateTimeOffset.UtcNow; } } + private async Task UpdateIfStaleAsync() + { + if (IsStale) + { + try + { + await GetUpdatedListFromWikiAsync(); + LastUpdatedAtUtc = DateTimeOffset.UtcNow; + } + catch (RedditHttpException ex) + { + Console.WriteLine("Failed to update cache: {0}", ex.Message); + } + catch (OperationCanceledException) + { + Console.WriteLine("Failed to update cache: timed out"); + } + } + } + private async Task GetUpdatedListFromWikiAsync() { String mdData = (await SrWiki.GetPageAsync(pageName)).MarkdownContent; Cache = JsonConvert.DeserializeObject(mdData); + Cache.ValidateSupportedVersion(2, 2); + } + + public async Task> GetDefaultBannedGroupsAsync() + { + await UpdateIfStaleAsync(); + return Cache.GetDefaultActionedOnGroups(); } } } diff --git a/src/Models/CachedSubreddit.cs b/src/Models/CachedSubreddit.cs index 42130e0..d3e7ab1 100644 --- a/src/Models/CachedSubreddit.cs +++ b/src/Models/CachedSubreddit.cs @@ -34,11 +34,12 @@ public CachedSubreddit(Subreddit subreddit, IConfigurationLoader ReadConfigFromWikiAsync() { - SubredditConfig config = await ConfigurationLoader.LoadConfigAsync((await RedditSubreddit.GetWiki.GetPageAsync(pageName)).MarkdownContent); + SubredditConfig config = await ConfigurationLoader.LoadConfigAsync((await RedditSubreddit.GetWiki.GetPageAsync(pageName.ToLowerInvariant())).MarkdownContent); config.ValidateSupportedVersion(minSupportedVersion, maxSupportedVersion); return config; } diff --git a/src/Models/Group.cs b/src/Models/Group.cs new file mode 100644 index 0000000..5f4d9ec --- /dev/null +++ b/src/Models/Group.cs @@ -0,0 +1,26 @@ +using Newtonsoft.Json; +using System; +using System.Collections.Generic; +using System.Text; + +namespace BotTerminator.Models +{ + [JsonObject] + public class Group + { + [JsonProperty("actionByDefault")] + public bool ActionByDefault { get; private set; } + + [JsonProperty("id", Required = Required.Always)] + public Guid Id { get; private set; } + + [JsonProperty("name", Required = Required.Always)] + public String Name { get; private set; } + + [JsonProperty("postFlairCssClass", Required = Required.Always)] + public String PostFlairCssClass { get; private set; } + + [JsonProperty("members")] + public ISet Members { get; private set; } = new HashSet(); + } +} diff --git a/src/Modules/CacheFreshenerModule.cs b/src/Modules/CacheFreshenerModule.cs index 84239e1..f66735e 100644 --- a/src/Modules/CacheFreshenerModule.cs +++ b/src/Modules/CacheFreshenerModule.cs @@ -15,7 +15,7 @@ public CacheFreshenerModule(BotTerminator bot) : base(bot) public override async Task RunOnceAsync() { - await bot.UserLookup.UpdateUserAsync(BotTerminator.CacheFreshenerUserName, false); + await bot.UserLookup.UpdateUserAsync(BotTerminator.CacheFreshenerUserName, String.Empty, false); await bot.UpdateSubredditCacheAsync(); } diff --git a/src/Modules/ScannerModule.cs b/src/Modules/ScannerModule.cs index 79ed5dc..f5b17dd 100644 --- a/src/Modules/ScannerModule.cs +++ b/src/Modules/ScannerModule.cs @@ -5,6 +5,7 @@ using RedditSharp.Things; using System; using System.Collections.Generic; +using System.Linq; using System.Text; using System.Threading.Tasks; @@ -34,20 +35,23 @@ protected override sealed Boolean PreRunItem(T thing) protected override sealed async Task RunItemAsync(T thing) { - if (await bot.CheckShouldBanAsync(thing)) + String subredditName = thing["subreddit"].Value(); + if (!bot.SubredditLookup.ContainsKey(subredditName)) + { + await bot.CacheSubredditAsync(subredditName); + } + CachedSubreddit subreddit = bot.SubredditLookup[subredditName]; + AbstractSubredditOptionSet options = new ShadedOptionSet(new[] { subreddit?.Options, GlobalConfig.GlobalOptions }, true); + if (!options.Enabled) return; + if (!options.ScanPosts && thing is Post) return; + if (!options.ScanComments && thing is Comment) return; + + IReadOnlyCollection bannedGroups = await bot.GetBannedGroupsAsync(options); + + if (await bot.CheckShouldBanAsync(thing, bannedGroups.Select(group => group.Name))) { - String subredditName = thing["subreddit"].Value(); - if (!bot.SubredditLookup.ContainsKey(subredditName)) - { - await bot.CacheSubredditAsync(subredditName); - } - CachedSubreddit subreddit = bot.SubredditLookup[subredditName]; - AbstractSubredditOptionSet options = new ShadedOptionSet(new[] { subreddit?.Options, GlobalConfig.GlobalOptions }, true); - if (!options.Enabled) return; try { - if (!options.ScanPosts && thing is Post) return; - if (!options.ScanComments && thing is Comment) return; if (options.RemovalType == RemovalType.Spam) { await thing.RemoveSpamAsync(); diff --git a/src/Modules/UpdateBanListModule.cs b/src/Modules/UpdateBanListModule.cs index 10b6015..22ac6b8 100644 --- a/src/Modules/UpdateBanListModule.cs +++ b/src/Modules/UpdateBanListModule.cs @@ -1,4 +1,5 @@ -using Newtonsoft.Json.Linq; +using BotTerminator.Configuration; +using Newtonsoft.Json.Linq; using RedditSharp.Things; using System; using System.Collections.Generic; @@ -28,33 +29,45 @@ public override Task SetupAsync() public override async Task TeardownAsync() { - await bot.UserLookup.UpdateUserAsync(BotTerminator.CacheFreshenerUserName, false, true); + await bot.UserLookup.UpdateUserAsync(BotTerminator.CacheFreshenerUserName, String.Empty, false, true); } protected override async Task PostRunItemsAsync(ICollection subredditPosts) { // hide all of them at once + BanListConfig config = await bot.UserLookup.GetConfigAsync(); + ICollection hideable = subredditPosts.Where(post => config.ShouldHide(post)).ToList(); if (subredditPosts.Count > 0) { const String requestVerb = "POST"; - for (int i = 0; i < subredditPosts.Count; i += 25) + IList tasks = new List(); + for (int i = 0; i < hideable.Count; i += 25) { - String formattedUrl = String.Format("{0}?id={1}", BotTerminator.HideUrl, String.Join(",", subredditPosts.Select(s => s.FullName).Skip(i).Take(25))); - await bot.WebAgent.ExecuteRequestAsync(() => bot.WebAgent.CreateRequest(formattedUrl, requestVerb)); + String formattedUrl = String.Format("{0}?id={1}", BotTerminator.HideUrl, String.Join(",", subredditPosts.Select(post => post.FullName).Skip(i).Take(25))); + tasks.Add(bot.WebAgent.ExecuteRequestAsync(() => bot.WebAgent.CreateRequest(formattedUrl, requestVerb))); } + await Task.WhenAll(tasks); } } protected override Boolean PreRunItem(Post subredditPost) { - return !subredditPost.IsHidden && subredditPost.LinkFlairText != null && (subredditPost.LinkFlairText == "Banned" || subredditPost.LinkFlairText == "Meta"); + String[] groups = GetGroupNames(subredditPost.LinkFlairCssClass); + // TODO: maybe add async override or something? + if (groups.Length == 0 || groups.Contains("await")) return false; + return !subredditPost.IsHidden && subredditPost.LinkFlairCssClass != null; + } + + private String[] GetGroupNames(String linkFlairCssClass) + { + return linkFlairCssClass.Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); } protected override async Task RunItemAsync(Post subredditPost) { // We don't need to even look at meta posts - if (subredditPost.LinkFlairText == "Meta") return; - + BanListConfig config = await bot.UserLookup.GetConfigAsync(); + IEnumerable groups = GetGroupNames(subredditPost.LinkFlairCssClass).Where(group => !config.NonGroupFlairCssClasses.Contains(group)); /* * We don't use the post.Url property here because if the Url is not a * well formed URI, RedditSharp throws an UriFormatException. The cases @@ -64,7 +77,10 @@ protected override async Task RunItemAsync(Post subredditPost) if (match == null || match.Groups.Count != 2) return; Console.WriteLine("Found new bot to ban " + match.Groups[1].Value); String targetUserName = match.Groups[1].Value; - await bot.UserLookup.UpdateUserAsync(targetUserName, true, first); + foreach (String group in groups) + { + await bot.UserLookup.UpdateUserAsync(targetUserName, group, true, first); + } first = false; } }