diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..5c8161f0 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,5 @@ +* +!requirements.txt +!GearBot/* +!lang/* +!template.json \ No newline at end of file diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..b4a8b3f3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,6 @@ +FROM python:3.9 +WORKDIR /GearBot +COPY requirements.txt ./ +RUN pip3 install --no-cache-dir -r requirements.txt +COPY . . +CMD ["python", "./GearBot/GearBot.py"] \ No newline at end of file diff --git a/GearBot/Cogs/AntiSpam.py b/GearBot/Cogs/AntiSpam.py index d77c84ca..f13b1de2 100644 --- a/GearBot/Cogs/AntiSpam.py +++ b/GearBot/Cogs/AntiSpam.py @@ -304,7 +304,7 @@ async def ban_punishment(self, v: Violation): async def censor_detector(self): - # reciever taks for someone gets censored + # reciever task for someone gets censored while self.running: try: message = None diff --git a/GearBot/Cogs/PromMonitoring.py b/GearBot/Cogs/PromMonitoring.py index 2a9546cc..3a9f2d8b 100644 --- a/GearBot/Cogs/PromMonitoring.py +++ b/GearBot/Cogs/PromMonitoring.py @@ -45,7 +45,8 @@ async def create_site(self): runner = web.AppRunner(metrics_app) await self.bot.loop.create_task(runner.setup()) - site = web.TCPSite(runner, 'localhost', 8090 + self.bot.cluster) + site = web.TCPSite(runner, host='0.0.0.0', port=8090) + await site.start() self.metric_server = site diff --git a/GearBot/GearBot.py b/GearBot/GearBot.py index 2b43d40d..00b36df5 100644 --- a/GearBot/GearBot.py +++ b/GearBot/GearBot.py @@ -1,6 +1,9 @@ # force it to use v6 instead of v7 +import asyncio + import discord.http -discord.http.Route.BASE = 'https://discordapp.com/api/v6' + +discord.http.Route.BASE = 'http://http-proxy/api/v6' import os from argparse import ArgumentParser @@ -9,10 +12,30 @@ from Bot.GearBot import GearBot from Util import Configuration, GearbotLogging from discord import Intents, MemberCacheFlags +from kubernetes import client, config def prefix_callable(bot, message): return TheRealGearBot.prefix_callable(bot, message) +async def node_init(generation, resource_version): + from database import DatabaseConnector + from database.DatabaseConnector import Node + await DatabaseConnector.init() + hostname = os.uname()[1] + GearbotLogging.info(f"GearBot clusternode {hostname} (generation {generation}). Trying to figure out where i fit in") + existing = await Node.filter(hostname=hostname, generation=generation).get_or_none() + if existing is None: + count = 0 + while count < 100: + + try: + await Node.create(hostname=hostname, generation=generation, resource_version=resource_version, shard=count) + return count + except Exception as ex: + GearbotLogging.exception("did something go wrong?", ex) + count += 1 + else: + return existing.shard if __name__ == '__main__': parser = ArgumentParser() @@ -34,6 +57,9 @@ def prefix_callable(bot, message): else: token = input("Please enter your Discord token: ") + loop = asyncio.new_event_loop() + asyncio.set_event_loop(loop) + args = { "command_prefix": prefix_callable, "case_insensitive": True, @@ -69,12 +95,30 @@ def prefix_callable(bot, message): "cluster": offset, "shard_ids": [*range(offset * num_shards, (offset * num_shards) + num_shards)] }) + elif os.environ['namespace']: + GearbotLogging.info("Determining scaling information from kubernetes ...") + namespace = os.environ['namespace'] + config.load_incluster_config() + kubeclient = client.AppsV1Api() + deployment = kubeclient.read_namespaced_deployment("gearbot", "gearbot") + print(deployment) + cluster = loop.run_until_complete(node_init(deployment.status.observed_generation, deployment.metadata.annotations["deployment.kubernetes.io/revision"])) + num_clusters = deployment.spec.replicas + args.update({ + "shard_count": num_clusters, + "cluster": cluster, + "shard_ids": [cluster] + }) + + + gearbot = GearBot(**args) gearbot.remove_command("help") - GearbotLogging.info("Ready to go, spinning up the gears") + GearbotLogging.info(f"Ready to go, spinning up as instance {args['cluster'] + 1}/{args['shard_count']}") gearbot.run(token) GearbotLogging.info("GearBot shutting down, cleaning up") gearbot.database_connection.close() GearbotLogging.info("Cleanup complete") + diff --git a/GearBot/Util/Configuration.py b/GearBot/Util/Configuration.py index 8d00b9a0..9148d689 100644 --- a/GearBot/Util/Configuration.py +++ b/GearBot/Util/Configuration.py @@ -393,7 +393,7 @@ def move_keys(config, section, *keys): async def initialize(bot: commands.Bot): global CONFIG_VERSION, BOT, TEMPLATE BOT = bot - TEMPLATE = Utils.fetch_from_disk("config/template") + TEMPLATE = Utils.fetch_from_disk("template") CONFIG_VERSION = TEMPLATE["VERSION"] GearbotLogging.info(f"Current template config version: {CONFIG_VERSION}") # GearbotLogging.info(f"Loading configurations for {len(bot.guilds)} guilds.") @@ -415,7 +415,7 @@ def load_config(guild): SERVER_CONFIGS[guild] = update_config(guild, config) if len(config.keys()) is 0: GearbotLogging.info(f"No config available for {guild}, creating a blank one.") - SERVER_CONFIGS[guild] = Utils.fetch_from_disk("config/template") + SERVER_CONFIGS[guild] = Utils.fetch_from_disk("template") save(guild) validate_config(guild) Features.check_server(guild) diff --git a/GearBot/database/DatabaseConnector.py b/GearBot/database/DatabaseConnector.py index 29cc8e6b..fa44f9c8 100644 --- a/GearBot/database/DatabaseConnector.py +++ b/GearBot/database/DatabaseConnector.py @@ -1,7 +1,7 @@ from tortoise.models import Model from tortoise import fields, Tortoise -from Util import Configuration +from Util import Configuration, GearbotLogging class LoggedMessage(Model): @@ -76,10 +76,17 @@ class RaidAction(Model): action = fields.CharField(max_length=20) infraction = fields.ForeignKeyField("models.Infraction", related_name="RaiderAction", source_field="infraction_id", null=True) +class Node(Model): + hostname = fields.CharField(max_length=50, pk=True) + generation = fields.IntField() + shard = fields.IntField() + resource_version = fields.CharField(max_length=50) + async def init(): + GearbotLogging.info("Connecting to the database...") await Tortoise.init( - db_url=f"mysql://{Configuration.get_master_var('DATABASE_USER')}:{Configuration.get_master_var('DATABASE_PASS')}@{Configuration.get_master_var('DATABASE_HOST')}:{Configuration.get_master_var('DATABASE_PORT')}/{Configuration.get_master_var('DATABASE_NAME')}", + db_url=Configuration.get_master_var('DATABASE'), modules={"models": ["database.DatabaseConnector"]} ) await Tortoise.generate_schemas() diff --git a/config/.gitignore b/config/.gitignore index 872b12de..b1114535 100644 --- a/config/.gitignore +++ b/config/.gitignore @@ -1,3 +1,3 @@ *.json -!template.json +!../template.json !*.example \ No newline at end of file diff --git a/config/template.json b/config/template.json deleted file mode 100644 index 1cbe02a2..00000000 --- a/config/template.json +++ /dev/null @@ -1,91 +0,0 @@ -{ - "VERSION": 32, - "GENERAL": { - "LANG": "en_US", - "PERM_DENIED_MESSAGE": true, - "PREFIX": "!", - "TIMESTAMPS": true, - "NEW_USER_THRESHOLD": 86400, - "TIMEZONE": "Europe/Brussels" - }, - "ROLES": { - "SELF_ROLES": [], - "ROLE_LIST": [], - "ROLE_LIST_MODE": true, - "MUTE_ROLE": 0 - }, - "LOG_CHANNELS": {}, - "MESSAGE_LOGS": { - "ENABLED": false, - "IGNORED_CHANNELS_CHANGES": [], - "IGNORED_CHANNELS_OTHER": [], - "IGNORED_USERS": [], - "EMBED": false - }, - "CENSORING": { - "ENABLED": false, - "ALLOW_TRUSTED_BYPASS": false, - "WORD_CENSORLIST": [], - "TOKEN_CENSORLIST": [], - "ALLOWED_INVITE_LIST": [], - "DOMAIN_LIST_ALLOWED": false, - "DOMAIN_LIST": [], - "FULL_MESSAGE_LIST": [], - "CENSOR_EMOJI_ONLY_MESSAGES": false - }, - "FLAGGING": { - "WORD_LIST": [], - "TOKEN_LIST": [] - }, - "INFRACTIONS": { - "DM_ON_WARN": false, - "DM_ON_UNMUTE": false, - "DM_ON_MUTE": false, - "DM_ON_KICK": false, - "DM_ON_BAN": false, - "DM_ON_TEMPBAN": false, - "WARNING": null, - "UNMUTE": null, - "MUTE": null, - "KICK": null, - "BAN": null, - "TEMPBAN": null - }, - "PERM_OVERRIDES": {}, - "RAID_HANDLING": { - "ENABLED": false, - "HANDLERS": [], - "INVITE": "" - }, - "ANTI_SPAM": { - "CLEAN": false, - "ENABLED": false, - "EXEMPT_ROLES": [], - "EXEMPT_USERS": [], - "BUCKETS": [] - }, - "DASH_SECURITY": { - "ACCESS": 2, - "INFRACTION": 2, - "VIEW_CONFIG": 2, - "ALTER_CONFIG": 3 - }, - "PERMISSIONS": { - "LVL4_ROLES": [], - "LVL4_USERS": [], - "ADMIN_ROLES": [], - "ADMIN_USERS": [], - "MOD_ROLES": [], - "MOD_USERS": [], - "TRUSTED_ROLES": [], - "TRUSTED_USERS": [] - }, - "SERVER_LINKS": [], - "CUSTOM_COMMANDS": { - "ROLES": [], - "ROLE_REQUIRED": false, - "CHANNELS": [], - "CHANNELS_IGNORED": true, - "MOD_BYPASS": true - } -} diff --git a/datamove.py b/datamove.py new file mode 100644 index 00000000..d5a94fb9 --- /dev/null +++ b/datamove.py @@ -0,0 +1,229 @@ +import logging +import sys + +from tortoise import Model, fields, Tortoise, run_async + + +class LoggedMessage(Model): + messageid = fields.BigIntField(pk=True, generated=False) + content = fields.CharField(max_length=2000, collation="utf8mb4_general_ci", null=True) + author = fields.BigIntField() + channel = fields.BigIntField() + server = fields.BigIntField() + type = fields.IntField(null=True) + pinned = fields.BooleanField(default=False) + + class Meta: + app = "models" + +class NewLoggedMessage(Model): + messageid = fields.BigIntField(pk=True, generated=False) + content = fields.CharField(max_length=2000, collation="utf8mb4_general_ci", null=True) + author = fields.BigIntField() + channel = fields.BigIntField() + server = fields.BigIntField() + type = fields.IntField(null=True) + pinned = fields.BooleanField(default=False) + + class Meta: + app = "new" + + +class LoggedAttachment(Model): + id = fields.BigIntField(pk=True, generated=False) + name = fields.CharField(max_length=100) + isImage = fields.BooleanField() + message = fields.ForeignKeyField("models.LoggedMessage", related_name='attachments', source_field='message_id') + + class Meta: + app = "models" + + +class NewLoggedAttachment(Model): + id = fields.BigIntField(pk=True, generated=False) + name = fields.CharField(max_length=100) + isImage = fields.BooleanField() + message = fields.ForeignKeyField("models.LoggedMessage", related_name='attachments', source_field='message_id') + + class Meta: + app = "new" + + +class CustomCommand(Model): + id = fields.IntField(pk=True, generated=True) + serverid = fields.BigIntField() + trigger = fields.CharField(max_length=20, collation="utf8mb4_general_ci") + response = fields.CharField(max_length=2000, collation="utf8mb4_general_ci") + created_by = fields.BigIntField(null=True) + + class Meta: + app = "models" + + +class NewCustomCommand(Model): + id = fields.IntField(pk=True, generated=True) + serverid = fields.BigIntField() + trigger = fields.CharField(max_length=20, collation="utf8mb4_general_ci") + response = fields.CharField(max_length=2000, collation="utf8mb4_general_ci") + created_by = fields.BigIntField(null=True) + + class Meta: + app = "new" + + +class Infraction(Model): + id = fields.IntField(pk=True, generated=True) + guild_id = fields.BigIntField() + user_id = fields.BigIntField() + mod_id = fields.BigIntField() + type = fields.CharField(max_length=10, collation="utf8mb4_general_ci") + reason = fields.CharField(max_length=2000, collation="utf8mb4_general_ci") + start = fields.BigIntField() + end = fields.BigIntField(null=True) + active = fields.BooleanField(default=True) + + class Meta: + app = "models" + +class NewInfraction(Model): + id = fields.IntField(pk=True, generated=True) + guild_id = fields.BigIntField() + user_id = fields.BigIntField() + mod_id = fields.BigIntField() + type = fields.CharField(max_length=10, collation="utf8mb4_general_ci") + reason = fields.CharField(max_length=2000, collation="utf8mb4_general_ci") + start = fields.BigIntField() + end = fields.BigIntField(null=True) + active = fields.BooleanField(default=True) + + class Meta: + app = "new" + + +class Reminder(Model): + id = fields.IntField(pk=True, generated=True) + user_id = fields.BigIntField() + channel_id = fields.BigIntField() + guild_id = fields.CharField(max_length=20, null=True) + message_id = fields.BigIntField() + dm = fields.BooleanField() + to_remind = fields.CharField(max_length=1800, collation="utf8mb4_general_ci") + send = fields.BigIntField(null=True) + time = fields.BigIntField() + status = fields.IntField() + + class Meta: + app = "models" + + +class NewReminder(Model): + id = fields.IntField(pk=True, generated=True) + user_id = fields.BigIntField() + channel_id = fields.BigIntField() + guild_id = fields.CharField(max_length=20, null=True) + message_id = fields.BigIntField(null=True) + dm = fields.BooleanField() + to_remind = fields.CharField(max_length=1800, collation="utf8mb4_general_ci") + send = fields.BigIntField(null=True) + time = fields.BigIntField() + status = fields.IntField() + + class Meta: + app = "new" + + +LOGGER = logging.getLogger('datamover') +LOGGER.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s:%(levelname)s:%(name)s: %(message)s') +handler = logging.StreamHandler(stream=sys.stdout) +handler.setLevel(logging.INFO) +handler.setFormatter(formatter) +LOGGER.addHandler(handler) + +async def run(): + old_url = input("Source database url: ") + new_url = input("Destination database url: ") + await Tortoise.init( + config={ + 'connections': { + 'old': old_url, + 'new': new_url + }, + 'apps': { + 'my_app': { + 'models': ['__main__'], + # If no default_connection specified, defaults to 'default' + 'default_connection': 'old', + }, + 'models': {'models': ['__main__'], 'default_connection': 'old'} + } + } + ) + new = Tortoise.get_connection('new') + LOGGER.info("Database connections established") + + + LOGGER.info("Staring custom commands migration") + ccs = await CustomCommand.filter() + LOGGER.info(f"Found {len(ccs)} custom commands, migrating...") + await CustomCommand.bulk_create([NewCustomCommand(id=cc.id, serverid=cc.serverid, trigger=cc.trigger, response=cc.response,created_by=cc.created_by) for cc in ccs], new) + await new.execute_script("select setval('customcommand_id_seq', (select max(id) from customcommand))") + LOGGER.info("Custom commands migrated") + + + LOGGER.info("Starting reminders migration") + reminders = await Reminder.filter() + LOGGER.info(f"Found {len(reminders)} reminders, migrating...") + await Reminder.bulk_create([NewReminder(id=reminder.id, user_id=reminder.user_id, channel_id=reminder.channel_id, guild_id=reminder.guild_id, message_id=reminder.message_id, dm=reminder.dm, to_remind=reminder.to_remind, send=reminder.send, time=reminder.time, status=reminder.status) for reminder in reminders], new) + await new.execute_script("select setval('reminder_id_seq', (select max(id) from reminder))") + LOGGER.info("Reminders migrated") + + LOGGER.info("Starting infractions migration") + todo = await Infraction.filter().count() + LOGGER.info(f"Found {todo} infractions to migrate") + done = 0 + while done < todo: + chunk = await Infraction.filter().offset(done).limit(10000) + new_infractions=[NewInfraction(id=infraction.id, guild_id=infraction.guild_id, user_id=infraction.user_id, mod_id=infraction.mod_id, type=infraction.type, reason=infraction.reason, start=infraction.start, end=infraction.end, active=infraction.active) for infraction in chunk] + await Infraction.bulk_create(new_infractions, new) + done += len(new_infractions) + LOGGER.info(f"{done}/{todo} infractions migrated...") + await new.execute_script("select setval('infraction_id_seq', (select max(id) from infraction))") + LOGGER.info("Infractions migrated") + + LOGGER.info("Starting logged messages migration") + todo = await LoggedMessage.filter().count() + LOGGER.info(f"Found {todo} logged messages to migrate") + done = 0 + while done < todo: + chunk = await LoggedMessage.filter().order_by("-messageid").offset(done).limit(10000) + new_messages=[NewLoggedMessage(messageid=message.messageid, content=message.content, author=message.author, channel=message.channel, server=message.server, type=message.type, pinned=message.pinned) for message in chunk] + try: + await LoggedMessage.bulk_create(new_messages, new) + except: + LOGGER.error("Logged message chunk failed!") + done += len(new_messages) + LOGGER.info(f"{done}/{todo} logged messages migrated...") + LOGGER.info("Logged messages migrated") + + + LOGGER.info("Starting logged attachments migration") + todo = await LoggedAttachment.filter().count() + LOGGER.info(f"Found {todo} logged attachments to migrate") + done = 0 + while done < todo: + chunk = await LoggedAttachment.filter().order_by("-id").offset(done).limit(10000) + new_attachments=[NewLoggedAttachment(id=attachment.id, name=attachment.name, message=attachment.message, isImage=attachment.isImage) for attachment in chunk] + await LoggedAttachment.bulk_create(new_attachments, new) + done += len(new_attachments) + LOGGER.info(f"{done}/{todo} logged attachments migrated...") + LOGGER.info("Logged attachments migrated") + + + + + + + + +run_async(run()) diff --git a/requirements.txt b/requirements.txt index e026ac93..94794bf1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,5 +7,6 @@ sentry-sdk==0.19.2 pytz==2020.4 pyseeyou==1.0.1 prometheus_client==0.8.0 -aiomysql==0.0.20 -emoji==0.6.0 \ No newline at end of file +asyncpg==0.22.0 +emoji==0.6.0 +kubernetes==12.0.1 \ No newline at end of file