Merge pull request #359 from barrycarey/feature/onlyfans-messages

Feature/onlyfans messages

docker-compose.yml

@@ -134,9 +134,9 @@ services:
         environment:
             - RUN_ENV=production
             - db_user=ingest
-            - LOG_LEVEL=ERROR
+            - LOG_LEVEL=INFO
             - CELERY_IMPORTS=redditrepostsleuth.core.celery.tasks.ingest_tasks
-        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_ingest -n ingest_worker --autoscale=3,16
+        entrypoint: celery -A redditrepostsleuth.core.celery worker -Q post_ingest -n ingest_worker --autoscale=16,1
 
     link_repost_worker:
         container_name: link-repost-worker

docs/dev_docs/modifying_monitored_sub.md

@@ -0,0 +1,5 @@
+
+### Adding or Removing Config Values
+* Add / Remove config values in core/db/databasemodels.py
+* Add/Remove in core/util/default_bot_config.py
+* Update sub_monitor_exposed_config_options in the config json

redditrepostsleuth/core/celery/task_logic/scheduled_task_logic.py

@@ -57,35 +57,42 @@ def update_proxies(uowm: UnitOfWorkManager) -> None:
             )
         uow.commit()
 
-def update_top_reposts(uowm: UnitOfWorkManager):
+def update_top_reposts(uow: UnitOfWork, post_type_id: int, day_range: int = None):
     # reddit.info(reddit_ids_to_lookup):
-    post_types = [2, 3]
-    day_ranges = [1, 7, 14, 30, 365, None]
+    log.info('Getting top repostors for post type %s with range %s', post_type_id, day_range)
     range_query = "SELECT repost_of_id, COUNT(*) c FROM repost WHERE detected_at > NOW() - INTERVAL :days DAY AND post_type_id=:posttype GROUP BY repost_of_id HAVING c > 5 ORDER BY c DESC"
     all_time_query = "SELECT repost_of_id, COUNT(*) c FROM repost WHERE post_type_id=:posttype GROUP BY repost_of_id HAVING c > 5 ORDER BY c DESC"
-    with uowm.start() as uow:
-        for post_type in post_types:
-            for days in day_ranges:
-                log.info('Getting top reposts for post type %s with range %s', post_type, days)
-                if days:
-                    query = range_query
-                else:
-                    query = all_time_query
-                uow.session.execute(
-                    text('DELETE FROM stat_top_repost WHERE post_type_id=:posttype AND day_range=:days'),
-                    {'posttype': post_type, 'days': days})
-                uow.commit()
-                result = uow.session.execute(text(query), {'posttype': post_type, 'days': days})
-                for row in result:
-                    stat = StatsTopRepost()
-                    stat.post_id = row[0]
-                    stat.post_type_id = post_type
-                    stat.day_range = days
-                    stat.repost_count = row[1]
-                    stat.updated_at = func.utc_timestamp()
-                    stat.nsfw = False
-                    uow.stat_top_repost.add(stat)
-                    uow.commit()
+    if day_range:
+        query = range_query
+        uow.session.execute(text('DELETE FROM stat_top_repost WHERE post_type_id=:posttype AND day_range=:days'),
+                            {'posttype': post_type_id, 'days': day_range})
+    else:
+        query = all_time_query
+        uow.session.execute(text('DELETE FROM stat_top_repost WHERE post_type_id=:posttype AND day_range IS NULL'),
+                            {'posttype': post_type_id})
+
+    uow.commit()
+
+
+
+    result = uow.session.execute(text(query), {'posttype': post_type_id, 'days': day_range})
+    for row in result:
+        stat = StatsTopRepost()
+        stat.post_id = row[0]
+        stat.post_type_id = post_type_id
+        stat.day_range = day_range
+        stat.repost_count = row[1]
+        stat.updated_at = func.utc_timestamp()
+        stat.nsfw = False
+        uow.stat_top_repost.add(stat)
+        uow.commit()
+
+def run_update_top_reposts(uow: UnitOfWork) -> None:
+    post_types = [1, 2, 3]
+    day_ranges = [1, 7, 14, 30, None]
+    for post_type_id in post_types:
+        for days in day_ranges:
+            update_top_reposts(uow, post_type_id, days)
 
 def update_top_reposters(uow: UnitOfWork, post_type_id: int, day_range: int = None) -> None:
     log.info('Getting top repostors for post type %s with range %s', post_type_id, day_range)

redditrepostsleuth/core/celery/tasks/ingest_tasks.py

@@ -12,7 +12,7 @@
 
 
 @celery.task(bind=True, base=SqlAlchemyTask, ignore_reseults=True, serializer='pickle', autoretry_for=(ConnectionError,ImageConversionException,GalleryNotProcessed), retry_kwargs={'max_retries': 10, 'countdown': 300})
-def save_new_post(self, submission: dict):
+def save_new_post(self, submission: dict, repost_check: bool = True):
 
     # TODO: temp fix until I can fix imgur gifs
     if 'imgur' in submission['url'] and 'gifv' in submission['url']:
@@ -48,21 +48,22 @@ def save_new_post(self, submission: dict):
             log.exception('Database save failed: %s', str(e), exc_info=False)
             return
 
-    if post.post_type_id == 1:
-        celery.send_task('redditrepostsleuth.core.celery.tasks.repost_tasks.check_for_text_repost_task', args=[post])
-    elif post.post_type_id == 2:
-        celery.send_task('redditrepostsleuth.core.celery.tasks.repost_tasks.check_image_repost_save', args=[post])
-    elif post.post_type_id == 3:
-        celery.send_task('redditrepostsleuth.core.celery.tasks.repost_tasks.link_repost_check', args=[post])
+    if repost_check:
+        if post.post_type_id == 1:
+            celery.send_task('redditrepostsleuth.core.celery.tasks.repost_tasks.check_for_text_repost_task', args=[post])
+        elif post.post_type_id == 2:
+            celery.send_task('redditrepostsleuth.core.celery.tasks.repost_tasks.check_image_repost_save', args=[post])
+        elif post.post_type_id == 3:
+            celery.send_task('redditrepostsleuth.core.celery.tasks.repost_tasks.link_repost_check', args=[post])
 
     celery.send_task('redditrepostsleuth.core.celery.admin_tasks.check_user_for_only_fans', args=[post.author])
 
 
 
 @celery.task
-def save_new_posts(posts: list[dict]) -> None:
+def save_new_posts(posts: list[dict], repost_check: bool = True) -> None:
     for post in posts:
-        save_new_post.apply_async((post,))
+        save_new_post.apply_async((post, repost_check))
 
 @celery.task(bind=True, base=SqlAlchemyTask, ignore_results=True)
 def save_pushshift_results(self, data):

redditrepostsleuth/core/celery/tasks/scheduled_tasks.py

@@ -9,7 +9,7 @@
 from redditrepostsleuth.core.celery import celery
 from redditrepostsleuth.core.celery.basetasks import RedditTask, SqlAlchemyTask, AdminTask
 from redditrepostsleuth.core.celery.task_logic.scheduled_task_logic import update_proxies, update_top_reposts, \
-    token_checker, run_update_top_reposters, update_top_reposters, update_monitored_sub_data
+    token_checker, run_update_top_reposters, update_top_reposters, update_monitored_sub_data, run_update_top_reposts
 from redditrepostsleuth.core.db.databasemodels import MonitoredSub, StatsDailyCount
 from redditrepostsleuth.core.logging import configure_logger
 from redditrepostsleuth.core.util.reddithelpers import is_sub_mod_praw, get_bot_permissions
@@ -178,6 +178,13 @@ def update_daily_stats(self):
         log.exception('Problem updating stats')
 
 
+@celery.task(bind=True, base=SqlAlchemyTask)
+def update_all_top_reposts_task(self):
+    try:
+        with self.uowm.start() as uow:
+            run_update_top_reposts(uow)
+    except Exception as e:
+        log.exception('Unknown task error')
 
 @celery.task(bind=True, base=SqlAlchemyTask)
 def update_all_top_reposters_task(self):

redditrepostsleuth/core/config.py

@@ -214,7 +214,7 @@ def _initialize_attributes(self):
             'default_text_crosspost_filter',
             'default_text_max_days_old_filter',
             'default_text_target_distance',
-            'discord_logging_hook'
+            'discord_logging_hook',
 
         ]
 

redditrepostsleuth/core/db/databasemodels.py

@@ -47,6 +47,7 @@ def __repr__(self) -> str:
     reports = relationship('UserReport', back_populates='post')
     hashes = relationship('PostHash', back_populates='post')
     post_type = relationship('PostType') # lazy has to be set to JSON encoders don't fail for unbound session
+    #post_type = relationship('PostType', lazy='joined')
 
     def to_dict(self):
         return {
@@ -198,7 +199,7 @@ class RepostSearch(Base):
         Index('idx_post_type_searched_at', 'post_type_id', 'searched_at'),
         Index('idx_by_subreddit_and_type', 'subreddit', 'source', 'post_type_id', 'matches_found'),
         Index('idx_source', 'source'),
-        Index('idx_matches_found', 'matches_found')
+        Index('idx_matches_found', 'searched_at', 'source', 'matches_found')
     )
     id = Column(Integer, primary_key=True)
     post_id = Column(Integer, ForeignKey('post.id'))
@@ -351,10 +352,14 @@ class MonitoredSub(Base):
     adult_promoter_remove_post = Column(Boolean, default=False)
     adult_promoter_ban_user = Column(Boolean, default=False)
     adult_promoter_notify_mod_mail = Column(Boolean, default=False)
+    adult_promoter_removal_reason = Column(String(300))
+    adult_promoter_ban_reason = Column(String(300))
     high_volume_reposter_ban_user = Column(Boolean, default=False)
     high_volume_reposter_remove_post = Column(Boolean, default=False)
     high_volume_reposter_threshold = Column(Integer, default=100)
     high_volume_reposter_notify_mod_mail = Column(Boolean, default=False)
+    high_volume_reposter_removal_reason = Column(String(300))
+    high_volume_reposter_ban_reason = Column(String(300))
 
     post_checks = relationship("MonitoredSubChecks", back_populates='monitored_sub', cascade='all, delete', )
     config_revisions = relationship("MonitoredSubConfigRevision", back_populates='monitored_sub', cascade='all, delete')
@@ -422,7 +427,12 @@ def to_dict(self):
             'high_volume_reposter_ban_user': self.high_volume_reposter_ban_user,
             'high_volume_reposter_remove_post': self.high_volume_reposter_remove_post,
             'high_volume_reposter_threshold': self.high_volume_reposter_threshold,
-            'high_volume_reposter_notify_mod_mail': self.high_volume_reposter_notify_mod_mail
+            'high_volume_reposter_notify_mod_mail': self.high_volume_reposter_notify_mod_mail,
+            'high_volume_reposter_removal_reason': self.high_volume_reposter_removal_reason,
+            'high_volume_reposter_ban_reason': self.high_volume_reposter_ban_reason,
+            'adult_promoter_removal_reason': self.adult_promoter_removal_reason,
+            'adult_promoter_ban_reason': self.adult_promoter_ban_reason
+
 
         }
 

redditrepostsleuth/core/model/misc_models.py

@@ -11,6 +11,7 @@ class JobStatus(Enum):
 	TIMEOUT = auto()
 	PROXYERROR = auto()
 	ERROR = auto()
+	RATELIMIT = auto()
 
 @dataclass
 class BatchedPostRequestJob:

redditrepostsleuth/core/services/responsebuilder.py

@@ -130,7 +130,7 @@ def build_sub_comment(
 
         try:
             return self.build_default_comment(search_results, message, **kwargs)
-        except KeyError:
+        except KeyError as e:
             log.warning('Custom repost template for %s has a bad slug: %s', monitored_sub.name, monitored_sub.repost_response_template)
             return self.build_default_comment(search_results, **kwargs)
 

redditrepostsleuth/core/util/default_bot_config.py

@@ -33,9 +33,13 @@
             "adult_promoter_remove_post": False,
             "adult_promoter_ban_user": False,
             "adult_promoter_notify_mod_mail": False,
+            "adult_promoter_ban_reason": None,
+            "adult_promoter_removal_reason": None,
             "high_volume_reposter_ban_user": False,
             "high_volume_reposter_remove_post": False,
             "high_volume_reposter_threshold": 150,
             "high_volume_reposter_notify_mod_mail": False,
+            "high_volume_reposter_removal_reason": None,
+            "high_volume_reposter_ban_reason": None
 
 }