From 77271d596d1952b15c7010c994f820c8261c949d Mon Sep 17 00:00:00 2001 From: trin Date: Wed, 19 Jun 2024 22:30:36 +0300 Subject: [PATCH 1/3] =?UTF-8?q?=D0=98=D0=BC=D0=BF=D0=BE=D1=80=D1=82=20?= =?UTF-8?q?=D0=BF=D0=BE=D1=81=D1=82=D0=BE=D0=B2=20=D0=B2=20=D0=B4=D0=B5?= =?UTF-8?q?=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commands/import_posts_to_dev.py | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) create mode 100644 authn/management/commands/import_posts_to_dev.py diff --git a/authn/management/commands/import_posts_to_dev.py b/authn/management/commands/import_posts_to_dev.py new file mode 100644 index 000000000..ad5bf8885 --- /dev/null +++ b/authn/management/commands/import_posts_to_dev.py @@ -0,0 +1,137 @@ +import json +import urllib.request +from datetime import datetime, timedelta + +from django.conf import settings +from django.core.management import BaseCommand + +from posts.models.post import Post +from users.models.user import User +from common.markdown.markdown import markdown_text +from utils.strings import random_string + + +class Command(BaseCommand): + help = "Импорт постов с оригинального vas3k.club на dev/local сборки" + + def add_arguments(self, parser): + parser.add_argument( + "--pages", + type=int, + default=1, + help="Количество страниц, забираемых и фида", + ) + + parser.add_argument( + "--skip", + type=int, + default=0, + help="Количество страниц, которые надо пропустить", + ) + + parser.add_argument( + "--force", + action="store_true", + help="Заменять посты, если они уже существуют", + ) + + def handle(self, *args, **options): + if not settings.DEBUG: + return self.stdout.write("☢️ Только для запуска в DEBUG режиме") + + result = { + 'post_exists': 0, + 'post_created': 0, + 'user_created': 0 + } + + for x in range(options['skip'], options['pages'] + options['skip']): + url = "https://vas3k.club/feed.json?page={}".format(x + 1) + self.stdout.write("📁 {}".format(url)) + req = urllib.request.Request(url) + req.add_header('User-Agent', 'Mozilla/5.0') + response = urllib.request.urlopen(req) + data = json.load(response) + for item in data['items']: + # приватные нафиг + if not (item['_club']['is_public']): + continue + + [created, author] = create_user(item['authors'][0]) + if created: + result['user_created'] += 1 + self.stdout.write(" 👤 \"{}\" пользователь создан".format(author.full_name)) + + post_data = dict( + id=item['id'], + title=item['title'], + type=item['_club']['type'], + slug=random_string(10), + text=item['content_text'], + html=markdown_text(item['content_text']), + image=author.avatar, # хак для постов типа "проект", чтобы не лазить по вастрику лишний раз + created_at=item['date_published'], + last_activity_at=item['date_modified'], + comment_count=item['_club']['comment_count'], + view_count=item['_club']['view_count'], + upvotes=item['_club']['upvotes'], + is_visible=True, + is_visible_in_feeds=True, + is_commentable=True, + is_approved_by_moderator=True, + is_public=True, + author_id=author.id, + is_shadow_banned=False, + published_at=item['date_published'], + coauthors=[] + ) + + if Post.objects.filter(pk=item['id']).exists(): + if options['force']: + Post.objects.filter(pk=item['id']).update(**post_data) + else: + result['post_exists'] += 1 + self.stdout.write(" 📌 \"{}\" уже существует".format(item['title'])) + continue + else: + post = Post.objects.create(**post_data) + post.created_at = item['date_published'] + post.last_activity_at = item['date_modified'] + post.save() + + result['post_created'] += 1 + self.stdout.write(" 📄 \"{}\" запись создана".format(item['title'])) + + self.stdout.write("") + self.stdout.write("Итого:") + self.stdout.write("📄 Новых постов: {}".format(result['post_created'])) + self.stdout.write("📌 Уже существовало: {}".format(result['post_exists'])) + self.stdout.write("👤 Новых пользователей: {}".format(result['user_created'])) + + +def create_user(author): + split = author['url'].split('/') + slug = split[-2] + + if User.objects.filter(slug=slug).count() > 0: + return [False, User.objects.get(slug=slug)] + + user = User.objects.create( + slug=slug, + avatar=author['avatar'], + email=random_string(30), + full_name=author['name'], + company="FAANG", + position="Team Lead конечно", + balance=10000, + membership_started_at=datetime.utcnow(), + membership_expires_at=datetime.utcnow() + timedelta(days=365 * 100), + created_at=datetime.utcnow(), + updated_at=datetime.utcnow(), + is_email_verified=True, + moderation_status=User.MODERATION_STATUS_APPROVED, + roles=[], + ) + user.save() + + return [True, user] From d58a38ee41f40459281609a6782fe3295e81d41b Mon Sep 17 00:00:00 2001 From: trin Date: Thu, 20 Jun 2024 07:48:31 +0300 Subject: [PATCH 2/3] =?UTF-8?q?=D0=92=20`create=5Fuser`=20=D0=BF=D0=B5?= =?UTF-8?q?=D1=80=D0=B5=D1=88=D1=91=D0=BB=20=D0=BD=D0=B0=20`get=5For=5Fcre?= =?UTF-8?q?ate`.=20=D0=9F=D0=B5=D1=80=D0=B5=D0=BF=D0=B8=D1=81=D0=B0=D0=BB?= =?UTF-8?q?=20=D0=BB=D0=BE=D0=B3=D0=B8=D0=BA=D1=83=20=D1=81=D0=BE=D0=B7?= =?UTF-8?q?=D0=B4=D0=B0=D0=BD=D0=B8=D1=8F/=D1=80=D0=B5=D0=B4=D0=B0=D0=BA?= =?UTF-8?q?=D1=82=D0=B8=D1=80=D0=BE=D0=B2=D0=B0=D0=BD=D0=B8=D1=8F=20=D0=BF?= =?UTF-8?q?=D0=BE=D1=81=D1=82=D0=B0=D0=B2,=20=D0=B7=D0=B0=D0=B2=D1=8F?= =?UTF-8?q?=D0=B7=D0=B0=D0=BB=20=D0=B2=D1=81=D1=91=20=D0=BD=D0=B0=20`save`?= =?UTF-8?q?,=20=D1=87=D1=82=D0=BE=D0=B1=D1=8B=20=D0=B1=D1=8B=D0=BB=D0=BE?= =?UTF-8?q?=20=D0=BF=D1=80=D0=BE=D1=89=D0=B5=20=D0=B2=D0=B5=D1=88=D0=B0?= =?UTF-8?q?=D1=82=D1=8C=20=D1=82=D1=80=D0=B8=D0=B3=D0=B3=D0=B5=D1=80=D1=8B?= =?UTF-8?q?=20=D0=B4=D0=BB=D1=8F=20=D1=82=D0=BE=D0=B3=D0=BE=20=D0=B6=D0=B5?= =?UTF-8?q?=20manticore,=20=D0=B5=D1=81=D0=BB=D0=B8=20=D0=B1=D1=83=D0=B4?= =?UTF-8?q?=D0=B5=D0=BC=20=D0=B2=D0=BD=D0=B5=D0=B4=D1=80=D1=8F=D1=82=D1=8C?= =?UTF-8?q?.=20=D0=9D=D0=B5=20=D0=BD=D1=80=D0=B0=D0=B2=D0=B8=D1=82=D1=81?= =?UTF-8?q?=D1=8F=2093=20=D1=81=D1=82=D1=80=D0=BE=D1=87=D0=BA=D0=B0=20;((?= =?UTF-8?q?=20=D0=94=D0=BE=D0=B1=D0=B0=D0=B2=D0=B8=D0=BB=20=D0=BE=D0=BF?= =?UTF-8?q?=D0=B8=D1=81=D0=B0=D0=BD=D0=B8=D0=B5=20=D0=B2=20=D0=B8=D0=BD?= =?UTF-8?q?=D1=84=D0=BE=20=D0=B4=D0=BB=D1=8F=20=D1=80=D0=B0=D0=B7=D1=80?= =?UTF-8?q?=D0=B0=D0=B1=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../commands/import_posts_to_dev.py | 47 +++++++++---------- docs/setup.md | 22 +++++++++ 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/authn/management/commands/import_posts_to_dev.py b/authn/management/commands/import_posts_to_dev.py index ad5bf8885..83d30087b 100644 --- a/authn/management/commands/import_posts_to_dev.py +++ b/authn/management/commands/import_posts_to_dev.py @@ -19,7 +19,7 @@ def add_arguments(self, parser): "--pages", type=int, default=1, - help="Количество страниц, забираемых и фида", + help="Количество страниц, забираемых из фида", ) parser.add_argument( @@ -57,12 +57,12 @@ def handle(self, *args, **options): if not (item['_club']['is_public']): continue - [created, author] = create_user(item['authors'][0]) + author, created = create_user(item['authors'][0]) if created: result['user_created'] += 1 self.stdout.write(" 👤 \"{}\" пользователь создан".format(author.full_name)) - post_data = dict( + defaults = dict( id=item['id'], title=item['title'], type=item['_club']['type'], @@ -71,7 +71,6 @@ def handle(self, *args, **options): html=markdown_text(item['content_text']), image=author.avatar, # хак для постов типа "проект", чтобы не лазить по вастрику лишний раз created_at=item['date_published'], - last_activity_at=item['date_modified'], comment_count=item['_club']['comment_count'], view_count=item['_club']['view_count'], upvotes=item['_club']['upvotes'], @@ -86,18 +85,20 @@ def handle(self, *args, **options): coauthors=[] ) - if Post.objects.filter(pk=item['id']).exists(): - if options['force']: - Post.objects.filter(pk=item['id']).update(**post_data) - else: - result['post_exists'] += 1 - self.stdout.write(" 📌 \"{}\" уже существует".format(item['title'])) - continue - else: - post = Post.objects.create(**post_data) - post.created_at = item['date_published'] - post.last_activity_at = item['date_modified'] - post.save() + exists = False + try: + post = Post.objects.get(id=item['id']) + exists = True + except Post.DoesNotExist: + post = Post.objects.create(**defaults) + + if exists and not options['force']: + result['post_exists'] += 1 + self.stdout.write(" 📌 \"{}\" уже существует".format(item['title'])) + continue + + post.__dict__.update(defaults) + post.save() result['post_created'] += 1 self.stdout.write(" 📄 \"{}\" запись создана".format(item['title'])) @@ -113,10 +114,7 @@ def create_user(author): split = author['url'].split('/') slug = split[-2] - if User.objects.filter(slug=slug).count() > 0: - return [False, User.objects.get(slug=slug)] - - user = User.objects.create( + defaults = dict( slug=slug, avatar=author['avatar'], email=random_string(30), @@ -124,14 +122,15 @@ def create_user(author): company="FAANG", position="Team Lead конечно", balance=10000, - membership_started_at=datetime.utcnow(), - membership_expires_at=datetime.utcnow() + timedelta(days=365 * 100), created_at=datetime.utcnow(), updated_at=datetime.utcnow(), + membership_started_at=datetime.now(), + membership_expires_at=datetime.utcnow() + timedelta(days=365 * 100), is_email_verified=True, moderation_status=User.MODERATION_STATUS_APPROVED, roles=[], ) - user.save() - return [True, user] + user, created = User.objects.get_or_create(slug=slug, defaults=defaults) + + return user, created diff --git a/docs/setup.md b/docs/setup.md index 1c6a812d7..b5870ffde 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -87,3 +87,25 @@ To run telegram bot you have to: ## Docker-compose Check out our [docker-compose.yml](https://github.com/vas3k/vas3k.club/blob/master/docker-compose.yml) to understand the infrastructure. + +## Load posts from main vas3k.club to dev/local database + +Sometimes you need fill saome posts/users data from real project. For this case you can use `import_posts_to_dev` command. + +Command fetch https://vas3k.club/feed.json and copy `is_public=True` posts to your database: +```bash +# fetch first page +$ python3 manage.py import_posts_to_dev + +# fetch first 10 pages +$ python3 manage.py import_posts_to_dev --pages 10 + +# fetch 10 pages, starts from page 5 +$ python3 manage.py import_posts_to_dev --pages 10 --skip 5 + +# fetch 10 pages, starts from page 5 and update exists posts +$ python3 manage.py import_posts_to_dev --pages 10 --skip 5 --force + +# if use docker-compose +$ docker exec -it club_app python3 manage.py import_posts_to_dev --pages 2 +``` From 2ba1a804b909711e1625561c34c1bf0d65c7fea5 Mon Sep 17 00:00:00 2001 From: trin Date: Thu, 20 Jun 2024 13:41:21 +0300 Subject: [PATCH 3/3] =?UTF-8?q?=D0=9F=D1=80=D0=B0=D0=B2=D0=BA=D0=B8=20?= =?UTF-8?q?=D0=BE=D1=82=20@igoose1=20=D0=BA=D1=80=D0=BE=D0=BC=D0=B5=20?= =?UTF-8?q?=D0=B4=D0=BE=D0=BA=D0=B8,=20=D0=B4=D0=BE=D0=BA=D0=B0=20=D0=BD?= =?UTF-8?q?=D0=B0=20=D0=BC=D0=BD=D0=B5=D0=BD=D0=B8=D0=B5=20=D1=81=D0=BE?= =?UTF-8?q?=D0=BE=D0=B1=D1=89=D0=B5=D1=81=D1=82=D0=B2=D0=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- authn/management/commands/import_posts_to_dev.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/authn/management/commands/import_posts_to_dev.py b/authn/management/commands/import_posts_to_dev.py index 83d30087b..f99ba81ec 100644 --- a/authn/management/commands/import_posts_to_dev.py +++ b/authn/management/commands/import_posts_to_dev.py @@ -49,12 +49,12 @@ def handle(self, *args, **options): url = "https://vas3k.club/feed.json?page={}".format(x + 1) self.stdout.write("📁 {}".format(url)) req = urllib.request.Request(url) - req.add_header('User-Agent', 'Mozilla/5.0') + req.add_header('User-Agent', 'posts-to-dev') response = urllib.request.urlopen(req) data = json.load(response) for item in data['items']: # приватные нафиг - if not (item['_club']['is_public']): + if not item['_club']['is_public']: continue author, created = create_user(item['authors'][0]) @@ -111,8 +111,7 @@ def handle(self, *args, **options): def create_user(author): - split = author['url'].split('/') - slug = split[-2] + *_, slug, _ = author['url'].split('/') defaults = dict( slug=slug, @@ -124,13 +123,11 @@ def create_user(author): balance=10000, created_at=datetime.utcnow(), updated_at=datetime.utcnow(), - membership_started_at=datetime.now(), + membership_started_at=datetime.utcnow(), membership_expires_at=datetime.utcnow() + timedelta(days=365 * 100), is_email_verified=True, moderation_status=User.MODERATION_STATUS_APPROVED, roles=[], ) - user, created = User.objects.get_or_create(slug=slug, defaults=defaults) - - return user, created + return User.objects.get_or_create(slug=slug, defaults=defaults)