Merge branch 'master' of https://github.com/timgrossmann/InstaPy

.gitignore

@@ -7,6 +7,7 @@ __pycache__/
 *$py.class
 
 quickstart.py
+docker_quickstart.py
 
 # C extensions
 *.so

CHANGELOG.md

@@ -3,37 +3,59 @@ The **goal** of this file is explaining to the users of our project the notable
 
 _The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html)_.
 
+
+## [Unreleased] - 2019-01-17
+### Changed
+- Optimizing Dockerfile for smaller docker image
+
+### Fixed
+- Fix "_Unable to locate element: ...xpath","selector":"//div[text()=\'Likes\'..._" error.
+
+
+## [Unreleased] - 2019-01-16
+### Fixed
+- Fix "_Failed to load desired amount of users!_" issue.
+
+
 ## [Unreleased] - 2019-01-15
 ### Fixed
-- Handle A/B-Test for comments (graphql edge)
+- Handle A/B-Test for comments (graphql edge).
+
 
 ## [Unreleased] - 2019-01-13
 ### Fixed 
-- Adjust docker-compose.yml according to new Dockerfile
+- Adjust docker-compose.yml according to new Dockerfile.
+
 
 ## [Unreleased] - 2019-01-11
 ### Fixed
-- Correctly mount Docker volume, make it work properly with chromedriver installed in assets folder
+- Correctly mount Docker volume, make it work properly with chromedriver installed in assets folder.
+
 
 ## [Unreleased] - 2019-01-10
 ### Added 
-- Feature to remove outgoing unapproved follow requests from private accounts
+- Feature to remove outgoing unapproved follow requests from private accounts.
+
 
 ## [Unreleased] - 2019-01-05
 ### Changed
-- Resolve security warning with new pyyaml version, updated pyyaml to version 4.2b1
+- Resolve security warning with new pyyaml version, updated pyyaml to version 4.2b1.
+
 
 ## [Unreleased] - 2019-01-04
 ### Fixed
-- Fix for non-authenticated proxies in chrome headless browser
+- Fix for non-authenticated proxies in chrome headless browser.
+
 
 ## [Unreleased] - 2019-01-02
 ### Fixed
--  User without timestamp will use the timestamp of previous user
+-  User without timestamp will use the timestamp of previous user.
+
 
 ## [Unreleased] - 2019-01-01
 ### Changed
-- PEP8 layout changes
+- PEP8 layout changes.
+
 
 ## [Unreleased] - 2018-12-17
 ### Added
@@ -49,10 +71,12 @@ _The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 - Fix error occured while liking a comment (raised at #3594).
 - Fix Follow-Likers feature which couldn't fetch likers properly (raised at #3573).
 
+
 ## [Unreleased] - 2018-12-16
 ### Added
 - Save account progress information into database adding the possibility for external tools to collect and organize the account progress.
 
+
 ## [Unreleased] - 2018-12-10
 ### Fixed
 - Fix `person_id` missing in post_unfollow_cleanup() [line 1152].

README.md

@@ -70,7 +70,7 @@ Table of Contents
   * [Skipping user for private account, no profile picture, business account](#skipping-user-for-private-account-no-profile-picture-business-account)
   * [Liking based on the number of existing likes a post has](#liking-based-on-the-number-of-existing-likes-a-post-has)
   * [Commenting based on the number of existing comments a post has](#commenting-based-on-the-number-of-existing-comments-a-post-has)
-  * [Commenting based on madatory words in the description or first comment](#commenting-based-on-madatory-words-in-the-description-or-first-comment)
+  * [Commenting based on mandatory words in the description or first comment](#commenting-based-on-mandatory-words-in-the-description-or-first-comment)
   * [Comment by Locations](#comment-by-locations)
   * [Like by Locations](#like-by-locations)
   * [Like by Tags](#like-by-tags)
@@ -187,7 +187,7 @@ Whenever you run the script, the virtual enviornment must be active.
 
 If you're not familiar with virtualenv, please [read about it here](https://virtualenv.pypa.io/en/stable/) and use it to your advantage.
 In essence, this is be the _only_ Python library you should install as root (e.g., with sudo). All other Python libraries should be inside a virtualenv.
-Running `source venv/bin/activate` will activate the correct Python to run InstaPy. To exit an activated virtualenv run `deactivate'.
+Running `source venv/bin/activate` will activate the correct Python to run InstaPy. To exit an activated virtualenv run `deactivate`.
 Now, copy/paste the `quickstart.py` Python code below and run your first InstaPy script. Remember to run it with Python from the virtualenv, so from `venv/bin/python`. To make sure which Python is used, run `which python`, it will tell you which Python is 'active'.
 
 ### Set it up yourself with this Basic Setup

docker_conf/python/Dockerfile

@@ -1,4 +1,16 @@
-FROM python:3.6
-RUN apt-get update && apt-get install -y chromedriver
-COPY ./requirements.txt /
-RUN pip install -r /requirements.txt
+FROM python:3.6-slim-stretch
+COPY ./requirements.txt /tmp
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends --no-install-suggests \
+      chromedriver \
+      wget \
+      gcc \
+      g++ \
+    && pip install --no-cache-dir -r /tmp/requirements.txt \
+    && apt-get purge -y --auto-remove \
+      gcc \
+      g++ \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+CMD ["./code/wait-for-selenium.sh", "http://selenium:4444/wd/hub", "--", "python", "code/docker_quickstart.py"]

instapy/commenters_util.py

@@ -2,6 +2,7 @@
 # code created by modification of original code copied from
 # https://github.com/timgrossmann/instagram-profilecrawl/blob/master/util
 # /extractor.py
+import time
 from time import sleep
 from datetime import datetime, timedelta
 import random
@@ -14,6 +15,10 @@
 from .util import update_activity
 from .util import web_address_navigator
 from .util import username_url_to_username
+from .util import remove_duplicates
+from .util import scroll_bottom
+from .util import extract_text_from_element
+from .relationship_tools import progress_tracker
 
 from selenium.common.exceptions import NoSuchElementException
 
@@ -288,6 +293,8 @@ def users_liked(browser, photo_url, amount=100):
 
 
 def likers_from_photo(browser, amount=20):
+    """ Get the list of users from the 'Likes' dialog of a photo """
+
     liked_counter_button = "//div/article/div[2]/section[2]/div/div/a"
 
     try:
@@ -326,7 +333,7 @@ def likers_from_photo(browser, amount=20):
 
         # find dialog box
         dialog = browser.find_element_by_xpath(
-            "//div[text()='Likes']/following-sibling::div")
+            "//h1[text()='Likes']/../../following-sibling::div/div")
 
         # scroll down the page
         previous_len = -1
@@ -336,37 +343,32 @@ def likers_from_photo(browser, amount=20):
         update_activity()
         sleep(1)
 
-        follow_buttons = dialog.find_elements_by_xpath(
-            "//div/div/button[text()='Follow']")
+        start_time = time.time()
+        user_list = []
+
+        while (not user_list
+               or (len(user_list) != previous_len)
+               and (len(user_list) < amount)):
 
-        while (len(follow_buttons) != previous_len) and (
-                len(follow_buttons) < amount):
             if previous_len + 10 >= amount:
-                print("Scrolling finished")
+                print("\nScrolling finished")
                 sleep(1)
                 break
 
-            previous_len = len(follow_buttons)
-            browser.execute_script(
-                "arguments[0].scrollTop = arguments[0].scrollHeight", dialog)
-            update_activity()
-            sleep(1)
+            previous_len = len(user_list)
+            scroll_bottom(browser, dialog, 2)
 
-            follow_buttons = dialog.find_elements_by_xpath(
-                "//div/div/button[text()='Follow']")
-            print("Scrolling down... ", previous_len, "->",
-                  len(follow_buttons), " / ", amount)
+            user_blocks = dialog.find_elements_by_tag_name('a')
+            loaded_users = [extract_text_from_element(u) for u in user_blocks
+                            if extract_text_from_element(u)]
+            user_list.extend(loaded_users)
+            user_list = remove_duplicates(user_list, True, None)
 
-        person_list = []
+            # write & update records at Progress Tracker
+            progress_tracker(len(user_list), amount, start_time, None)
 
-        for person in follow_buttons:
-            username_url = person.find_element_by_xpath(
-                "../../../*").find_element_by_tag_name("a").get_attribute(
-                'href')
-            username = username_url_to_username(username_url)
-            person_list.append(username)
-
-        random.shuffle(person_list)
+        print('\n')
+        random.shuffle(user_list)
         sleep(1)
 
         try:
@@ -380,11 +382,11 @@ def likers_from_photo(browser, amount=20):
         print(
             "Got {} likers shuffled randomly whom you can follow:\n{}"
             "\n".format(
-                len(person_list), person_list))
-        return person_list
+                len(user_list), user_list))
+        return user_list
 
     except Exception as exc:
-        print("Some problem occred!\n\t{}".format(str(exc).encode("utf-8")))
+        print("Some problem occured!\n\t{}".format(str(exc).encode("utf-8")))
         return []
 
 
@@ -419,3 +421,4 @@ def get_photo_urls_from_profile(browser, username, links_to_return_amount=1,
     # Code below is unreachable - perhaps get rid of?
     print("Error: Couldnt get pictures links.")
     return []
+

instapy/relationship_tools.py

@@ -1242,5 +1242,8 @@ def progress_tracker(current_value, highest_value, initial_time, logger):
             sys.stdout.flush()
 
     except Exception as exc:
+        if not logger:
+            logger = Settings.logger
+
         logger.info("Error occurred with Progress Tracker:\n{}".format(
             str(exc).encode("utf-8")))

instapy/unfollow_util.py

@@ -737,7 +737,7 @@ def get_users_through_dialog(browser,
     sc_rolled = 0
 
     # find dialog box
-    dialog_address = "//div[3]/div/div/div[2]"
+    dialog_address = "//body/div[2]/div/div/div[2]"
     dialog = browser.find_element_by_xpath(dialog_address)
 
     # scroll to end of follower list to initiate first load which hides the

instapy/util.py

@@ -1115,6 +1115,9 @@ def remove_duplicates(container, keep_order, logger):
             result = set(container)
 
     else:
+        if not logger:
+            logger = Settings.logger
+
         logger.warning("The given data type- '{}' is not supported "
                        "in `remove_duplicates` function, yet!"
                        .format(type(container)))