Merge pull request #20 from pablodav/feature/19_compliance_totsize_fo…

…r_status_not_idle

Feature/19 compliance totsize for status not idle

.circleci/config.yml

@@ -0,0 +1,62 @@
+# This configuration was automatically generated from a CircleCI 1.0 config.
+# It should include any build commands you had along with commands that CircleCI
+# inferred from your project structure. We strongly recommend you read all the
+# comments in this file to understand the structure of CircleCI 2.0, as the idiom
+# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather
+# than the prescribed lifecycle of 1.0. In general, we recommend using this generated
+# configuration as a reference rather than using it in production, though in most
+# cases it should duplicate the execution of your original 1.0 config.
+version: 2
+jobs:
+  build:
+    working_directory: ~/pablodav/burp_server_reports
+    parallelism: 1
+    shell: /bin/bash --login
+    # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did.
+    # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables .
+    #environment:
+    #  CIRCLE_ARTIFACTS: /tmp/circleci-artifacts
+    #  CIRCLE_TEST_REPORTS: /tmp/circleci-test-results
+    #  TOX_PY34: 3.4.3
+    #  TOX_PY35: 3.5.2
+    #  TOX_PY36: 3.6.1
+    # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages.
+    # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images.
+    # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job.
+    # We have selected a pre-built image that mirrors the build environment we use on
+    # the 1.0 platform, but we recommend you choose an image more tailored to the needs
+    # of each job. For more information on choosing an image (or alternatively using a
+    # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/
+    # To see the list of pre-built images that CircleCI provides for most common languages see
+    # https://circleci.com/docs/2.0/circleci-images/
+    docker:
+    - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37
+      command: /sbin/init
+    steps:
+    # https://circleci.com/docs/2.0/building-docker-images/
+    - setup_remote_docker
+    # Machine Setup
+    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each
+    # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out.
+    - checkout
+    # This is based on your 1.0 configuration file or project settings
+    - run:
+        working_directory: ~/pablodav/burp_server_reports
+        command: echo -e "export TOX_PY34=3.4.3\nexport TOX_PY35=3.5.2\nexport TOX_PY36=3.6.1" >> $BASH_ENV
+    - run:
+        working_directory: ~/pablodav/burp_server_reports
+        command: 'sudo docker info >/dev/null 2>&1 || sudo service docker start; '
+    # This is based on your 1.0 configuration file or project settings
+    - run: pip -V
+    - run: pip install -U pip
+    - run: pip install --upgrade tox
+    - run: pip install --upgrade tox-pyenv
+    - run: pyenv local $TOX_PY34 $TOX_PY35 $TOX_PY36
+    - run: docker pull greenmail/standalone:1.5.5
+    - run: docker run -d -p 3025:3025 -p 3110:3110 -p 3143:3143 -p 3465:3465 -p 3993:3993 -p 3995:3995 greenmail/standalone:1.5.5
+    # Test
+    #   This would typically be a build job when using workflows, possibly combined with build
+    # This is based on your 1.0 configuration file or project settings
+    - run: tox -v --recreate
+    # Teardown
+    #   If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each

.gitignore

@@ -85,3 +85,5 @@ test_write_config.conf
 # tests
 inventory_email.csv
 
+# vscode
+.vscode

.pytest_cache/v/cache/lastfailed

@@ -0,0 +1 @@
+{}

.travis.yml

@@ -0,0 +1,21 @@
+# refresh travis
+sudo: required
+language: python
+python:
+  - "3.4.3"
+  - "3.5.2"
+  - "3.6.1"
+services:
+  - docker
+before_install:
+  - sudo apt-get -qq update
+  - pip -V
+  - pip install -U pip
+  - pip install --upgrade tox-travis
+  #- pip install --upgrade tox
+  #- pip install --upgrade tox-pyenv
+  - docker pull greenmail/standalone:1.5.5
+  - docker run -d -p 3025:3025 -p 3110:3110 -p 3143:3143 -p 3465:3465 -p 3993:3993 -p 3995:3995 greenmail/standalone:1.5.5
+# install:
+script:
+  - tox -v --recreate

CHANGELOG

@@ -1,3 +1,7 @@
+1.4.0:
+       * feature #19 - be smarter and get backup report for clients without status idle, see "Smarter check by default for outdated" in readme.
+       * Migrate ci to travis-ci
+       * Increase code coverage to from 94% to 96%
 1.3.3:
        * upgrade invpy_libs to 0.4.2 to fix errors with encoding on csv files
 1.3.1:

README.rst

@@ -1,11 +1,8 @@
 .. image:: https://badge.fury.io/py/burp-reports.svg
     :target: https://badge.fury.io/py/burp-reports
 
-.. image:: https://circleci.com/gh/pablodav/burp_server_reports.svg?style=svg
-    :target: https://circleci.com/gh/pablodav/burp_server_reports
-
-.. image:: https://circleci.com/gh/pablodav/burp_server_reports.svg?style=svg
-    :target: https://circleci.com/gh/pablodav/burp_server_reports
+.. image:: https://travis-ci.org/pablodav/burp_server_reports.svg?branch=master
+    :target: https://travis-ci.org/pablodav/burp_server_reports
 
 .. image:: https://codecov.io/gh/pablodav/burp_server_reports/branch/master/graph/badge.svg
   :target: https://codecov.io/gh/pablodav/burp_server_reports
@@ -337,7 +334,7 @@ See outdated::
     burp_reports -ui http://burpui_apiurl:port -c config_file.conf --report outdated
     burp_reports -ui http://burpui_apiurl:port --report outdated
 
-See outdated with more details (very recommended as it will also check if backup has 0B and report as never)::
+See outdated with more details::
 
     burp_reports -ui http://burpui_apiurl:port -c config_file.conf --report outdated --detail
 
@@ -357,6 +354,35 @@ See all clients with details::
 
     burp_reports -ui http://burpui_apiurl:port -c config_file.conf --report print --detail
 
+
+Smarter check by default for outdated
+=====================================
+
+feature #19
+
+Example of normal report with burpui demo::
+
+    burp report                                                                                      2018-03-10 18:54:50
+                            Name     Date(local)  Time(local)  State        Phase
+                            agent   ---          ---         idle          ---
+                        demo-pablo  2018-03-10   17:35:50     client cras   ---
+                            demo1  2018-03-10   15:42:02     idle          ---
+                            demo2  2018-03-10   14:17:02     server cras   ---
+                            demo3  2018-03-10   17:24:07     idle          ---
+                            demo4  2018-03-10   16:59:03     idle          ---
+    [pablo@localhost burp_server_reports]$ burp-reports -c burp_reports/data/test_config_demo.conf --report outdated
+
+    burp report                                                                                      2018-03-10 18:55:01
+                            Name     Date(local)  Time(local)  State        Phase        Status
+                        demo-pablo   ---          ---          ---          ---         never
+
+
+As you can see, demo-pablo has date of backup: so in the past if check for dates It can think it is updated/ok!  
+but now burp-reports is smarter and checks for clients non idle by default and identifies this kind of client without backup!
+This client was created in: https://git.ziirish.me/ziirish/burp-ui/issues/252#note_2644 for demostrating this behaviour.
+
+When you use --detail parameter, it checks for backup size for every client, doesn't matter its status, so is slower but more accurate too.
+
 Packaging: 
 ----------
 

burp_reports/VERSION

@@ -1 +1 @@
-1.3.3
+1.4.0

burp_reports/backends/burpui_api.py

@@ -13,18 +13,23 @@ class Clients:
 
     """
 
-    def __init__(self, apiurl):
+    def __init__(self, apiurl, check_idle=True):
         """
         should be the api to connect
         like: http:/user:password@server:port/api/
 
+        Use get_clients_stats to get the default list of clients
+
         :param apiurl
+        :param check_idle: Will enable on function like get_clients_stats to verify each client for those without idle state and 
+        a latest date of backup.
         """
 
         self.apiurl = apiurl
 
         self.IsMultiAgent = self._is_multi_agent()
         self.empty_backup_report = default_client_backup_report()
+        self.check_idle = check_idle
 
     @lru_cache(maxsize=32)
     def _is_multi_agent(self):
@@ -220,9 +225,63 @@ def _get_client_report_stats(self, client, server=None):
 
         return client_report
 
+    def _check_idle_state_clients_stats(self, clients_stats):
+        """
+        Receive list generated by api client, like get_clients_stats returns.
+        Verifies each client state and last not never
+        Then get get_client_report information to have totsize in backup_report nested dict
+        :param clients_stats exmple:
+        [{
+            "phase": 'null',
+            "percent": 0,
+            "state": "idle",
+            "last": "2016-06-23 14:33:06-03:00",
+            "name": "monitor"},
+            {
+                "last": "never",
+                "name": "client2",
+                "state": "idle",
+                "phase": "phase2",
+                "percent": 42,
+            }
+        ]
+        """
+        # Create a new list to return
+        clients_report = []
+
+        for i, values in enumerate(clients_stats):
+            client_state = clients_stats[i].get('state', None)
+            client_last = clients_stats[i].get('last', None)   
+            # Create dict with all data of the client's dict
+            client_report_dict = defaultdict(dict)
+            client_report_dict.update(values)
+
+            if client_state != 'idle' and client_last not in ['never', None]:
+
+                client = client_report_dict.get('name', None)
+                logging.debug("client {} has status != idle and client_last not in never, getting totsize".format(client))
+                if not client:
+                    continue
+                server = client_report_dict.get('server', None)
+                # running clients on server
+                server_running = self.get_clients_running(server)
+                # Omit getting stats for clients running a backup
+                # burpui doesn't return data when client is running.
+                if client in server_running:
+                    continue
+
+            client_report_dict = self._get_client_report_backups(client_report_dict)
+
+            clients_report.append(client_report_dict)
+
+        return clients_report
+
     @lru_cache(maxsize=32)
     def get_clients_stats(self):
         """
+        # Default function used to get a list of clients with fastest as possible.
+        # Now adding a verification of status: idle to identify clients without backup
+        # Will be slower but could be disabled
         #  server.get_all_clients()
         #
         :return list: example: [{
@@ -256,13 +315,16 @@ def get_clients_stats(self):
             serviceurl = self.apiurl + 'clients/stats'
             clients_stats = get_url_data(serviceurl=serviceurl)
 
+        if self.check_idle:
+            clients_stats = self._check_idle_state_clients_stats(clients_stats)
+
         return clients_stats
 
     @lru_cache(maxsize=32)
     def get_clients_reports_brief(self) -> list:
 
         # For multi server:
-        # https://burp-ui.readthedocs.io/en/latest/api.html#get--api-clients-(server)-report
+        # https://burp-ui.readthedocs.io/en/stable/api.html#get--api-clients-(server)-report
         # For single server:
         # https://burp-ui.readthedocs.io/en/latest/api.html#get--api-clients-report
         # GET /api/clients/report
@@ -333,55 +395,71 @@ def get_clients_reports(self):
 
         for cli in range(len(clients_stats)):
 
-            # Server, client is required to fetch report_stats
-            server = clients_stats[cli].get('server', None)
-            logging.debug('server: {}'.format(server))
-            client = clients_stats[cli].get('name', None)
-
-            # Omit getting stats for clients running a backup
-            # burpui doesn't return data when client is running.
-            server_running = self.get_clients_running(server)
-            if client in server_running:
-                continue
-
             # Create dict with all data of the client's dict
             client_report_dict = defaultdict(dict)
             client_report_dict.update(clients_stats[cli])
-            # Create new list to use a list of numbers of backups only
-            backups = []
-
+            client = client_report_dict.get('name', None)
             if not client:
                 continue
+            server = client_report_dict.get('server', None)
+            # running clients on server
+            server_running = self.get_clients_running(server)
+            # Omit getting stats for clients running a backup
+            # burpui doesn't return data when client is running.
+            if client in server_running:
+                continue
 
-            if clients_stats[cli].get('last', 'None') not in ['None', 'never']:
-
-                # Get client_report_stats ;
-                # It is a list of all backups stats
-                cr_stats = self._get_client_report_stats(client, server=server)
-                # and create a list of backups numbers only
-                for n in range(len(cr_stats)):
-                    backups.append(cr_stats[n].get('number'))
-
-                if backups:
-                    # Get the maximum number of backup to use
-                    # The first backup could have date but not being reported with number, so no statistics.
-                    # For that reason I add this if backups:
-                    number = max(backups)
-
-                    # Add the backup_report to the dict of the client
-                    client_report_dict['backup_report'] = self._get_backup_report_stats(client, number, server=server)
-                else:
-                    client_report_dict['backup_report'] = self.empty_backup_report
-            else:
-                client_report_dict['backup_report'] = self.empty_backup_report
-
-            if not client_report_dict['backup_report'].get('totsize'):
-                client_report_dict['backup_report']['totsize'] = 0
-            if not client_report_dict['backup_report'].get('received'):
-                client_report_dict['backup_report']['received'] = 0
-            if not client_report_dict['backup_report'].get('duration'):
-                client_report_dict['backup_report']['duration'] = 0
+            client_report_dict = self._get_client_report_backups(client_report_dict)
 
             clients_report.append(client_report_dict)
 
         return clients_report
+
+    def _get_client_report_backups(self, client_report_dict):
+        """
+
+        :param client_report_dict: should be in format of default dict, with the data of clients_stats dict for
+        one client.
+        Generate it with:
+            # Create dict with all data of the client's dict
+            client_report_dict = defaultdict(dict)
+            client_report_dict.update(clients_stats[cli])
+        :return: same dict with appended 'backup_report' as nested dict with totsize, received, duration, etc.
+        """
+        # Create new list to use a list of numbers of backups only
+        backups = []
+        # Server, client is required to fetch report_stats
+        server = client_report_dict.get('server', None)
+        logging.debug('server: {}'.format(server))
+        client = client_report_dict.get('name', None)
+
+        if client_report_dict.get('last', 'None') not in ['None', 'never']:
+
+            # Get client_report_stats ;
+            # It is a list of all backups stats
+            cr_stats = self._get_client_report_stats(client, server=server)
+            # and create a list of backups numbers only
+            for n in range(len(cr_stats)):
+                backups.append(cr_stats[n].get('number'))
+
+            if backups:
+                # Get the maximum number of backup to use
+                # The first backup could have date but not being reported with number, so no statistics.
+                # For that reason I add this if backups:
+                number = max(backups)
+
+                # Add the backup_report to the dict of the client
+                client_report_dict['backup_report'] = self._get_backup_report_stats(client, number, server=server)
+            else:
+                client_report_dict['backup_report'] = self.empty_backup_report
+        else:
+            client_report_dict['backup_report'] = self.empty_backup_report
+
+        if not client_report_dict['backup_report'].get('totsize'):
+            client_report_dict['backup_report']['totsize'] = 0
+        if not client_report_dict['backup_report'].get('received'):
+            client_report_dict['backup_report']['received'] = 0
+        if not client_report_dict['backup_report'].get('duration'):
+            client_report_dict['backup_report']['duration'] = 0
+
+        return client_report_dict