diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 0000000..87b6983 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,62 @@ +# This configuration was automatically generated from a CircleCI 1.0 config. +# It should include any build commands you had along with commands that CircleCI +# inferred from your project structure. We strongly recommend you read all the +# comments in this file to understand the structure of CircleCI 2.0, as the idiom +# for configuration has changed substantially in 2.0 to allow arbitrary jobs rather +# than the prescribed lifecycle of 1.0. In general, we recommend using this generated +# configuration as a reference rather than using it in production, though in most +# cases it should duplicate the execution of your original 1.0 config. +version: 2 +jobs: + build: + working_directory: ~/pablodav/burp_server_reports + parallelism: 1 + shell: /bin/bash --login + # CircleCI 2.0 does not support environment variables that refer to each other the same way as 1.0 did. + # If any of these refer to each other, rewrite them so that they don't or see https://circleci.com/docs/2.0/env-vars/#interpolating-environment-variables-to-set-other-environment-variables . + #environment: + # CIRCLE_ARTIFACTS: /tmp/circleci-artifacts + # CIRCLE_TEST_REPORTS: /tmp/circleci-test-results + # TOX_PY34: 3.4.3 + # TOX_PY35: 3.5.2 + # TOX_PY36: 3.6.1 + # In CircleCI 1.0 we used a pre-configured image with a large number of languages and other packages. + # In CircleCI 2.0 you can now specify your own image, or use one of our pre-configured images. + # The following configuration line tells CircleCI to use the specified docker image as the runtime environment for you job. + # We have selected a pre-built image that mirrors the build environment we use on + # the 1.0 platform, but we recommend you choose an image more tailored to the needs + # of each job. For more information on choosing an image (or alternatively using a + # VM instead of a container) see https://circleci.com/docs/2.0/executor-types/ + # To see the list of pre-built images that CircleCI provides for most common languages see + # https://circleci.com/docs/2.0/circleci-images/ + docker: + - image: circleci/build-image:ubuntu-14.04-XXL-upstart-1189-5614f37 + command: /sbin/init + steps: + # https://circleci.com/docs/2.0/building-docker-images/ + - setup_remote_docker + # Machine Setup + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each + # The following `checkout` command checks out your code to your working directory. In 1.0 we did this implicitly. In 2.0 you can choose where in the course of a job your code should be checked out. + - checkout + # This is based on your 1.0 configuration file or project settings + - run: + working_directory: ~/pablodav/burp_server_reports + command: echo -e "export TOX_PY34=3.4.3\nexport TOX_PY35=3.5.2\nexport TOX_PY36=3.6.1" >> $BASH_ENV + - run: + working_directory: ~/pablodav/burp_server_reports + command: 'sudo docker info >/dev/null 2>&1 || sudo service docker start; ' + # This is based on your 1.0 configuration file or project settings + - run: pip -V + - run: pip install -U pip + - run: pip install --upgrade tox + - run: pip install --upgrade tox-pyenv + - run: pyenv local $TOX_PY34 $TOX_PY35 $TOX_PY36 + - run: docker pull greenmail/standalone:1.5.5 + - run: docker run -d -p 3025:3025 -p 3110:3110 -p 3143:3143 -p 3465:3465 -p 3993:3993 -p 3995:3995 greenmail/standalone:1.5.5 + # Test + # This would typically be a build job when using workflows, possibly combined with build + # This is based on your 1.0 configuration file or project settings + - run: tox -v --recreate + # Teardown + # If you break your build into multiple jobs with workflows, you will probably want to do the parts of this that are relevant in each diff --git a/.gitignore b/.gitignore index 920af60..1c6d382 100644 --- a/.gitignore +++ b/.gitignore @@ -85,3 +85,5 @@ test_write_config.conf # tests inventory_email.csv +# vscode +.vscode diff --git a/.pytest_cache/v/cache/lastfailed b/.pytest_cache/v/cache/lastfailed new file mode 100644 index 0000000..9e26dfe --- /dev/null +++ b/.pytest_cache/v/cache/lastfailed @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9ac76fd --- /dev/null +++ b/.travis.yml @@ -0,0 +1,21 @@ +# refresh travis +sudo: required +language: python +python: + - "3.4.3" + - "3.5.2" + - "3.6.1" +services: + - docker +before_install: + - sudo apt-get -qq update + - pip -V + - pip install -U pip + - pip install --upgrade tox-travis + #- pip install --upgrade tox + #- pip install --upgrade tox-pyenv + - docker pull greenmail/standalone:1.5.5 + - docker run -d -p 3025:3025 -p 3110:3110 -p 3143:3143 -p 3465:3465 -p 3993:3993 -p 3995:3995 greenmail/standalone:1.5.5 +# install: +script: + - tox -v --recreate diff --git a/CHANGELOG b/CHANGELOG index b5b409f..9156cd4 100644 --- a/CHANGELOG +++ b/CHANGELOG @@ -1,3 +1,7 @@ +1.4.0: + * feature #19 - be smarter and get backup report for clients without status idle, see "Smarter check by default for outdated" in readme. + * Migrate ci to travis-ci + * Increase code coverage to from 94% to 96% 1.3.3: * upgrade invpy_libs to 0.4.2 to fix errors with encoding on csv files 1.3.1: diff --git a/README.rst b/README.rst index 7fc0c38..d36da61 100644 --- a/README.rst +++ b/README.rst @@ -1,11 +1,8 @@ .. image:: https://badge.fury.io/py/burp-reports.svg :target: https://badge.fury.io/py/burp-reports -.. image:: https://circleci.com/gh/pablodav/burp_server_reports.svg?style=svg - :target: https://circleci.com/gh/pablodav/burp_server_reports - -.. image:: https://circleci.com/gh/pablodav/burp_server_reports.svg?style=svg - :target: https://circleci.com/gh/pablodav/burp_server_reports +.. image:: https://travis-ci.org/pablodav/burp_server_reports.svg?branch=master + :target: https://travis-ci.org/pablodav/burp_server_reports .. image:: https://codecov.io/gh/pablodav/burp_server_reports/branch/master/graph/badge.svg :target: https://codecov.io/gh/pablodav/burp_server_reports @@ -337,7 +334,7 @@ See outdated:: burp_reports -ui http://burpui_apiurl:port -c config_file.conf --report outdated burp_reports -ui http://burpui_apiurl:port --report outdated -See outdated with more details (very recommended as it will also check if backup has 0B and report as never):: +See outdated with more details:: burp_reports -ui http://burpui_apiurl:port -c config_file.conf --report outdated --detail @@ -357,6 +354,35 @@ See all clients with details:: burp_reports -ui http://burpui_apiurl:port -c config_file.conf --report print --detail + +Smarter check by default for outdated +===================================== + +feature #19 + +Example of normal report with burpui demo:: + + burp report 2018-03-10 18:54:50 + Name Date(local) Time(local) State Phase + agent --- --- idle --- + demo-pablo 2018-03-10 17:35:50 client cras --- + demo1 2018-03-10 15:42:02 idle --- + demo2 2018-03-10 14:17:02 server cras --- + demo3 2018-03-10 17:24:07 idle --- + demo4 2018-03-10 16:59:03 idle --- + [pablo@localhost burp_server_reports]$ burp-reports -c burp_reports/data/test_config_demo.conf --report outdated + + burp report 2018-03-10 18:55:01 + Name Date(local) Time(local) State Phase Status + demo-pablo --- --- --- --- never + + +As you can see, demo-pablo has date of backup: so in the past if check for dates It can think it is updated/ok! +but now burp-reports is smarter and checks for clients non idle by default and identifies this kind of client without backup! +This client was created in: https://git.ziirish.me/ziirish/burp-ui/issues/252#note_2644 for demostrating this behaviour. + +When you use --detail parameter, it checks for backup size for every client, doesn't matter its status, so is slower but more accurate too. + Packaging: ---------- diff --git a/burp_reports/VERSION b/burp_reports/VERSION index 31e5c84..e21e727 100644 --- a/burp_reports/VERSION +++ b/burp_reports/VERSION @@ -1 +1 @@ -1.3.3 +1.4.0 \ No newline at end of file diff --git a/burp_reports/backends/burpui_api.py b/burp_reports/backends/burpui_api.py index fcae914..5f7061d 100644 --- a/burp_reports/backends/burpui_api.py +++ b/burp_reports/backends/burpui_api.py @@ -13,18 +13,23 @@ class Clients: """ - def __init__(self, apiurl): + def __init__(self, apiurl, check_idle=True): """ should be the api to connect like: http:/user:password@server:port/api/ + Use get_clients_stats to get the default list of clients + :param apiurl + :param check_idle: Will enable on function like get_clients_stats to verify each client for those without idle state and + a latest date of backup. """ self.apiurl = apiurl self.IsMultiAgent = self._is_multi_agent() self.empty_backup_report = default_client_backup_report() + self.check_idle = check_idle @lru_cache(maxsize=32) def _is_multi_agent(self): @@ -220,9 +225,63 @@ def _get_client_report_stats(self, client, server=None): return client_report + def _check_idle_state_clients_stats(self, clients_stats): + """ + Receive list generated by api client, like get_clients_stats returns. + Verifies each client state and last not never + Then get get_client_report information to have totsize in backup_report nested dict + :param clients_stats exmple: + [{ + "phase": 'null', + "percent": 0, + "state": "idle", + "last": "2016-06-23 14:33:06-03:00", + "name": "monitor"}, + { + "last": "never", + "name": "client2", + "state": "idle", + "phase": "phase2", + "percent": 42, + } + ] + """ + # Create a new list to return + clients_report = [] + + for i, values in enumerate(clients_stats): + client_state = clients_stats[i].get('state', None) + client_last = clients_stats[i].get('last', None) + # Create dict with all data of the client's dict + client_report_dict = defaultdict(dict) + client_report_dict.update(values) + + if client_state != 'idle' and client_last not in ['never', None]: + + client = client_report_dict.get('name', None) + logging.debug("client {} has status != idle and client_last not in never, getting totsize".format(client)) + if not client: + continue + server = client_report_dict.get('server', None) + # running clients on server + server_running = self.get_clients_running(server) + # Omit getting stats for clients running a backup + # burpui doesn't return data when client is running. + if client in server_running: + continue + + client_report_dict = self._get_client_report_backups(client_report_dict) + + clients_report.append(client_report_dict) + + return clients_report + @lru_cache(maxsize=32) def get_clients_stats(self): """ + # Default function used to get a list of clients with fastest as possible. + # Now adding a verification of status: idle to identify clients without backup + # Will be slower but could be disabled # server.get_all_clients() # :return list: example: [{ @@ -256,13 +315,16 @@ def get_clients_stats(self): serviceurl = self.apiurl + 'clients/stats' clients_stats = get_url_data(serviceurl=serviceurl) + if self.check_idle: + clients_stats = self._check_idle_state_clients_stats(clients_stats) + return clients_stats @lru_cache(maxsize=32) def get_clients_reports_brief(self) -> list: # For multi server: - # https://burp-ui.readthedocs.io/en/latest/api.html#get--api-clients-(server)-report + # https://burp-ui.readthedocs.io/en/stable/api.html#get--api-clients-(server)-report # For single server: # https://burp-ui.readthedocs.io/en/latest/api.html#get--api-clients-report # GET /api/clients/report @@ -333,55 +395,71 @@ def get_clients_reports(self): for cli in range(len(clients_stats)): - # Server, client is required to fetch report_stats - server = clients_stats[cli].get('server', None) - logging.debug('server: {}'.format(server)) - client = clients_stats[cli].get('name', None) - - # Omit getting stats for clients running a backup - # burpui doesn't return data when client is running. - server_running = self.get_clients_running(server) - if client in server_running: - continue - # Create dict with all data of the client's dict client_report_dict = defaultdict(dict) client_report_dict.update(clients_stats[cli]) - # Create new list to use a list of numbers of backups only - backups = [] - + client = client_report_dict.get('name', None) if not client: continue + server = client_report_dict.get('server', None) + # running clients on server + server_running = self.get_clients_running(server) + # Omit getting stats for clients running a backup + # burpui doesn't return data when client is running. + if client in server_running: + continue - if clients_stats[cli].get('last', 'None') not in ['None', 'never']: - - # Get client_report_stats ; - # It is a list of all backups stats - cr_stats = self._get_client_report_stats(client, server=server) - # and create a list of backups numbers only - for n in range(len(cr_stats)): - backups.append(cr_stats[n].get('number')) - - if backups: - # Get the maximum number of backup to use - # The first backup could have date but not being reported with number, so no statistics. - # For that reason I add this if backups: - number = max(backups) - - # Add the backup_report to the dict of the client - client_report_dict['backup_report'] = self._get_backup_report_stats(client, number, server=server) - else: - client_report_dict['backup_report'] = self.empty_backup_report - else: - client_report_dict['backup_report'] = self.empty_backup_report - - if not client_report_dict['backup_report'].get('totsize'): - client_report_dict['backup_report']['totsize'] = 0 - if not client_report_dict['backup_report'].get('received'): - client_report_dict['backup_report']['received'] = 0 - if not client_report_dict['backup_report'].get('duration'): - client_report_dict['backup_report']['duration'] = 0 + client_report_dict = self._get_client_report_backups(client_report_dict) clients_report.append(client_report_dict) return clients_report + + def _get_client_report_backups(self, client_report_dict): + """ + + :param client_report_dict: should be in format of default dict, with the data of clients_stats dict for + one client. + Generate it with: + # Create dict with all data of the client's dict + client_report_dict = defaultdict(dict) + client_report_dict.update(clients_stats[cli]) + :return: same dict with appended 'backup_report' as nested dict with totsize, received, duration, etc. + """ + # Create new list to use a list of numbers of backups only + backups = [] + # Server, client is required to fetch report_stats + server = client_report_dict.get('server', None) + logging.debug('server: {}'.format(server)) + client = client_report_dict.get('name', None) + + if client_report_dict.get('last', 'None') not in ['None', 'never']: + + # Get client_report_stats ; + # It is a list of all backups stats + cr_stats = self._get_client_report_stats(client, server=server) + # and create a list of backups numbers only + for n in range(len(cr_stats)): + backups.append(cr_stats[n].get('number')) + + if backups: + # Get the maximum number of backup to use + # The first backup could have date but not being reported with number, so no statistics. + # For that reason I add this if backups: + number = max(backups) + + # Add the backup_report to the dict of the client + client_report_dict['backup_report'] = self._get_backup_report_stats(client, number, server=server) + else: + client_report_dict['backup_report'] = self.empty_backup_report + else: + client_report_dict['backup_report'] = self.empty_backup_report + + if not client_report_dict['backup_report'].get('totsize'): + client_report_dict['backup_report']['totsize'] = 0 + if not client_report_dict['backup_report'].get('received'): + client_report_dict['backup_report']['received'] = 0 + if not client_report_dict['backup_report'].get('duration'): + client_report_dict['backup_report']['duration'] = 0 + + return client_report_dict diff --git a/burp_reports/data/test_config_demo.conf b/burp_reports/data/test_config_demo.conf new file mode 100644 index 0000000..25fb2cd --- /dev/null +++ b/burp_reports/data/test_config_demo.conf @@ -0,0 +1,50 @@ +[common] +burpui_apiurl = https://admin:admin@demo.burp-ui.org/api/ +csv_delimiter = ; +excluded_clients = monitor,agent +days_outdated = 31 + +[inventory_columns] +sub_status = status (detailed) +status = status +server = server +client_name = device name + +[inventory_status] +in_inventory_updated = ok +spare_in_burp = wrong spare in burp +spare = spare +in_many_servers = duplicated +inactive_not_in_burp = ignored inactive +inactive_in_burp = wrong not active +spare_not_in_burp = ignored spare +in_inventory_not_in_burp = absent +active = active +not_inventory_in_burp = not in inventory + +[email_notification] +smtp_login = +smtp_port = 25 +smtp_server = localhost +smtp_password = +smtp_mode = normal +foot_notes = A sample notes +email_from = bpowerhome@domain.com +subject = burp report from bpowerhome +email_to = root@localhost + +[format_text] +all_column_length = 11 +name_length = 15 + +[email_inventory] +imap_search = ALL +imap_port = 3993 +imap_folder = INBOX +imap_host = localhost +attachment_save_directory = . +imap_password = pwd1 +email_subject = inventory +attachment_filename = inventory_email.csv +imap_user = test1 + diff --git a/burp_reports/interfaces/burpui_api_translate.py b/burp_reports/interfaces/burpui_api_translate.py index 1dac74b..034451d 100644 --- a/burp_reports/interfaces/burpui_api_translate.py +++ b/burp_reports/interfaces/burpui_api_translate.py @@ -20,12 +20,21 @@ def translate_clients_function(self, data_t): """ :param data_t: list of clients dicts to use for translation + ej: + { + 'b_phase': 'phase', + 'b_state': 'state', + 'b_orig_state': 'state', + 'b_last': 'last', + 'client_name': 'name' + } + :return: d_clients dictionary of clients translated to use in burp_reports example from clients stats: {'client_name': { 'b_last' : 'YYYY-MM-DDTHH:mm:ssZZ', - 'b_state' : 'working/current', + 'b_state' : 'idle/working/current', 'b_phase' : 'phase1/phase2', 'b_date' : 'date(local)', 'b_time' : 'time(local)', @@ -83,10 +92,11 @@ def translate_clients(self): :return: {'client_name': { 'b_last' : '2016-06-23T14:33:06-03:00', - 'b_state' : 'working/current', - 'b_phase' : 'phase1/phase2' - 'b_date' : 'local time date' - 'b_time' : 'local time' + 'b_state' : 'idle/working/current', + 'b_phase' : 'phase1/phase2', + 'b_orig_state': 'idle/working/current', + 'b_date' : 'local time date', + 'b_time' : 'local time', 'backup_report' : 'dict with backup report, data like duration, totsize, received' } } @@ -94,10 +104,11 @@ def translate_clients(self): # Dictionary to use for translation data_t = { - "b_phase": 'phase', - "b_state": "state", - "b_last": "last", - "client_name": "name"} + 'b_phase': 'phase', + 'b_state': 'state', + 'b_orig_state': 'state', + 'b_last': 'last', + 'client_name': 'name'} # Check if key backup_report exists in clients if 'backup_report' in self.clients[0]: diff --git a/burp_reports/reports/clients_reports.py b/burp_reports/reports/clients_reports.py index 4e74f8d..da4b594 100644 --- a/burp_reports/reports/clients_reports.py +++ b/burp_reports/reports/clients_reports.py @@ -95,8 +95,8 @@ def _get_outdated(self): outdated_clients[k] = v outdated_clients[k]['b_status'] = 'outdated' - # When having details, check the totsize of the client) - if self.detail: + # When having backup_report, check the totsize of the client) + if v.get('backup_report'): if v.get('backup_report').get('totsize') == 0: outdated_clients[k]['b_status'] = 'never' diff --git a/burp_reports/tests/test_burpui_demo.py b/burp_reports/tests/test_burpui_demo.py index 83267da..d437319 100644 --- a/burp_reports/tests/test_burpui_demo.py +++ b/burp_reports/tests/test_burpui_demo.py @@ -1,6 +1,7 @@ #!python3 from ..interfaces.burpui_api_interface import BUIClients +from ..backends.burpui_api import Clients burpui_demo='https://admin:admin@demo.burp-ui.org/api/' burpui_standalone='http://admin:admin@172.17.0.2:8080/api/' @@ -20,3 +21,15 @@ def test_bui_demo_detail(): assert isinstance(clients_dict, dict) return clients_dict + +def test_burpui_api_multi(): + bui_api_obj = Clients(apiurl=burpui_demo) + clients = bui_api_obj._get_clients_report_multi() + + assert isinstance(clients, list) + +def test_burpui_api_brief(): + bui_api_obj = Clients(apiurl=burpui_demo) + clients = bui_api_obj.get_clients_reports_brief() + + assert isinstance(clients, list) diff --git a/circle.yml b/circle.yml deleted file mode 100644 index a7b4339..0000000 --- a/circle.yml +++ /dev/null @@ -1,24 +0,0 @@ -machine: - environment: - TOX_PY34: '3.4.3' - TOX_PY35: '3.5.2' - TOX_PY36: '3.6.1' - python: - version: 3.4.3 - # https://circleci.com/docs/1.0/docker/ - services: - - docker - -dependencies: - override: - - pip -V - - pip install -U pip - - pip install --upgrade tox - - pip install --upgrade tox-pyenv - - pyenv local $TOX_PY34 $TOX_PY35 $TOX_PY36 - - docker pull greenmail/standalone:1.5.5 - - docker run -d -p 3025:3025 -p 3110:3110 -p 3143:3143 -p 3465:3465 -p 3993:3993 -p 3995:3995 greenmail/standalone:1.5.5 - -test: - override: - - tox -v --recreate diff --git a/tox.ini b/tox.ini index a7616bd..e5898a5 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ usedevelop = True #setenv = VIRTUAL_ENV={envdir} install_command = pip install -U {opts} {packages} #setenv= TOX_ENV_NAME={envname} -passenv = TOX_* CI CIRCLECI CIRCLE_* +#passenv = TOX_* CI CIRCLECI CIRCLE_* deps = codecov -r{toxinidir}/requirements.txt -r{toxinidir}/test-requirements.txt @@ -35,3 +35,8 @@ basepython = python3.5 [testenv:py36] basepython = python3.6 +[travis] +python = + 3.4: py34 + 3.5: py35 + 3.6: py36