diff --git a/.dockerignore b/.dockerignore index a1ea644..5812c09 100644 --- a/.dockerignore +++ b/.dockerignore @@ -4,3 +4,12 @@ .github/logo .github/workflows + +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..a6d2ea2 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +.PHONY: all prep-dev venv clean lint test docker-test + +# virtual environment for development +VENV_NAME?=venv +VENV_ACTIVATE=. $(VENV_NAME)/bin/activate +PYTHON=${VENV_NAME}/bin/python3 +# help messages for make, it runs in `make` or `make all` +all: + @echo "\033[92m make prep-dev \033[0m" + @echo "---> Prepares dev environment, use only once" + @echo "\033[92m make test \033[0m" + @echo "---> Runs test cases in virtual environment" + @echo "\033[92m make lint \033[0m" + @echo "---> Linting project with autopep8" + @echo "\033[92m make clean \033[0m" + @echo "---> Cleans project cache and other stuffs" + @echo "\033[92m make docker-test \033[0m" + @echo "---> Runs test cases in docker environment" + + +prep-dev: + python3 -m pip install virtualenv ## virtual environment for development purposes + make venv + +venv: $(VENV_NAME)/bin/activate +$(VENV_NAME)/bin/activate: requirements.txt + test -d $(VENV_NAME) || virtualenv -p python3 $(VENV_NAME) + ${PYTHON} -m pip install -U pip setuptools + ${PYTHON} -m pip install -U autopep8 coverage isort + ${PYTHON} -m pip install -U -r requirements.txt + touch $(VENV_NAME)/bin/activate + +clean: + rm -rf $(VENV_NAME) *.eggs *.egg-info dist build docs/_build .cache .coverage + rm -rf .pytest* # cache file for Intellij PyCharm + +sort: venv + isort -rc . --skip_glob docs/* + + +lint: venv + autopep8 --in-place --recursive --max-line-length=100 --exclude docs/source/conf.py,venv,__pycache__,old,build,dist . + +test: venv + coverage run --source=richkit -m pytest -Werror --ignore src/python-whois + +docker-test: clean + docker build -t richkit-docker-test -f Dockerfile.test . + docker run -e MAXMIND_LICENSE_KEY=$MAXMIND_LICENSE_KEY richkit-docker-test \ No newline at end of file diff --git a/richkit/analyse/analyse.py b/richkit/analyse/analyse.py index 4b04c18..77712cd 100644 --- a/richkit/analyse/analyse.py +++ b/richkit/analyse/analyse.py @@ -123,11 +123,11 @@ def get_domain_name_features(domain): """ domain_array = domain.split('.') num_tokens = len(domain_array) - len2ld = len(get_sld(domain)) + len2ld = len(get_sld(domain)) len_domain = sum([len(el) for el in domain_array]) domain_name_features = { "num_tokens": str(num_tokens), - "len2ld" : str(len2ld), + "len2ld": str(len2ld), "len_domain": str(len_domain) } return domain_name_features @@ -271,7 +271,6 @@ def ngram_count(domain, counts, counts_vc): def get_num_numeric_2ld(domain): - """ :param domain: @@ -281,7 +280,6 @@ def get_num_numeric_2ld(domain): def get_radio_numeric_2ld(domain): - """ :param domain: diff --git a/richkit/retrieve/cert_sh.py b/richkit/retrieve/cert_sh.py index 8e2fe78..d9b3c36 100644 --- a/richkit/retrieve/cert_sh.py +++ b/richkit/retrieve/cert_sh.py @@ -40,7 +40,7 @@ def get_certificates(self, domain): return json.loads(content) except Exception as e: logger.error('Error while retrieving certificates: %s', e) - raise e + return None def get_all(self): """ diff --git a/richkit/test/analyse/test_analyse.py b/richkit/test/analyse/test_analyse.py index f69f8ba..cf4a269 100644 --- a/richkit/test/analyse/test_analyse.py +++ b/richkit/test/analyse/test_analyse.py @@ -153,7 +153,7 @@ def test_nld(self): def test_n_label(self): for k, v in self.domain.items(): n_label3 = analyse.n_label(k, 3) - self.assertEqual(n_label3,v['n_label']) + self.assertEqual(n_label3, v['n_label']) def test_depth(self): for k, v in self.domain.items(): diff --git a/richkit/test/retrieve/test_urlvoid.py b/richkit/test/retrieve/test_urlvoid.py index ca1e409..33cb9ad 100644 --- a/richkit/test/retrieve/test_urlvoid.py +++ b/richkit/test/retrieve/test_urlvoid.py @@ -36,7 +36,7 @@ def test_domain_registration_date(self): for k, v in self.test_urls.items(): instance = URLVoid(k) domain_registration = instance.domain_registration_date()[:-15] - self.assertEqual(domain_registration,v["domain_registration"]) + self.assertEqual(domain_registration, v["domain_registration"]) def test_get_detection_rate(self): for k, v in self.test_urls.items(): diff --git a/richkit/test/retrieve/test_x509.py b/richkit/test/retrieve/test_x509.py index b1564be..51788fc 100644 --- a/richkit/test/retrieve/test_x509.py +++ b/richkit/test/retrieve/test_x509.py @@ -5,7 +5,8 @@ class Test_x509(unittest.TestCase): def setUp(self): - self.sans = ['*.google.com', 'mail.google.com', 'example.com', 'test.example.dk', 'test_domain.co.uk'] + self.sans = ['*.google.com', 'mail.google.com', + 'example.com', 'test.example.dk', 'test_domain.co.uk'] def test_unique_apex(self): assert unique_apex(self.sans) == 4 @@ -18,4 +19,3 @@ def test_unique_sld(self): def test_lcs(self): assert get_lcs_apex(self.sans) == 11 -