diff --git a/.gitignore b/.gitignore index 14e478c..f2faeae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,8 @@ __pycache__ +*.pyc *.csv *.txt +ckan.access*.log +*.tgz +*.gz .vscode/ diff --git a/locust/.gitignore b/locust/.gitignore index 5ed46e9..27bb49b 100644 --- a/locust/.gitignore +++ b/locust/.gitignore @@ -1,2 +1,3 @@ apache.log result* +ckan.access*.log diff --git a/locust/__pycache__/base.cpython-38.pyc b/locust/__pycache__/base.cpython-38.pyc deleted file mode 100644 index fa3143f..0000000 Binary files a/locust/__pycache__/base.cpython-38.pyc and /dev/null differ diff --git a/locust/__pycache__/from_apache.cpython-38.pyc b/locust/__pycache__/from_apache.cpython-38.pyc deleted file mode 100644 index 7f9f82e..0000000 Binary files a/locust/__pycache__/from_apache.cpython-38.pyc and /dev/null differ diff --git a/locust/__pycache__/locustfile.cpython-38.pyc b/locust/__pycache__/locustfile.cpython-38.pyc deleted file mode 100644 index 614b7f8..0000000 Binary files a/locust/__pycache__/locustfile.cpython-38.pyc and /dev/null differ diff --git a/locust/advanced.py b/locust/advanced.py index 6a3520e..d627132 100644 --- a/locust/advanced.py +++ b/locust/advanced.py @@ -142,7 +142,10 @@ def random_organizations(self): return None results = data.get('result', []) # pick just some organizations randmonly - random_results = sample(results, 5) + if len(results) < 5: + random_results = results + else: + random_results = sample(results, 5) for org in random_results: url = f'/organization/{org}' self.pending_organizations.append(url) @@ -163,7 +166,10 @@ def all_groups(self): return None results = data.get('result', []) # pick just some organizations randmonly - random_results = sample(results, 5) + if len(results) < 5: + random_results = results + else: + random_results = sample(results, 5) for group in random_results: url = f'/group/{group}' self.pending_groups.append(url) diff --git a/locust/base.py b/locust/base.py index d214ac4..dfeb17b 100644 --- a/locust/base.py +++ b/locust/base.py @@ -18,3 +18,7 @@ def orgs(self): @task def groups(self): self.client.get('/group') + + @task + def dataset(self): + self.client.get('/dataset') diff --git a/locust/parse_apache_logs.py b/locust/parse_apache_logs.py index 457d396..94407d5 100644 --- a/locust/parse_apache_logs.py +++ b/locust/parse_apache_logs.py @@ -34,7 +34,7 @@ def process_line(data): if data['request_path'].startswith(ss): return None - if data['method'] != 'GET': + if data['method'] not in ['GET', 'POST']: return None return data @@ -59,9 +59,9 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0): 'harvests': {'regex': re.compile(r'(\/[A-Za-z_]+)?\/harvest(\/)?$'), 'total': 0}, 'api-pkg-search': - {'regex': re.compile(r'/api/(3)?/action/package_search.*'), 'total': 0}, + {'regex': re.compile(r'/api(/3)?/action/package_search.*'), 'total': 0}, 'api-pkg-show': - {'regex': re.compile(r'/api/(3)?/action/package_show.*'), 'total': 0}, + {'regex': re.compile(r'/api(/3)?/action/package_show.*'), 'total': 0}, 'api-search': {'regex': re.compile(r'/api/search.*'), 'total': 0}, 'organization': @@ -76,10 +76,18 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0): {'regex': re.compile(r'(\/[A-Za-z_]+)?\/group\?(.*)'), 'total': 0}, 'groups': {'regex': re.compile(r'(\/[A-Za-z_]+)?\/group(\/)?$'), 'total': 0}, + 'status-show': + {'regex': re.compile(r'/api/action/status_show'), 'total': 0}, + 'tracking': + {'regex': re.compile(r'/\_tracking'), 'total': 0}, + 'csw': + {'regex': re.compile(r'/csw(-all)?\?[A-Za-z_&=]+'), 'total': 0}, 'home': {'regex': re.compile(r'(\/[A-Za-z_]+)?\/$'), 'total': 0}, } + status_code = {} + c = 0 for line in f: c += 1 @@ -91,6 +99,11 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0): continue out.write(data['request_path'] + "\n") + if data['status_code'] not in status_code: + status_code[data['status_code']] = 1 + else: + status_code[data['status_code']] += 1 + found = False for name in weigths: print(name) @@ -110,6 +123,7 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0): nf.close() print([{k: weigths[k]['total']} for k in weigths]) + print(status_code) parser = argparse.ArgumentParser()