Cloud gov enhancements

.gitignore

@@ -1,4 +1,8 @@
 __pycache__
+*.pyc
 *.csv
 *.txt
+ckan.access*.log
+*.tgz
+*.gz
 .vscode/

locust/.gitignore

@@ -1,2 +1,3 @@
 apache.log
 result*
+ckan.access*.log

locust/advanced.py

@@ -142,7 +142,10 @@ def random_organizations(self):
             return None
         results = data.get('result', [])
         # pick just some organizations randmonly
-        random_results = sample(results, 5)
+        if len(results) < 5:
+            random_results = results
+        else:
+            random_results = sample(results, 5)
         for org in random_results:
             url = f'/organization/{org}'
             self.pending_organizations.append(url)
@@ -163,7 +166,10 @@ def all_groups(self):
             return None
         results = data.get('result', [])
         # pick just some organizations randmonly
-        random_results = sample(results, 5)
+        if len(results) < 5:
+            random_results = results
+        else:
+            random_results = sample(results, 5)
         for group in random_results:
             url = f'/group/{group}'
             self.pending_groups.append(url)

locust/base.py

@@ -18,3 +18,7 @@ def orgs(self):
     @task
     def groups(self):
         self.client.get('/group')
+
+    @task
+    def dataset(self):
+        self.client.get('/dataset')

locust/parse_apache_logs.py

@@ -34,7 +34,7 @@ def process_line(data):
         if data['request_path'].startswith(ss):
             return None
 
-    if data['method'] != 'GET':
+    if data['method'] not in ['GET', 'POST']:
         return None
 
     return data
@@ -59,9 +59,9 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0):
         'harvests':
             {'regex': re.compile(r'(\/[A-Za-z_]+)?\/harvest(\/)?$'), 'total': 0},
         'api-pkg-search':
-            {'regex': re.compile(r'/api/(3)?/action/package_search.*'), 'total': 0},
+            {'regex': re.compile(r'/api(/3)?/action/package_search.*'), 'total': 0},
         'api-pkg-show':
-            {'regex': re.compile(r'/api/(3)?/action/package_show.*'), 'total': 0},
+            {'regex': re.compile(r'/api(/3)?/action/package_show.*'), 'total': 0},
         'api-search':
             {'regex': re.compile(r'/api/search.*'), 'total': 0},
         'organization':
@@ -76,10 +76,18 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0):
             {'regex': re.compile(r'(\/[A-Za-z_]+)?\/group\?(.*)'), 'total': 0},
         'groups':
             {'regex': re.compile(r'(\/[A-Za-z_]+)?\/group(\/)?$'), 'total': 0},
+        'status-show':
+            {'regex': re.compile(r'/api/action/status_show'), 'total': 0},
+        'tracking':
+            {'regex': re.compile(r'/\_tracking'), 'total': 0},
+        'csw':
+            {'regex': re.compile(r'/csw(-all)?\?[A-Za-z_&=]+'), 'total': 0},
         'home':
             {'regex': re.compile(r'(\/[A-Za-z_]+)?\/$'), 'total': 0},
     }
 
+    status_code = {}
+
     c = 0
     for line in f:
         c += 1
@@ -91,6 +99,11 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0):
             continue
         out.write(data['request_path'] + "\n")
 
+        if data['status_code'] not in status_code:
+            status_code[data['status_code']] = 1
+        else:
+            status_code[data['status_code']] += 1
+
         found = False
         for name in weigths:
             print(name)
@@ -110,6 +123,7 @@ def parse_file(apache_log_path, output_path='results.txt', limit=0):
     nf.close()
 
     print([{k: weigths[k]['total']} for k in weigths])
+    print(status_code)
 
 
 parser = argparse.ArgumentParser()