Merge pull request #154 from Privado-Inc/python

Python Alpha Release

buildspec.yml

@@ -17,13 +17,20 @@ phases:
             LATEST_TAG="$(git describe --tags --abbrev=0)"
             VERSION=${LATEST_TAG:1}
         else
-            LATEST_TAG=$(curl -i "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/repos/Privado-Inc/privado/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+            LATEST_TAG=$(curl -i "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/repos/Privado-Inc/privado-core/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
             VERSION_INITIAL=${LATEST_TAG:1}
             VERSION=$VERSION_INITIAL-$CODEBUILD_BUILD_NUMBER
         fi
-      - TAG_1=latest
+      - echo $VERSION
+      - echo $BRANCH_NAME
+      - |
+        if [[ $AWS_REGION = "eu-west-1" ]]
+        then
+            TAG_1=latest
+        else
+            TAG_1=${CODEBUILD_WEBHOOK_TRIGGER:7}
+        fi
       - TAG_2=$VERSION
-      - echo $CODEBUILD_WEBHOOK_HEAD_REF
       - TAG_3=${CODEBUILD_WEBHOOK_HEAD_REF##*/}
 
   build:

config/exclusions/python.yaml

@@ -0,0 +1,10 @@
+exclusions:
+  - id: Exclusions.Test
+    name: Exclude test source code
+    patterns:
+      - "(.*/test(s)?.*)|/Test[A-Z]|Test[.]"
+
+  - id: Exclusions.Empty
+    name: Exclude file which cannot be read
+    patterns:
+      - "<empty>|<unknownFullName>"

config/sinkSkipList/python.yaml

@@ -0,0 +1,12 @@
+sinkSkipList:
+  - id: SinkSkipList.ThirdParties
+    name: Skip Third Party Sinks
+    patterns:
+      - "(dict|list|str|tuple|int)\\.__.*"
+      - "(django.*\\.py).*"
+      - "(?i)(builtins|json|re|copy|argparse|datetime|time|decimal|collections|asyncio|subprocess|pytest|logging|traceback|sys|os|oss2|unittest|base64|dateutil|importlib|socket|urllib3|urllib|io|uuid|gzip|math|bson|random)\\.py.*"
+
+  - id: SinkSkipList.BuiltInLib
+    name: Skip built in language libraries
+    patterns:
+      - "(?i)(<empty>|<operator>\\.|<operators>\\.|<unresolvedNamespace.*>.|dict\\.|list\\.|str\\.|tuple\\.).*"

docs/understanding-rules/sinks/overview.md

@@ -12,15 +12,15 @@ sinks:
     domains:
       - aws.amazon.com
     patterns: 
-      - "(i?).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:get|list|head|select).*"
+      - "(?i).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:get|list|head|select).*"
     tags:
 
   - id: Storages.AmazonS3.Write
     name: Amazon S3(Write)
     domains:
       - aws.amazon.com
     patterns: 
-      - "(i?).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:abortMultipartUpload|completeMultipartUpload|copy|create|delete|put|uploadPart).*"
+      - "(?i).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:abortMultipartUpload|completeMultipartUpload|copy|create|delete|put|uploadPart).*"
     tags:
 ```
 

rules/collections/default/python.yaml

@@ -0,0 +1,6 @@
+collections:
+  - id: Collections.FastAPI
+    name: FastAPI framework restendpoint
+    patterns: 
+      - "(?i)fastapi.py.*(post|get|all|delete|put|patch|head|subscribe|unsubscribe).*"
+    tags:

rules/sinks/internal_apis/api/python.yaml

@@ -0,0 +1,6 @@
+sinks:
+  - id: Sinks.API.InternalAPI
+    name: Internal APIs
+    patterns: 
+      - "((http|https|ftp|ssh):\\/\\/){0,1}(((25[0-5]|(2[0-4]|1\\d|[1-9]|)\\d)\\.?\\b){4}|(localhost))(:[0-9]{2,4}){0,1}(\\/([a-z]){0,1}){0,1}.*"
+    tags:

rules/sinks/leakages/logs/python.yaml

@@ -0,0 +1,38 @@
+sinks:
+
+  - id: Leakages.Log.Error
+    name: Log Error
+    patterns:
+      - "(?i).*(?:log|logger).*(?:error|severe|fatal).*"
+    tags:
+
+  - id: Leakages.Log.Warn
+    name: Log Warn
+    patterns:
+      - "(?i).*(?:log|logger).*(warn|warning).*"
+    tags:
+
+  - id: Leakages.Log.Debug
+    name: Log Debug
+    patterns:
+      - "(?i).*(?:log|logger).*(debug|trace).*"
+      - "(?i).*(?:logger).*(log).*"
+    tags:
+
+  - id: Leakages.Log.Info
+    name: Log Info
+    patterns:
+      - "(?i).*(?:log|logger).*(info).*"
+    tags:
+
+  - id: Leakages.Log
+    name: Log
+    patterns:
+      - "(?i).*[.]log"
+    tags:
+
+  - id: Leakages.Log.Console
+    name: Log Console
+    patterns: 
+      - "builtins[.]py:<module>[.]print"
+    tags:

rules/sinks/storages/amazonS3/java.yaml

@@ -8,13 +8,13 @@ sinks:
     domains:
       - aws.amazon.com
     patterns: 
-      - "(i?).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:get|list|head|select).*"
+      - "(?i).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:get|list|head|select).*"
     tags:
 
   - id: Storages.AmazonS3.Write
     name: Amazon S3(Write)
     domains:
       - aws.amazon.com
     patterns: 
-      - "(i?).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:abortMultipartUpload|completeMultipartUpload|copy|create|delete|put|uploadPart).*"
+      - "(?i).*(?:AmazonS3ClientBuilder|S3Client[.]builder|AmazonS3EncryptionClient|software.amazon.awssdk.services.s3).*(?:abortMultipartUpload|completeMultipartUpload|copy|create|delete|put|uploadPart).*"
     tags:

rules/sinks/storages/amazonS3/python.yaml

@@ -0,0 +1,13 @@
+sinks:
+
+  - id: Storages.AmazonS3.Read
+    name: Amazon S3
+    domains:
+      - s3.amazon.com
+    patterns:
+      - "(?i)(aws_cdk.*aws_s3).*"
+      - "(?i)(aws_cdk.*aws_s3_assets).*"
+      - "(?i)(aws_cdk.*aws_s3_notifications).*"
+      - "(?i)(aws_helper|s3transfer|templates_s3|s3utils|s3url|url_for_s3|s3_parse_url|ctodd_python_lib_aws|awss3lib|s3file|mypy_boto3_s3|s3_encryption|mypy_boto3_s3|pip_services3_aws|bits3).py.*"
+      - "(?i)boto.*(?:get|bucket|download.{0,1}file|upload.{0,1}file|download.{0,1}fileobj|presigned.{0,1}url).*"
+    tags:

rules/sinks/storages/arangodb/java.yaml

@@ -8,13 +8,13 @@ sinks:
     domains:
       - arangodb.com
     patterns:
-      - "(i?).*(?:com[.]arangodb[.]Arango|org[.]eclipse[.]jnosql[.]mapping[.]arangodb).*(?:get|list|head|select|view|find|search|match).*"
+      - "(?i).*(?:com[.]arangodb[.]Arango|org[.]eclipse[.]jnosql[.]mapping[.]arangodb).*(?:get|list|head|select|view|find|search|match).*"
     tags:
 
   - id: Storages.ArangoDB.Write
     name: ArangoDB(Write)
     domains:
       - arangodb.com
     patterns:
-      - "(i?).*(?:com[.]arangodb[.]Arango|org[.]eclipse[.]jnosql[.]mapping[.]arangodb).*(?:add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|rename|save|set|update|bulkWrite).*"
+      - "(?i).*(?:com[.]arangodb[.]Arango|org[.]eclipse[.]jnosql[.]mapping[.]arangodb).*(?:add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|rename|save|set|update|bulkWrite).*"
     tags:

rules/sinks/storages/arangodb/python.yaml

@@ -0,0 +1,9 @@
+sinks:
+
+  - id: Storages.ArangoDB.ReadAndWrite
+    name: ArangoDB
+    domains:
+      - arangodb.com
+    patterns: 
+      - "(?i)pyarango.*(?:createdocument|save|query|get|list|head|select|view|find|search|match).*"
+    tags:

rules/sinks/storages/bigtable/java.yaml

@@ -9,5 +9,5 @@ sinks:
       - cloud.google.com
       - google.com
     patterns:
-      - "(i?).*com[.]google[.]cloud[.]bigquery.*"
+      - "(?i).*com[.]google[.]cloud[.]bigquery.*"
     tags:

rules/sinks/storages/bigtable/python.yaml

@@ -0,0 +1,13 @@
+
+# Sink Rules for storage database Google Cloud Bigtable - https://cloud.google.com/bigtable
+
+sinks:
+
+  - id: Storages.CloudBigtable.ReadAndWrite
+    name: Google Cloud Bigtable
+    domains:
+      - cloud.google.com
+      - google.com
+    patterns: 
+      - "(?i)pyarango.*(?:row.{0,1}filters|read.{0,1}row|delete|query|update.{0,1}dataset|insert.{0,1}rows.{0,1}json|load.{0,1}table.{0,1}from.{0,1}uri|query|list.{0,1}dataset).*"
+    tags:

rules/sinks/storages/cassandra/java.yaml

@@ -9,5 +9,5 @@ sinks:
       - cassandra.apache.org
       - apache.org
     patterns:
-      - "(i?).*(?:CassandraConnector|Cluster[.]builder|com[.]datastax[.]oss|com[.]datastax[.]driver[.]core|com[.]noorq[.]casser|org[.]eclipse[.]jnosql[.]mapping[.]cassandra).*"
+      - "(?i).*(?:CassandraConnector|Cluster[.]builder|com[.]datastax[.]oss|com[.]datastax[.]driver[.]core|com[.]noorq[.]casser|org[.]eclipse[.]jnosql[.]mapping[.]cassandra).*"
     tags:

rules/sinks/storages/cassandra/python.yaml

@@ -0,0 +1,13 @@
+
+# Sink Rules for storage database Apache Cassandra - https://cassandra.apache.org/
+
+sinks:
+
+  - id: Storages.ApacheCassandra.ReadAndWrite
+    name: Apache Cassandra
+    domains:
+      - cassandra.apache.org
+      - apache.org
+    patterns:
+      - "(?i).*cassandra.*cluster.*(?:execute|save|query|get|list|head|select|find|search|match|create|insert).*"
+    tags:

rules/sinks/storages/cognito/java.yaml

@@ -9,7 +9,7 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?).*(?:CognitoIdentityProviderClient[.]|CognitoIdentityClient[.]|AWSCognitoIdentityProviderClientBuilder).*(describe|get|list|lookup|equals|subscribe).*"
+      - "(?i).*(?:CognitoIdentityProviderClient[.]|CognitoIdentityClient[.]|AWSCognitoIdentityProviderClientBuilder).*(describe|get|list|lookup|equals|subscribe).*"
     tags:
 
   - id: Storages.AmazonCognito.Write
@@ -18,5 +18,5 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?).*(?:CognitoIdentityProviderClient[.]|CognitoIdentityClient[.]|AWSCognitoIdentityProviderClientBuilder).*(create|delete|set|tag|unlink|link|untag|update|merge|bulkPublish|createIdentityPool|temporaryPassword|).*"
+      - "(?i).*(?:CognitoIdentityProviderClient[.]|CognitoIdentityClient[.]|AWSCognitoIdentityProviderClientBuilder).*(create|delete|set|tag|unlink|link|untag|update|merge|bulkPublish|createIdentityPool|temporaryPassword|).*"
     tags:

rules/sinks/storages/cognito/python.yaml

@@ -0,0 +1,10 @@
+sinks:
+
+  - id: Storages.AmazonCognito.Read
+    name: Amazon Cognito(Read)
+    domains:
+      - aws.amazon.com
+      - amazon.com
+    patterns:
+      - "(?i)boto.*(?:sign.{0,1}up|confirm.{0,1}sign.{0,1}up|resend.{0,1}confirmation.{0,1}code|forgot.{0,1}password|confirm.{0,1}forgot.{0,1}password|initiate.{0,1}auth|admin.{0,1}get.{0,1}user|confirm.{0,1}device|list.{0,1}users).*"
+    tags:

rules/sinks/storages/cosmosdb/java.yaml

@@ -9,7 +9,7 @@ sinks:
       - azure.microsoft.com
       - microsoft.com
     patterns:
-      - "(i?).*(?:com[.]azure[.]cosmos[.]|CosmosClientBuilder|CosmosClient|CosmosDatabase|CosmosContainer|CosmosAsyncContainer|CosmosAsyncClient|CosmosAsyncScripts|CosmosAsyncStoredProcedure).*(?:read|readItem|readMany|readAll|query|get|find|list|select).*"
+      - "(?i).*(?:com[.]azure[.]cosmos[.]|CosmosClientBuilder|CosmosClient|CosmosDatabase|CosmosContainer|CosmosAsyncContainer|CosmosAsyncClient|CosmosAsyncScripts|CosmosAsyncStoredProcedure).*(?:read|readItem|readMany|readAll|query|get|find|list|select).*"
     tags:
 
   - id: Storages.AzureCosmosDb.Write
@@ -18,5 +18,5 @@ sinks:
       - azure.microsoft.com
       - microsoft.com
     patterns:
-      - "(i?).*(?:com[.]azure[.]cosmos[.]|CosmosClientBuilder|CosmosClient|CosmosDatabase|CosmosContainer|CosmosAsyncContainer|CosmosAsyncClient|CosmosAsyncScripts|CosmosAsyncStoredProcedure).*(?:upsert|create|patch|delete|replace|set|update|save).*"
+      - "(?i).*(?:com[.]azure[.]cosmos[.]|CosmosClientBuilder|CosmosClient|CosmosDatabase|CosmosContainer|CosmosAsyncContainer|CosmosAsyncClient|CosmosAsyncScripts|CosmosAsyncStoredProcedure).*(?:upsert|create|patch|delete|replace|set|update|save).*"
     tags:

rules/sinks/storages/cosmosdb/python.yaml

@@ -0,0 +1,13 @@
+
+# Sink Rules for storage database Azure Cosmos DB - https://azure.microsoft.com/en-us/services/cosmos-db/
+
+sinks:
+
+  - id: Storages.AzureCosmosDb
+    name: Azure Cosmos DB
+    domains:
+      - azure.microsoft.com
+      - microsoft.com
+    patterns:
+      - "(?i).*(?:azure.*cosmos).*(?:create.{0,1}item|read.{0,1}item|delete.{0,1}item|query).*"
+    tags:

rules/sinks/storages/couchdb/java.yaml

@@ -9,7 +9,7 @@ sinks:
       - couchdb.apache.org
       - apache.org
     patterns:
-      - "(i?).*(?:com[.]fourspaces[.]couchdb|org[.]ektorp[.].*couchdb|org[.]lightcouch[.]couchdb|org.*jnosql.*couch.*|com[.]couchbase[.]client).*(?:get|list|head|select|view|find|search|match).*"
+      - "(?i).*(?:com[.]fourspaces[.]couchdb|org[.]ektorp[.].*couchdb|org[.]lightcouch[.]couchdb|org.*jnosql.*couch.*|com[.]couchbase[.]client).*(?:get|list|head|select|view|find|search|match).*"
     tags:
 
   - id: Storages.ApacheCouchDb.Write
@@ -18,5 +18,5 @@ sinks:
       - couchdb.apache.org
       - apache.org
     patterns:
-      - "(i?).*(?:com[.]fourspaces[.]couchdb|.*org[.]ektorp[.].*couchdb|.*org[.]lightcouch[.]CouchDb|org.*jnosql.*couch.*|com[.]couchbase[.]client).*(?:add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|rename|save|set|update|bulkWrite).*" 
+      - "(?i).*(?:com[.]fourspaces[.]couchdb|.*org[.]ektorp[.].*couchdb|.*org[.]lightcouch[.]CouchDb|org.*jnosql.*couch.*|com[.]couchbase[.]client).*(?:add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|rename|save|set|update|bulkWrite).*" 
     tags:

rules/sinks/storages/couchdb/python.yaml

@@ -0,0 +1,30 @@
+# Sink Rules for storage database Apache Couch DB - https://couchdb.apache.org/
+
+sinks:
+
+  - id: Storages.ApacheCouchDb.Read
+    name: Apache Couch DB(Read)
+    domains:
+      - couchdb.apache.org
+      - apache.org
+    patterns:
+      - "(?i)(?:couchdb).*(?:find|get|get.{0,1}attachment|list|put.{0,1}attachment).*"
+    tags:
+
+  - id: Storages.ApacheCouchDb.Write
+    name: Apache Couch DB(Write)
+    domains:
+      - couchdb.apache.org
+      - apache.org
+    patterns:
+      - "(?i)(?:couchdb).*(?:create|put.{0,1}attachment|delete|update).*"
+    tags:
+
+  - id: Storages.ApacheCouchDb.ReadAndWrite
+    name: Apache Couch DB
+    domains:
+      - couchdb.apache.org
+      - apache.org
+    patterns:
+      - "(?i)(?:couchdb).*(?:query|save).*"
+    tags:

rules/sinks/storages/documentdb/java.yaml

@@ -9,7 +9,7 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?).*(?:DocumentDBClient|docdb[.]amazonaws[.]com|rds-combined-ca-bundle[.]pem).*(?:(get|list|describe|count|distinct|aggregate|count).*|find|findOne|findMany)"
+      - "(?i).*(?:DocumentDBClient|docdb[.]amazonaws[.]com|rds-combined-ca-bundle[.]pem).*(?:(get|list|describe|count|distinct|aggregate|count).*|find|findOne|findMany)"
     tags:
 
   - id: Storages.AmazonDocumentDB.Write
@@ -18,5 +18,5 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?).*(?:DocumentDBClient|docdb[.]amazonaws[.]com|rds-combined-ca-bundle[.]pem).*(add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|findAnd|findOneAnd|rename|save|set|update|bulkWrite).*"
+      - "(?i).*(?:DocumentDBClient|docdb[.]amazonaws[.]com|rds-combined-ca-bundle[.]pem).*(add|copy|apply|create|delete|modify|remove|reset|restore|insert|drop|findAnd|findOneAnd|rename|save|set|update|bulkWrite).*"
     tags:

rules/sinks/storages/documentdb/python.yaml

@@ -0,0 +1,22 @@
+
+# Sink Rules for storage database Amazon DocumentDB - https://aws.amazon.com/documentdb/
+
+sinks:
+
+  - id: Storages.AmazonDocumentDB.Read
+    name: Amazon DocumentDB(Read)
+    domains:
+      - aws.amazon.com
+      - amazon.com
+    patterns:
+      - "(?i)boto.*(?:find.{0,1}one).*"
+    tags:
+
+  - id: Storages.AmazonDocumentDB.Write
+    name: Amazon DocumentDB(Write)
+    domains:
+      - aws.amazon.com
+      - amazon.com
+    patterns:
+      - "(?i)boto.*(?:insert.{0,1}(?:one|many)).*"
+    tags:

rules/sinks/storages/dynamodb/java.yaml

@@ -9,7 +9,7 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchE|e)xecuteStatement|executeTransaction)[:].*)))"
+      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchE|e)xecuteStatement|executeTransaction)[:].*)))"
     tags:
 
   - id: Storages.AmazonDynamoDB.Read
@@ -18,7 +18,7 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchG|g)etItem|query|scan|transactGetItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchLoad|load|parallelScan|query|queryPage|scan|scanPage|transactionLoad)[:].*))"
+      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchG|g)etItem|query|scan|transactGetItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchLoad|load|parallelScan|query|queryPage|scan|scanPage|transactionLoad)[:].*))"
     tags:
 
   - id: Storages.AmazonDynamoDB.Write
@@ -27,5 +27,5 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(i?)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchWrite|delete|put|update)Item|transactWriteItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchWrite|delete|save|transactionWrite)[:].*))"
+      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchWrite|delete|put|update)Item|transactWriteItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchWrite|delete|save|transactionWrite)[:].*))"
     tags: