Merge pull request #179 from Privado-Inc/dev

Release: 24th Feb 2023: Rule segregation

.github/workflows/comparison-result.yml

@@ -0,0 +1,78 @@
+name: Monitoring Stability and Comparing Results for privado 
+
+# Triggers when a pull_request or a push action is configured on master branch
+on:
+  pull_request_target:
+
+jobs:
+  setup_and_scan:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install JDK-18
+        uses: actions/setup-java@v3
+        with:
+          distribution: 'temurin'
+          java-version: '18'
+
+      - name: Export Java Home Path
+        run: export PATH=$JAVA_HOME/bin:$PATH
+
+      - name: Install sbt
+        run: mkdir -p ~/bin && curl -Ls https://raw.githubusercontent.com/dwijnand/sbt-extras/master/sbt > ~/bin/sbt && chmod 0755 ~/bin/sbt
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Clone standalone-monitoring-stability/main
+        uses: actions/checkout@v3
+        with:
+          repository: Privado-Inc/standalone-monitoring-stability
+          path: ./temp/standalone-monitoring-stability
+          ref: main 
+
+      - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}}
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -guf -urc
+
+      - name: Run aws-export
+        run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}}
+
+      - name: Move results to a folder
+        run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
+
+      - name: Upload output and result for next job
+        uses: actions/upload-artifact@master
+        with:
+          name: results
+          path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results
+
+  send-result:
+    needs: setup_and_scan
+    runs-on: ubuntu-latest
+    steps: 
+      - uses: actions/checkout@v3
+
+      - name: Download result folder
+        uses: actions/download-artifact@master
+        with:
+          name: results
+          path: ./results
+
+      - name: Zip the results
+        run: zip result-${{github.event.number}}.zip -r ./results
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Post results to slack
+        uses: MeilCli/slack-upload-file@v3
+        with:
+          slack_token: ${{ secrets.SLACK_TOKEN }}
+          channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
+          file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip"
+          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}"
+          file_type: "zip"

README.md

@@ -88,7 +88,7 @@ Apart from getting a comprehensive outlook of your data practices for Privacy Au
 
 Our free cloud platform can be used to generate RoPA reports for one or more synced repositories.
 
-## Data Safety Report
+## Data Safety Report 
 A Data Safety Report is a privacy form needed to publish any Android app on the Play Store. Most of the time, filling out a report means developers asking around the team to find what data they're collecting, spending hours reading SDK docs to see where information gets shared and navigating the complex Playstore form. With our scan, we pre-fill data types that are collected and shared, and our wizard guides you through generating the report.
 
 # Contribute

buildspec.yml

@@ -17,7 +17,7 @@ phases:
             LATEST_TAG="$(git describe --tags --abbrev=0)"
             VERSION=${LATEST_TAG:1}
         else
-            LATEST_TAG=$(curl -i "Accept: application/vnd.github+json" -H "Authorization: token $GITHUB_TOKEN" https://api.github.com/repos/Privado-Inc/privado-core/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
+            LATEST_TAG=$(curl -i https://api.github.com/repos/Privado-Inc/privado-core/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/')
             VERSION_INITIAL=${LATEST_TAG:1}
             VERSION=$VERSION_INITIAL-$CODEBUILD_BUILD_NUMBER
         fi

rules/sinks/storages/dynamodb/java.yaml

@@ -18,7 +18,7 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchG|g)etItem|query|scan|transactGetItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchLoad|load|parallelScan|query|queryPage|scan|scanPage|transactionLoad)[:].*))"
+      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync|document[.]Table)[.]((batchG|g)etItem|query|scan|transactGetItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchLoad|load|parallelScan|query|queryPage|scan|scanPage|transactionLoad)[:].*))"
     tags:
 
   - id: Storages.AmazonDynamoDB.Write
@@ -27,5 +27,5 @@ sinks:
       - aws.amazon.com
       - amazon.com
     patterns:
-      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync)[.]((batchWrite|delete|put|update)Item|transactWriteItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchWrite|delete|save|transactionWrite)[:].*))"
+      - "(?i)((?:com[.]amazonaws[.]services[.]dynamodbv2[.]((AmazonDynamoDB|AmazonDynamoDBAsyncClient|AmazonDynamoDBClient|AbstractAmazonDynamoDB|AbstractAmazonDynamoDBAsync|document[.]Table)[.]((batchWrite|delete|put|update)Item|transactWriteItems)[:].*))|(?:com[.]amazonaws[.]services[.]dynamodbv2[.]datamodeling[.]DynamoDBMapper[.](batchWrite|delete|save|transactionWrite)[:].*))"
     tags:

rules/sinks/storages/hibernate/java.yaml

@@ -1,15 +1,13 @@
-
-
 # Sink Rules for storage database Hibernate - https://hibernate.org/
 
 sinks:
-
   - id: Storages.Hibernate.Read
     name: Hibernate(Read)
     domains:
       - hibernate.org
-    patterns: 
-      - "(?i)((org[.]hibernate[.]Session[.]((byId|byMultipleIds|byNaturalId|bySimpleNaturalId|get|getEntityName|load|refresh)[:].*))|(org[.]hibernate[.]query[.]Query.*((getResultList|getResultStream|getSingleResult|list||stream|uniqueResult|uniqueResultOptional)[:].*)))"
+    patterns:
+      - "(?i)(org[.]hibernate[.]Session[.])((byId|byMultipleIds|byNaturalId|bySimpleNaturalId|get|getEntityName|load|refresh)[:])(.*)"
+      - "(?i)(org[.]hibernate[.])(query[.])?(Query.*)(getResultList|getResultStream|getSingleResult|list|stream|uniqueResult|uniqueResultOptional)[:](.*)"
 
     tags:
 
@@ -19,5 +17,5 @@ sinks:
       - hibernate.org
     patterns:
       - "(?i)(org[.]hibernate[.]Session[.]((delete|merge|persist|replicate|save|saveOrUpdate|update)[:].*))"
+      - "(?i)(org[.]hibernate[.](query[.])?Query[.]((update|execute(update)?)[:].*))"
     tags:
-

rules/sinks/storages/jdbc/java.yaml

@@ -16,13 +16,13 @@ sinks:
     domains:
       - oracle.com
     patterns:
-      - "(?i)((java[.]sql[.](Statement|PreparedStatement)[.](executeUpdate|executeLargeUpdate).*)|(.*repository.*[.](save.*|delete.*)[:].*)|(org[.]springframework[.]jdbc[.]core[.]JdbcTemplate[.].*update[:].*))" 
+      - "(?i)((java[.]sql[.](Statement|PreparedStatement)[.](executeUpdate|executeLargeUpdate).*)|(.*repository.*[.](save.*|delete.*)[:].*)|(org[.]springframework[.]jdbc[.]core[.](JdbcTemplate|(namedparam[.]){0,1}NamedParameterJdbcTemplate)[.].*update[:].*))" 
     tags:
 
   - id: Storages.SpringFramework.Jdbc.Read
     name: Java Database Connector(Read)
     domains:
       - oracle.com
     patterns:
-      - "(?i)((java[.]sql[.](Statement|PreparedStatement)[.](executeQuery).*)|(.*repository.*[.](count[:]long.*|find.*[:].*))|(org[.]springframework[.]jdbc[.]core[.]JdbcTemplate[.](query|execute).*[:].*))" 
+      - "(?i)((java[.]sql[.](Statement|PreparedStatement)[.](executeQuery).*)|(.*repository.*[.](count[:]long.*|find.*[:].*))|(org[.]springframework[.]jdbc[.]core[.](JdbcTemplate|(namedparam[.]){0,1}NamedParameterJdbcTemplate)[.](query|execute).*[:].*))" 
     tags:

rules/sinks/storages/messagingqueue/java.yaml

@@ -0,0 +1,49 @@
+sinks:
+
+  - id: Messaging.Queue.Kafka.Producer
+    name: Apache Kafka (Producer)
+    domains:
+      - apache.com
+    patterns: 
+      - "(?i).*(org[.]springframework[.]kafka[.]core[.]KafkaTemplate[.](doSend|send|sendAndReceive|convertAndSend|execute)[:]).*"
+    tags:
+
+  - id: Messaging.Queue.AMQP.Rabbit.Producer
+    name: RabbitMq (Producer)
+    domains:
+      - rabbitmq.com
+    patterns: 
+      - "(?i).*(org[.]springframework[.]amqp[.]rabbit[.]core[.]RabbitTemplate[.](doSend|send|sendAndReceive|convertAndSend|execute)[:]).*"
+    tags:
+
+  - id: Messaging.Service.JMS.Producer
+    name: JMS (Producer)
+    domains:
+      - springframework.org
+    patterns: 
+      - "(?i).*(org[.]springframework[.]jms[.]core[.]JmsTemplate[.](doSend|send|sendAndReceive|convertAndSend|execute)[:]|(jakarta[.]jms[.]JMSProducer|javax[.]jms[.]MessageProducer)[.](send|setProperty)[:]).*"
+    tags:
+
+  - id: Messaging.Service.AmazonSQS.Producer
+    name: Amazon SQS (Producer)
+    domains:
+      - amazonaws.com
+    patterns: 
+      - "(?i).*(com[.]amazonaws[.]services[.]sqs[.](model[.]SendMessageRequest[.](set|with)MessageBody[:]|AmazonSQS[.](sendMessage|sendMessageBatch))).*"
+    tags:
+
+  - id: Messaging.Service.AmazonSQS.Consumer
+    name: Amazon SQS (Consumer)
+    domains:
+      - amazonaws.com
+    patterns: 
+      - "(?i).*(com[.]amazonaws[.]services[.]sqs[.]AmazonSQS[.](receiveMessage|receiveMessageBatch|deleteMessage|deleteMessageBatch)).*"
+    tags:
+
+  - id: Messaging.Service.JMS.Consumer
+    name: JMS (Consumer)
+    domains:
+      - springframework.org
+    patterns: 
+      - "(?i).*(org[.]springframework[.]jms[.]core[.]JmsTemplate[.](doReceive|receive)[:]|(jakarta[.]jms[.]JMSConsumer|javax[.]jms[.]MessageConsumer)[.](receive|receiveBody|receiveNoWait|receiveBodyNoWait)[:]).*"
+    tags:

rules/sinks/storages/redis/java.yaml

@@ -0,0 +1,42 @@
+sinks:
+  - id: Storages.Redis.Redisson.Read
+    name: Redisson Redis(Read)
+    domains:
+      - redis.io
+      - redisson.pro
+    patterns:
+      - (?i)(org.redisson.api.RedissonClient[.])(m|p)?(get)(list|map|bucket|range|del|ex)?(:)(.*)
+      - (?i)(org.redisson.api.RedissonClient[.])(get)((pattern)?topic)(:)(.*)
+      - (?i)(org.redisson.api.RedissonClient[.])(get)((binary)?stream)(:)(.*)
+      - (?i)(org.redisson.api.RedissonClient[.])(lcs|strlen|substr)(:)(.*)
+    tags:
+
+  - id: Storages.Redis.Redisson.Write
+    name: Redisson Redis(Write)
+    domains:
+      - redis.io
+      - redisson.pro
+    patterns:
+      - (?i)(org.redisson.api.RedissonClient.getset:)(.*)
+      - (?i)(org.redisson.api.RedissonClient[.])(m|h|p)?(set)(one|user|range|nx|ex)?(:)(.*)
+      - (?i)(org.redisson.api.RedissonClient[.])(incr|decr)(by(float)?)?(:)(.*)
+    tags:
+
+  - id: Storages.Redis.Jedis.Read
+    name: Jedis Redis(Read)
+    domains:
+      - redis.io
+    patterns:
+      - (?i)(redis.clients.jedis[.](providers.ClusterConnectionProvider|connection)[.])(get)(node[s]?|one|raw)?(:)(.*)
+      - (?i)(redis.clients.jedis[.](providers.ClusterConnectionProvider|connection)[.])(get)((objectmulti)?bulk|integer|statuscode)(reply)(:)(.*)
+      - (?i)(redis.clients.jedis.CommandObjects[.])(m|h|p)?(get)(all|del)?(:)(.*)
+    tags:
+
+  - id: Storages.Redis.Jedis.Write
+    name: Jedis Redis(Write)
+    domains:
+      - redis.io
+    patterns:
+      - (?i)(redis.clients.jedis.CommandObjects.getset:)(.*)
+      - (?i)(redis.clients.jedis.CommandObjects[.])(m|h|p)?(set)(one|user|range|nx|ex)?(:)(.*)
+    tags: