Dev

.github/workflows/comparison-result.yml

@@ -1,15 +1,40 @@
-name: Monitoring Stability and Comparing Results for privado 
+name: Monitoring Stability and Comparing Results for privado
 
-# Triggers when a pull_request or a push action is configured on master branch
+# Triggers when a pull_request is created
 on:
   pull_request_target:
+    branches:
+      - "**"
 
 jobs:
+  start_workflow:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Send message to slack
+        id: initial-message
+        uses: archive/github-actions-slack@master
+        with:
+          slack-optional-parse: full
+          slack-bot-user-oauth-access-token: ${{ secrets.SLACK_TOKEN }}
+          slack-channel: ${{ secrets.SLACK_CHANNEL_ID }}
+          slack-text: "Comparison workflow started for ${{github.event.pull_request.html_url}}"
+
+      - name: Save output to env
+        id: save-output
+        run: echo "INIT_MSG_TS=${{ fromJson(steps.initial-message.outputs.slack-result).response.message.ts }}" >> $GITHUB_OUTPUT
+    outputs:
+      init_message_ts: ${{steps.save-output.outputs.INIT_MSG_TS}}
+
   setup_and_scan:
+    needs: start_workflow
+    strategy:
+      matrix:
+        language: ['java', 'python', 'js', 'ruby-1', 'ruby-2', 'go']
+    continue-on-error: true
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-          
+
       - name: Install JDK-18
         uses: actions/setup-java@v3
         with:
@@ -31,48 +56,91 @@ jobs:
         with:
           repository: Privado-Inc/standalone-monitoring-stability
           path: ./temp/standalone-monitoring-stability
-          ref: main 
-      
+          ref: main
+
       - name: Run the script for ${{github.head_ref}} and ${{github.base_ref}}
-        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./run.py -r ./repos/${{matrix.language}}.txt -rbb ${{github.base_ref}} -rbh ${{github.head_ref}} -brr ${{ github.event.pull_request.base.repo.html_url }} -hrr ${{ github.event.pull_request.head.repo.html_url }} -guf -urc
 
       - name: Run aws-export
-        run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{github.event.number}}
+        run: cd ./temp/standalone-monitoring-stability/ && python3 aws-export.py ${{matrix.language}}-${{github.event.number}}
 
       - name: Move results to a folder
-        run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{github.event.number}}.xlsx ./results/output-${{github.event.number}}.xlsx && mv ./temp/result-${{github.event.number}}.zip ./results/result-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
+        run: cd ./temp/standalone-monitoring-stability/ && mkdir results && mv output-${{matrix.language}}-${{github.event.number}}.xlsx ./results/output-${{matrix.language}}-${{github.event.number}}.xlsx && mv ./temp/result-${{matrix.language}}-${{github.event.number}}.zip ./results/result-${{matrix.language}}-${{github.event.number}}.zip && mv slack_summary.txt ./results/slack_summary.txt
+
+      - name: Zip the results
+        run: cd /home/runner/work/privado/privado/temp/standalone-monitoring-stability && zip result-${{matrix.language}}-${{github.event.number}}.zip -r ./results
+
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+
+      - name: Post results to slack
+        uses: adrey/slack-file-upload-action@master
+        with:
+          thread_ts: ${{needs.start_workflow.outputs.init_message_ts}}
+          channel: ${{ secrets.SLACK_CHANNEL_ID }} # check
+          path: "/home/runner/work/privado/privado/temp/standalone-monitoring-stability/result-${{matrix.language}}-${{github.event.number}}.zip"
+          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\n Language: ${{matrix.language}} \nSummary Report:\n ${{ env.MESSAGE }}"
+          filetype: "zip"
+          token: ${{ secrets.SLACK_TOKEN }}
+
+      - name: Export workflow output
+        run: cd ./temp/standalone-monitoring-stability && python3 ./workflow_check.py /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt
 
-      - name: Upload output and result for next job
+      - name: Set summary variable
+        run: |
+          echo "MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(cat ./temp/standalone-monitoring-stability/action_result.txt)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV 
+
+      - name: Upload summary file
         uses: actions/upload-artifact@master
         with:
-          name: results
-          path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results
+          name: ${{matrix.language}}
+          path: /home/runner/work/privado/privado/temp/standalone-monitoring-stability/results/slack_summary.txt
 
-  send-result:
-    needs: setup_and_scan
-    runs-on: ubuntu-latest
-    steps: 
-      - uses: actions/checkout@v3
+      - name: Workflow report analysis
+        if: ${{ env.MESSAGE != 'true' }}
+        run: exit 1
 
-      - name: Download result folder
+  collate_summary:
+    needs: [start_workflow, setup_and_scan]
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download summary file
         uses: actions/download-artifact@master
         with:
-          name: results
-          path: ./results
-
-      - name: Zip the results
-        run: zip result-${{github.event.number}}.zip -r ./results
+          path: ./language_summary
+
+      - name: Install Python 3.10
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.10'
+
+      - name: Clone standalone-monitoring-stability/flow-test
+        uses: actions/checkout@v3
+        with:
+          repository: Privado-Inc/standalone-monitoring-stability
+          path: ./temp/standalone-monitoring-stability
+          ref: main
+
+
+      - name: Collate summary
+        run: cd ./temp/standalone-monitoring-stability && pip install -r requirements.txt && python3 ./collate_summary.py -s /home/runner/work/privado/privado/language_summary
 
       - name: Set summary variable
         run: |
           echo "MESSAGE<<EOF" >> $GITHUB_ENV
-          echo "$(cat ./results/slack_summary.txt)" >> $GITHUB_ENV
+          echo "$(cat /home/runner/work/privado/privado/temp/standalone-monitoring-stability/global_summary.txt)" >> $GITHUB_ENV
           echo "EOF" >> $GITHUB_ENV
-      - name: Post results to slack
-        uses: MeilCli/slack-upload-file@v3
+
+      - name: Send summary to slack
+        uses: slackapi/slack-github-action@v1.24.0
         with:
-          slack_token: ${{ secrets.SLACK_TOKEN }}
-          channel_id: ${{ secrets.SLACK_CHANNEL_ID }}
-          file_path: "/home/runner/work/privado/privado/result-${{github.event.number}}.zip"
-          initial_comment: "Comparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}} \nSummary Report:\n ${{ env.MESSAGE }}"
-          file_type: "zip"
+          update-ts: ${{needs.start_workflow.outputs.init_message_ts}}
+          channel-id: ${{ secrets.SLACK_CHANNEL_ID }}
+          slack-message: "\nComparison Results generated on ${{github.event.repository.name}} by PR ${{github.event.number}} from branch ${{github.head_ref}} to ${{github.base_ref}} \nPR link https://github.com/Privado-Inc/privado/pull/${{github.event.number}}\nLanguage: All \nSummary Report:\n ${{ env.MESSAGE }}"
+        env:
+          SLACK_BOT_TOKEN: ${{ secrets.SLACK_TOKEN }}

.gitignore

@@ -246,3 +246,6 @@ dist
 # files
 privado
 notes.md
+
+#Directory created by IDE
+workspace

config/exclusions/go.yaml

@@ -0,0 +1,5 @@
+exclusions:
+  - id: Exclusions.Test
+    name: Exclude test source code
+    patterns:
+      - '.*_test\(s\)?.*'

config/exclusions/java.yaml

@@ -12,4 +12,4 @@ exclusions:
   - id: Exclusions.Empty
     name: Exclude file which cannot be read
     patterns:
-      - "<empty>"
+      - "<empty>"

config/exclusions/kotlin.yaml

@@ -0,0 +1,5 @@
+exclusions:
+  - id: Exclusions.Template
+    name: Exclude template file
+    patterns:
+      - "(?i)(.*template.kt|.*template(s)?/.*)"

config/systemConfig/default.yaml

@@ -0,0 +1,3 @@
+systemConfig:
+  - key: maxSocketCount
+    value: "4096"

config/systemConfig/go.yaml

@@ -0,0 +1,9 @@
+systemConfig:
+  - key: apiHttpLibraries
+    value: ^(?i)(net/http|github.com/parnurzeal/gorequest|(gopkg.in|github.com/go-resty)/resty|valyala/fasthttp|github.com/gojektech/heimdall/v\\d/httpclient|github.com/levigross/grequests|github.com/PuerkitoBio/rehttp|github.com/machinebox/graphql).*
+
+  - key: apiSinks
+    value: (?i)(?:url|client|open|request|execute|newCall|load|host|access|list|set|put|post|proceed|trace|patch|Path|send|remove|delete|write|read|postForEntity|call|createCall|createEndpoint|dispatch|invoke|getInput|getOutput|getResponse|do)
+
+  - key: apiIdentifier
+    value: (?i).*((hook|base|auth|prov|endp|install|request|service|gateway|route|resource)(.){0,12}url|(slack|web)(.){0,4}hook|(rest|api|request|service)(.){0,4}(endpoint|gateway|route)).*

config/systemConfig/kotlin.yaml

@@ -0,0 +1,12 @@
+systemConfig:
+  - key: apiHttpLibraries
+    value: ^(?i)(org.apache.http|okhttp|org.glassfish.jersey|com.mashape.unirest|java.net.http|java.net.URL|org.springframework.(web|core.io)|groovyx.net.http|org.asynchttpclient|kong.unirest.java|org.concordion.cubano.driver.http|javax.net.ssl|javax.xml.soap|org.apache.axis2|com.sun.xml.messaging.saaj|org.springframework.ws.client|com.eviware.soapui|org.apache.cxf|org.jboss.ws|com.ibm.websphere.sca.extensions.soap|com.sun.xml.ws|org.apache.camel.component.cxf|org.codehaus.xfire|org.apache.synapse|org.apache.wink.client|com.oracle.webservices.internal.api.databinding.Databinding|com.sap.engine.interfaces.webservices.runtime.client).*
+
+  - key: ignoredSinks
+    value: (?i).*(?<=map|list|jsonobject|json|array|arrays|jsonnode|objectmapper|objectnode).*(put:|get:).*
+
+  - key: apiSinks
+    value: (?i)(?:url|client|openConnection|request|execute|newCall|load|host|access|fetch|get|getInputStream|getApod|getForObject|getForEntity|list|set|put|post|proceed|trace|patch|Path|send|sendAsync|remove|delete|write|read|assignment|provider|exchange|postForEntity|call|createCall|createEndpoint|dispatch|invoke|newMessage|getInput|getOutput|getResponse|marshall|unmarshall|send|asyncSend)
+
+  - key: apiIdentifier
+    value: (?i).*((hook|base|auth|prov|endp|install|request|service|gateway|route|resource)(.){0,12}url|(slack|web)(.){0,4}hook|(rest|api|request|service)(.){0,4}(endpoint|gateway|route)).*

config/systemConfig/ruby.yaml

@@ -0,0 +1,12 @@
+systemConfig:
+  - key: apiHttpLibraries
+    value: (?i)(multipart|faraday|rest-client|httparty|http.client|net.http|curb|sawyer|unirest|excon|typhoeus|.*(Http(.){0,2}Client|RestClient|HTTParty|Faraday|Unirest)).*
+
+  - key: ignoredSinks
+    value: (?i).*(?<=map|list|jsonobject|json|array|arrays|jsonnode|objectmapper|objectnode).*(put:|get:).*
+
+  - key: apiSinks
+    value: (?i)(?:new|url|client|openConnection|request|execute|newCall|load|host|access|usequery|fetch|get|getInputStream|getApod|getForObject|getForEntity|list|set|put|post|proceed|trace|patch|Path|send|sendAsync|remove|delete|write|read|assignment|provider|exchange|postForEntity|call|createCall|createEndpoint|dispatch|invoke|newMessage|getInput|getOutput|getResponse|marshall|unmarshall|send|asyncSend|emit)
+
+  - key: apiIdentifier
+    value: (?i).*((hook|base|auth|prov|endp|install|cloud|host|request|service|gateway|route|resource|upload|api|worker)(.){0,12}url|(slack|web)(.){0,4}hook|(sentry|segment)(.){0,1}(dsn)|(rest|api|host|cloud|request|service)(.){0,4}(endpoint|gateway|route)).*

rules/collections/android/any.yaml

@@ -0,0 +1,63 @@
+collections:
+  - id: Collections.Android.Form.Email
+    name: Android Form Email
+    patterns:
+      - ".*(?i)email.*"
+    tags:
+      sourceId: Data.Sensitive.ContactData.EmailAddress
+
+  - id: Collections.Android.Form.User
+    name: Android Form User Account
+    patterns:
+      - ".*(?i)(user|login).*"
+    tags:
+      sourceId: Data.Sensitive.AccountData.AccountID
+
+  - id: Collections.Android.Form.OrderDetails
+    name: Android Form Order Details
+    patterns:
+      - "(?i).*((order|shipping|billing|invoice)(subscription|charge)?[^\\s/(;)#|,=!>]{0,5}(number|code|num|no|id))"
+    tags:
+      sourceId: Data.Sensitive.PurchaseData.OrderDetails
+
+  - id: Collections.Android.Form.FirstName
+    name: Android Form Personal Characterstics
+    patterns:
+      - "(?i).*((?:first|given)[^\\s/(;)#|,=!>]{0,5}|full)[_]?name"
+    tags:
+      sourceId: Data.Sensitive.PersonalIdentification.FirstName
+
+  - id: Collections.Android.Form.LastName
+    name: Android Form Personal Characterstics
+    patterns:
+      - "(?i).*((?:last|sur(?!geon))[^\\s/(;)#|,=!>]{0,5}name)"
+    tags:
+      sourceId: Data.Sensitive.PersonalIdentification.LastName
+
+  - id: Collections.Android.Form.Address
+    name: Android Form Address
+    patterns:
+      - ".*(?i)address.*"
+    tags:
+      sourceId: Data.Sensitive.ContactData.Address
+
+  - id: Collections.Android.Form.PhoneNumber
+    name: Android Form Phone Number
+    patterns:
+      - ".*(?i)phone.*"
+    tags:
+      sourceId: Data.Sensitive.ContactData.PhoneNumber
+
+  - id: Collections.Android.Form.ZipCode
+    name: Android Form Zip Code
+    patterns:
+      - ".*(?i)zip.*"
+    tags:
+      sourceId: Data.Sensitive.ContactData.Address
+
+  - id: Collections.Android.Form.Password
+    name: Android Form Password
+    patterns:
+      - ".*(?i)password.*"
+    tags:
+      sourceId: Data.Sensitive.AccountData.AccountPassword

rules/collections/annotations/java.yaml

@@ -2,7 +2,7 @@ collections:
   - id: Collections.Annotation.Spring
     name: Spring Web Interface Annotation
     patterns: 
-      - "RequestMapping|PostMapping|PutMapping|GetMapping|DeleteMapping"
+      - "RequestMapping|PostMapping|PutMapping|PatchMapping|GetMapping|DeleteMapping"
     tags:
 
   - id: Collections.Annotation.Struts

rules/collections/default/javascript.yaml

@@ -2,5 +2,5 @@ collections:
   - id: Collections.Express
     name: Express framework restendpoint
     patterns: 
-      - "express.(post|get|all|delete|put|patch|head|subscribe|unsubscribe)"
+      - "(?:express|fetch|@feathersjs/feathers|fastify|restify|@nestjs/cli|itty-router|koa-router|@ioc[:]Adonis|@adonisjs|@sails|sails|.*loopback|.*(?:socket[.](io|on|to).*)|(?:io[.]on.*(connection|leave-room|join-room))).*"
     tags:

rules/collections/webforms/any.yaml

@@ -2,5 +2,5 @@ collections:
   - id: Collections.Webforms
     name: Webform data collection
     patterns: 
-      - "^<(?i)(?:\\w{0,}(input|upload)\\w{0,}|\\w{0,}(textarea|Text|TextBox|Select|Field|Autocomplete|Checkbox))"
+      - "^<(?i)(?:\\w{0,}(input|upload)\\w{0,}|\\w{0,}(textarea|Text|TextBox|Select|Field|Autocomplete|Checkbox))[^>]*.*"
     tags:

rules/sinks/internal_apis/api/go.yaml

@@ -0,0 +1,6 @@
+sinks:
+  - id: Sinks.API.InternalAPI
+    name: Internal APIs
+    patterns: 
+      - "((http|https|ftp|ssh):\\/\\/){0,1}(((25[0-5]|(2[0-4]|1\\d|[1-9]|)\\d)\\.?\\b){4}|(localhost))(:[0-9]{2,4}){0,1}(\\/([a-z]){0,1}){0,1}.*"
+    tags:

rules/sinks/internal_apis/api/ruby.yaml

@@ -0,0 +1,6 @@
+sinks:
+  - id: Sinks.API.InternalAPI
+    name: Internal APIs
+    patterns: 
+      - "((http|https|ftp|ssh):\\/\\/){0,1}(((25[0-5]|(2[0-4]|1\\d|[1-9]|)\\d)\\.?\\b){4}|(localhost))(:[0-9]{2,4}){0,1}(\\/([a-z]){0,1}){0,1}.*"
+    tags: