Update script to better capture page data, capture video, etc

browserup · Sep 11, 2023 · 93ac045 · 93ac045
1 parent 9f7b402
commit 93ac045
Show file tree

Hide file tree

Showing 309 changed files with 17,009 additions and 3,220 deletions.
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -1,5 +1,9 @@
 # Contributing
 
+Please contribute upstream--we follow mitmproxy, but they 
+are the source of truth for the proxy in general.
+
+----------------------------------
 As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project
 forward, please consider contributing in the following areas:
 

diff --git a/README.md b/README.md
@@ -33,7 +33,9 @@ code contributions where possible.
 The main changes for our fork are in the folders below/
 
 mitmproxy/addons/browserup/
+
 test/mitmproxy/addons/browserup
+
 clients/
 
 The standard MITMProxy documentation follows below.

diff --git a/UpgradingMitmProxy.MD b/UpgradingMitmProxy.MD
@@ -0,0 +1,34 @@
+The following instructions are used for updating to a new
+version of mitmproxy.
+
+-----------------------------------------
+
+Fix websocket connection missing issue  for our internal websocket:
+
+mitmproxy/addons/browserup/test/mitmproxy/addons/browserupclients/
+
+mitmproxy/proxy/layers/http/__init__.py
+
+
+elif isinstance(command, SendHttp):    if command.connection in self.connections:        conn = self.connections[command.connection]        yield from self.event_to_child(conn, command.event)    else:        self.debug(f"No connection found for key: {command.connection}")
+
+
+Copy:
+
+mitmproxy/addons/browserup/
+test/mitmproxy/addons/browserup
+clients/
+
+Adjust setup.py to have the browserup executable entrypoint:
+setup.py
+
+
+entry_points={    "console_scripts": [        "mitmproxy = mitmproxy.tools.main:mitmproxy",        "mitmdump = mitmproxy.tools.main:mitmdump",        "mitmweb = mitmproxy.tools.main:mitmweb",        "browserup-proxy = mitmproxy.tools.main:browserupproxy",    ],    "pyinstaller40": [        "hook-dirs = mitmproxy.utils.pyinstaller:hook_dirs",    ]},
+
+Add scripts folder:
+
+scripts/browsertime/browser-data.js
+
+Add (falcon, etc) to:
+
+setup.py
diff --git a/browserup-proxy.schema.json b/browserup-proxy.schema.json
@@ -66,13 +66,7 @@
             }
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "head": {},
-      "patch": {},
-      "post": {},
-      "trace": {}
+      }
     },
     "/har/page": {
       "post": {
@@ -105,14 +99,7 @@
             }
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/verify/present/{name}": {
       "post": {
@@ -159,14 +146,7 @@
             "description": "The MatchCriteria are invalid."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/verify/not_present/{name}": {
       "post": {
@@ -213,14 +193,7 @@
             "description": "The MatchCriteria are invalid."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/verify/size/{size}/{name}": {
       "post": {
@@ -277,14 +250,7 @@
             "description": "The MatchCriteria are invalid."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/verify/sla/{time}/{name}": {
       "post": {
@@ -341,14 +307,7 @@
             "description": "The MatchCriteria are invalid."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/har/errors": {
       "post": {
@@ -376,14 +335,7 @@
             "description": "The Error was invalid."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/har/counters": {
       "post": {
@@ -411,14 +363,7 @@
             "description": "The counter was invalid."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "get": {},
-      "head": {},
-      "patch": {},
-      "put": {},
-      "trace": {}
+      }
     },
     "/healthcheck": {
       "get": {
@@ -432,14 +377,7 @@
             "description": "OK means all is well."
           }
         }
-      },
-      "options": {},
-      "delete": {},
-      "head": {},
-      "patch": {},
-      "post": {},
-      "put": {},
-      "trace": {}
+      }
     }
   },
   "openapi": "3.0.3",
@@ -448,53 +386,53 @@
       "PageTiming": {
         "type": "object",
         "properties": {
-          "_firstPaint": {
-            "type": "number",
-            "description": "firstPaint from the browser"
-          },
-          "_largestContentfulPaint": {
-            "type": "number",
-            "description": "largestContentfulPaint from the browser"
-          },
-          "_timeToFirstByte": {
+          "onContentLoad": {
             "type": "number",
-            "description": "Time to first byte of the page's first request per the browser"
+            "description": "onContentLoad per the browser"
           },
           "onLoad": {
             "type": "number",
             "description": "onLoad per the browser"
           },
-          "onContentLoad": {
+          "_firstInputDelay": {
             "type": "number",
-            "description": "onContentLoad per the browser"
+            "description": "firstInputDelay from the browser"
           },
-          "_firstContentfulPaint": {
+          "_firstPaint": {
             "type": "number",
-            "description": "firstContentfulPaint from the browser"
+            "description": "firstPaint from the browser"
           },
-          "_firstInputDelay": {
+          "_cumulativeLayoutShift": {
             "type": "number",
-            "description": "firstInputDelay from the browser"
+            "description": "cumulativeLayoutShift metric from the browser"
+          },
+          "_largestContentfulPaint": {
+            "type": "number",
+            "description": "largestContentfulPaint from the browser"
           },
           "_domInteractive": {
             "type": "number",
             "description": "domInteractive from the browser"
           },
-          "_href": {
-            "type": "string",
-            "description": "Top level href, including hashtag, etc per the browser"
-          },
-          "_ssl": {
+          "_firstContentfulPaint": {
             "type": "number",
-            "description": "Ssl connect time from the browser"
+            "description": "firstContentfulPaint from the browser"
           },
           "_dns": {
             "type": "number",
             "description": "dns lookup time from the browser"
           },
-          "_cumulativeLayoutShift": {
+          "_ssl": {
             "type": "number",
-            "description": "cumulativeLayoutShift metric from the browser"
+            "description": "Ssl connect time from the browser"
+          },
+          "_timeToFirstByte": {
+            "type": "number",
+            "description": "Time to first byte of the page's first request per the browser"
+          },
+          "_href": {
+            "type": "string",
+            "description": "Top level href, including hashtag, etc per the browser"
           }
         }
       },
@@ -633,13 +571,13 @@
             "type": "boolean",
             "description": "Result True / False"
           },
-          "type": {
-            "type": "string",
-            "description": "Type"
-          },
           "name": {
             "type": "string",
             "description": "Name"
+          },
+          "type": {
+            "type": "string",
+            "description": "Type"
           }
         }
       },
@@ -847,34 +785,6 @@
           }
         }
       },
-      "Video": {
-        "type": "object",
-        "properties": {
-          "filename": {
-            "type": "string"
-          },
-          "src": {
-            "type": "string"
-          },
-          "bufferedPercent": {
-            "type": "number"
-          },
-          "droppedVideoFrames": {
-            "type": "number"
-          },
-          "totalVideoFrames": {
-            "type": "number"
-          }
-        },
-        "required": [
-          "filename",
-          "src",
-          "bufferedPercent",
-          "droppedVideoFrames",
-          "totalVideoFrames"
-        ],
-        "additionalProperties": false
-      },
       "Har": {
         "type": "object",
         "required": [
@@ -1199,6 +1109,54 @@
                   "encoding": {
                     "type": "string"
                   },
+                  "_videoBufferedPercent": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoStallCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoDecodedByteCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoWaitingCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoErrorCount": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoDroppedFrames": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoTotalFrames": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
+                  "_videoAudioBytesDecoded": {
+                    "type": "integer",
+                    "format": "int64",
+                    "minimum": -1,
+                    "default": -1
+                  },
                   "comment": {
                     "type": "string"
                   }

diff --git a/clients/README.md b/clients/README.md
@@ -31,6 +31,11 @@ different. So see them, follow this pattern:
 `openapi-generator config-help -g python`
 
 
+* cd to clients folder
+* run the tests (to regen the schema file browserup-proxy.schema.json)
+* run the commands below to generate the clients
+
+
 ## Generate REST API-Docs
 `openapi-generator generate -g markdown -i /Users/ebeland/apps/mitmproxy/browserup-proxy.schema.json -o markdown`