add username and password options

Author: presto8

webpage2html.py

@@ -49,7 +49,7 @@ def absurl(index, relpath=None, normpath=None):
             return index
 
 
-def get(index, relpath=None, verbose=True, usecache=True, verify=True, ignore_error=False):
+def get(index, relpath=None, verbose=True, usecache=True, verify=True, ignore_error=False, username=None, password=None):
     global webpage2html_cache
     if index.startswith('http') or (relpath and relpath.startswith('http')):
         full_path = absurl(index, relpath)
@@ -68,8 +68,13 @@ def get(index, relpath=None, verbose=True, usecache=True, verify=True, ignore_er
         headers = {
             'User-Agent': 'Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.2; Win64; x64; Trident/6.0)'
         }
+
+        auth = None
+        if username and password:
+            auth = requests.auth.HTTPBasicAuth(username, password)
+
         try:
-            response = requests.get(full_path, headers=headers, verify=verify)
+            response = requests.get(full_path, headers=headers, verify=verify, auth=auth)
             if verbose:
                 log('[ GET ] %d - %s' % (response.status_code, response.url))
             if not ignore_error and (response.status_code >= 400 or response.status_code < 200):
@@ -202,12 +207,13 @@ def repl(matchobj):
 
 
 def generate(index, verbose=True, comment=True, keep_script=False, prettify=False, full_url=True, verify=True,
-             errorpage=False, **kwargs):
+             errorpage=False, username=None, password=None, **kwargs):
     """
     given a index url such as http://www.google.com, http://custom.domain/index.html
     return generated single html
     """
-    html_doc, extra_data = get(index, verbose=verbose, verify=verify, ignore_error=errorpage)
+    html_doc, extra_data = get(index, verbose=verbose, verify=verify, ignore_error=errorpage,
+                               username=username, password=password)
 
     if extra_data and extra_data.get('url'):
         index = extra_data['url']
@@ -364,6 +370,8 @@ def main():
     parser.add_argument('-s', '--script', action='store_true', help="keep javascript in the generated html")
     parser.add_argument('-k', '--insecure', action='store_true', help="ignore the certificate")
     parser.add_argument('-o', '--output', help="save output to")
+    parser.add_argument('-u', '--username', help="use HTTP basic auth with specified username")
+    parser.add_argument('-p', '--password', help="use HTTP basic auth with specified password")
     parser.add_argument('--errorpage', action='store_true', help="crawl an error page")
     parser.add_argument("url", help="the website to store")
     args = parser.parse_args()