Add files via upload

MalwareBazaar_AMP_Transfer.py

@@ -0,0 +1,300 @@
+import os
+import pandas as pd
+import requests
+from requests.auth import HTTPBasicAuth
+
+# Make df more reader friendly in 'Run' window
+pd.set_option('display.max_rows', None)
+pd.set_option('display.max_columns', None)
+pd.set_option('expand_frame_repr', False)
+
+# Add MalwareBazaar API key below *** Modify these lines ***
+api_key = ''
+
+# Add AMP API Username & Password below *** Modify these lines ***
+amp_api_username = ''
+amp_api_pw = ''
+
+# MalwareBazaar API URL
+mb_url = 'https://mb-api.abuse.ch/api/v1'
+
+# MalwareBazaar output directory
+sha_output_files = os.path.join(os.environ['USERPROFILE'], 'Desktop',
+                                'Python_Manipulated_Files', 'MalwareBazaar')
+
+# All MalwareBazaar hashes in one text file
+main_mb_df_file = os.path.join(os.environ['USERPROFILE'], 'Desktop',
+                               'Python_Manipulated_Files', 'MalwareBazaar', 'AA_Quarantine_&_Prevent_Install.txt')
+
+# All AMP hashes in one text file
+amp_main_file = os.path.join(os.environ['USERPROFILE'], 'Desktop',
+                             'Python_Manipulated_Files', 'MalwareBazaar_Comparison_File', 'comparison.txt')
+
+# URL for dominant Simple Custom Detection List After file_lists add your company's GUID *** Modify this line***
+main_simple_custom_detections_url = 'https://api.amp.cisco.com/v1/file_lists/add_your_guid_here/files'
+
+# URL for all Simple Custom Detections Lists
+all_simple_custom_detections_url = 'https://api.amp.cisco.com/v1/file_lists/simple_custom_detections'
+
+tags = [
+    'AgentTesla',
+    'AZORult',
+    'Babuk',
+    'BatLoader',
+    'BlackBasta',
+    'BlackCat',
+    'BruteRatel',
+    'BumbleBee',
+    'CobaltStrike',
+    'Emotet',
+    'FormBook',
+    'Hive',
+    'IceID',
+    'Lockbit',
+    'Lokibot',
+    'Luna',
+    'Magniber',
+    'Metasploit',
+    'Meterpreter',
+    'Mimikatz',
+    'Nighthawk',
+    'NJRat',
+    'Qbot',
+    'Raccoon',
+    'RedLine',
+    'Remcos',
+    'Sliver',
+    'SOCGholish',
+    'Trickbot',
+    'Ursnif',
+    'Venus',
+    'WannaCry']
+
+
+def pull_from_malware_bazaar_api():
+    # Loop through MalwareBazaar tags and normalize data
+    for tag in tags:
+        try:
+            headers = {'API-KEY': api_key}
+            data = {'query': 'get_taginfo',
+                    'tag': tag,
+                    'limit': '1000'}
+
+            response = requests.post(mb_url, headers=headers, data=data)
+            data = response.json()['data']
+
+        except Exception as e:
+            print(f'Failed to pull data from Malware Bazaar API for {tag}')
+            print(f'Exception is {e} for {tag}')
+
+        try:
+            # Create dataframe after normalizing
+            df = pd.json_normalize(data)
+
+            # If empty ignore it
+            if df.shape[0] == 1:
+                print(f'No matches found for {tag}!')
+
+            # If not empty
+            else:
+                # Keep only SHA-256 hashes
+                df = df['sha256_hash']
+
+                # Create name of the SHA-256 output file on the fly
+                sha_output_file = os.path.join(os.environ['USERPROFILE'], 'Desktop',
+                                               'Python_Manipulated_Files', 'MalwareBazaar', tag + '.txt')
+                # Create txt file on the fly
+                df.to_csv(sha_output_file, index=False, header=0)
+                print(f'Successfully sent text file to folder. Total count is {df.shape[0]} for {tag}.')
+
+        except Exception as e:
+            print(f'Failed to send text file to folder for {tag}')
+            print(f'Exception is {e} for {tag}')
+            continue
+
+
+def create_master_df_from_malware_bazaar():
+    # Delete main_df_file
+    try:
+        os.remove(main_mb_df_file)
+        print(f'Successfully removed main_df_file!')
+
+    except Exception as e:
+        print(f'main_df_file not found! Exception is {e}.')
+
+    # Create main df
+    main_df = pd.DataFrame()
+
+    # Start loop to aggregate MalwareBazaar hashes
+    for filename in os.listdir(sha_output_files):
+        f = os.path.join(sha_output_files, filename)
+        # Check if file
+        if os.path.isfile(f):
+            # This file should not exist yet. Thus ignore it if it does
+            if filename != 'AA_Quarantine_&_Prevent_Install.txt':
+                df = pd.read_csv(f, header=None)
+
+                # Adding how many rows
+                print(f'{filename} is adding another {df.shape[0]} rows to main df.')
+
+                # Append df to main_df
+                main_df = main_df.append(df)
+
+                # Delete df
+                del df
+                print(f'{main_df.shape[0]} after appending {filename}.')
+
+    # Send aggregated dataframe to directory
+    main_df.to_csv(main_mb_df_file, index=False, header=0)
+    print(f'------------------------------------------------------------------')
+    print(f'Successfully created AA_Quarantine_&_Prevent_Install in directory!')
+
+
+def pull_main_hash_list():
+    # Use the request library to pull data. Pass in id and key or you will receive 401 error
+    request = requests.get(main_simple_custom_detections_url,
+                           auth=HTTPBasicAuth(username=amp_api_username, password=amp_api_pw))
+
+    # Pull data in json format. Slice data based off 'data' element
+    json_response = request.json()
+    data = request.json()['data']['items']
+
+    # Create dataframe
+    df = pd.json_normalize(data)
+    df = df['sha256']
+
+    # Build base dataframe before breaking out data in items column
+    while 'next' in json_response['metadata']['links']:
+        next_url = json_response['metadata']['links']['next']
+        request = requests.get(next_url, auth=HTTPBasicAuth(username=amp_api_username, password=amp_api_pw))
+        json_response = request.json()
+        data = request.json()['data']['items']
+        df_append = pd.json_normalize(data)
+        df_append = df_append['sha256']
+        df = df.append(df_append, ignore_index=True)
+        print(f'The shape of the SHA_256 raw df is {df.shape[0]}')
+
+    # Send amp df to txt file
+    df.to_csv(amp_main_file, index=False, header=False)
+    del df
+
+
+def compare_differences():
+    # Create AMP main df from all AMP hashes
+    main_df = pd.read_csv(amp_main_file, header=None, encoding='utf-8')
+
+    # Add column name to make coding easier
+    main_df.columns = ['SHA']
+
+    # Create AMP set off main df
+    amp_sha_set = set(main_df['SHA'].unique())
+
+    # Loop through directory to start pruning unique hashes
+    for filename in os.listdir(sha_output_files):
+        f = os.path.join(sha_output_files, filename)
+        # Check if file
+        if os.path.isfile(f):
+
+            # Create df per file
+            df = pd.read_csv(f, header=None)
+
+            # Add column name to make coding easier
+            df.columns = ['SHA']
+
+            # Create Malware Bazaar Set
+            malware_bazaar_set = set(df['SHA'].unique())
+
+            # Find IPs missing from AMP by using set logic to compare MalwareBazaar with AMP.
+            missing_sha_from_amp = malware_bazaar_set - amp_sha_set
+
+            # Convert set to list
+            missing_sha_list = list(missing_sha_from_amp)
+            missing_sha_df = pd.DataFrame(missing_sha_list)
+            missing_sha_df.rename(columns={0: 'SHA'}, inplace=True)
+
+            # If set is not empty
+            if missing_sha_from_amp:
+                missing_sha_df.to_csv(f, index=False, header=False)
+                print(f'Missing SHA_256 files are: {missing_sha_from_amp}')
+
+            else:
+                # If set is empty remove file
+                os.remove(f)
+
+
+def insert_amp_sha256():
+    """
+    Imports SHA_256 hashes into their respective buckets in AMP's Simple Custom Detection Lists.
+    """
+    # Create requests for all Simple Custom Detection GUIDs for loop
+    request = requests.get(all_simple_custom_detections_url,
+                           auth=HTTPBasicAuth(username=amp_api_username, password=amp_api_pw))
+
+    # Pull data in json format. Slice data based off 'data' element
+    # json_response = request.json()
+
+    data = request.json()['data']
+
+    # Create dataframe
+    amp_df = pd.json_normalize(data)
+    print(amp_df)
+
+    counter = 0
+
+    # Iterate over MalwareBazaar directory
+    for filename in os.listdir(sha_output_files):
+        f = os.path.join(sha_output_files, filename)
+        # Check if file
+        if os.path.isfile(f):
+            # Remove '.txt' so file so files names match in AMP and directory
+            name_of_file = filename.replace('.txt', '')
+            print(name_of_file)
+
+            # Create df from file
+            df = pd.read_csv(f, header=None)
+
+            # Add header
+            df.columns = ['SHA']
+
+            # Query AMP df
+            new_amp_df = amp_df.loc[amp_df['name'] == name_of_file]
+
+            # Reset index to pull the correct GUID
+            new_amp_df = new_amp_df.reset_index()
+
+            # Pull GUID to loop through
+            try:
+                guid = new_amp_df.guid[0]
+                print(f'Guid is {guid}')
+                del new_amp_df
+            except Exception as e:
+                print(f'Exception is {e}. GUID is {guid}')
+
+            # URL to add hashes to
+            temporary_url = f'https://api.amp.cisco.com/v1/file_lists/{guid}/files'
+
+            # This will hit the production environment!
+            for index, row in enumerate(df.itertuples(index=False)):
+
+                # Add hash to Simple Custom Detections
+                temporary_url = f'https://api.amp.cisco.com/v1/file_lists/{guid}/files/{row.SHA}'
+
+                # Send post request
+                requests.post(temporary_url, auth=(amp_api_username, amp_api_pw))
+                print(f'Successfully updated {guid} with hash {row.SHA}')
+
+                counter += 1
+
+            # Delete df
+            del df
+
+    print(f'Total hashes added = {counter}')
+
+
+if __name__ == '__main__':
+    pull_from_malware_bazaar_api()
+    create_master_df_from_malware_bazaar()
+    pull_main_hash_list()
+    compare_differences()
+    insert_amp_sha256()
+