Initial commit

Author: diije

.gitignore

@@ -0,0 +1,58 @@
+*.ini
+!*.sample.ini
+*.csv
+!*.sample.csv
+*.txt
+!*.sample.txt
+!requirements.txt
+*.json
+!*.sample.json
+
+
+# General
+.DS_Store
+.AppleDouble
+.LSOverride
+
+# Icon must end with two \r
+Icon
+
+# Thumbnails
+._*
+
+# Files that might appear in the root of a volume
+.DocumentRevisions-V100
+.fseventsd
+.Spotlight-V100
+.TemporaryItems
+.Trashes
+.VolumeIcon.icns
+.com.apple.timemachine.donotpresent
+
+# Directories potentially created on remote AFP share
+.AppleDB
+.AppleDesktop
+Network Trash Folder
+Temporary Items
+.apdisk
+
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# Scrapy stuff:
+.scrapy
+
+# pyenv
+.python-version
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/

README.md

@@ -0,0 +1,12 @@
+# DataForSEO SERP API Python Client
+
+## Setup
+
+## Usage
+
+### 1. Post tasks
+
+### 2. See if tasks are ready
+
+### 3. Get results
+

client.py

@@ -0,0 +1,35 @@
+from http.client import HTTPSConnection
+from base64 import b64encode
+from json import loads
+from json import dumps
+
+class RestClient:
+    domain = "api.dataforseo.com"
+#    domain = "sandbox.dataforseo.com"
+
+    def __init__(self, username, password):
+        self.username = username
+        self.password = password
+
+    def request(self, path, method, data=None):
+        connection = HTTPSConnection(self.domain)
+        try:
+            base64_bytes = b64encode(
+                ("%s:%s" % (self.username, self.password)).encode("ascii")
+                ).decode("ascii")
+            headers = {'Authorization' : 'Basic %s' %  base64_bytes}
+            connection.request(method, path, headers=headers, body=data)
+            response = connection.getresponse()
+            return loads(response.read().decode())
+        finally:
+            connection.close()
+
+    def get(self, path):
+        return self.request(path, 'GET')
+
+    def post(self, path, data):
+        if isinstance(data, str):
+            data_str = data
+        else:
+            data_str = dumps(data)
+        return self.request(path, 'POST', data_str)

config.sample.ini

@@ -0,0 +1,3 @@
+[general]
+user = 
+password = 

requirements.txt

@@ -0,0 +1,25 @@
+astroid==2.2.5
+beautifulsoup4==4.6.0
+bs4==0.0.1
+certifi==2017.11.5
+charade==1.0.3
+chardet==3.0.4
+cmdline-csv2keychain==0.1.3
+idna==2.6
+isort==4.3.21
+JPype1==0.6.2
+lazy-object-proxy==1.4.1
+lxml==4.1.1
+mccabe==0.6.1
+nltk==3.3
+numpy==1.15.2
+pandas==0.23.4
+pylint==2.3.1
+python-dateutil==2.7.3
+pytz==2018.5
+requests==2.18.4
+six==1.11.0
+typed-ast==1.4.0
+urllib3==1.22
+validate-email==1.3
+wrapt==1.11.2

tasks_get.py

@@ -0,0 +1,101 @@
+####
+## DATAFORSEO SERPS API
+##
+## Get available tasks results
+####
+
+import csv
+import configparser
+import argparse
+import time
+import datetime
+from client import RestClient
+
+
+if __name__ == '__main__': 
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', default="config.ini",
+            type=str, help='Global config file (default: "config.ini")')
+    parser.add_argument('--output', default="keyword-results",
+            type=str, help='Output basename (default: "keyword-results")')
+    parser.add_argument('--delay', default=10,
+            type=float, help='Delay in seconds between batches of requests (default: 10)')
+    args = parser.parse_args()
+
+    conf = configparser.ConfigParser()
+    conf.read(args.config)
+    user = conf['general']['user']
+    password = conf['general']['password']
+
+    # Output headers
+    fields=['task_id','status','request','request_type','domain','location_code','language_code','timestamp','results_count','rank_group','rank_absolute','result_type','title','description','url','breadcrumb']
+    # Output name
+    timestr = time.strftime("%Y%m%d-%H%M%S")
+    tag = args.output + "-" + timestr
+    filename = tag + ".csv"
+
+    with open(filename,'w',newline='') as file:
+        writer = csv.DictWriter(file, fieldnames=fields, delimiter=";")
+        writer.writeheader()
+        file.close()
+
+    client = RestClient(user,password)
+
+    # While there are results, request the next batch
+    next_batch = True
+    while next_batch:
+        response = client.get("/v3/serp/google/organic/tasks_ready")
+        if response['status_code'] == 20000:
+            tasks_available = response["tasks"][0]["result_count"]
+            print("{} tasks available".format(tasks_available))
+            if tasks_available < 1:
+                next_batch = False
+            results = []
+            for task in response['tasks']:
+                if (task['result'] and (len(task['result']) > 0)):
+                    for resultTaskInfo in task['result']:
+                        if(resultTaskInfo['endpoint_regular']):
+                            results.append(client.get(resultTaskInfo['endpoint_regular']))                
+
+            for result in results:
+                for task in result["tasks"]:
+                    task_id = task['id']
+                    status = task['status_message']
+                    for kw in task["result"]:
+                        keyword = kw["keyword"]
+                        request_type = kw["type"]
+                        domain = kw["se_domain"]
+                        location_code = kw["location_code"]
+                        language_code = kw["language_code"]
+                        timestamp = kw["datetime"]
+                        results_count = kw["se_results_count"]
+
+                        for item in kw["items"]:
+                            row = dict()
+                            row["task_id"] = task_id
+                            row["status"] = status
+                            row["request"] = keyword
+                            row["request_type"] = request_type
+                            row["domain"] = domain
+                            row["location_code"] = location_code
+                            row["language_code"] = language_code
+                            row["timestamp"] = timestamp
+                            row["results_count"] = results_count
+                            row["result_type"] = item["type"]
+                            row["rank_group"] = item["rank_group"]
+                            row["rank_absolute"] = item["rank_absolute"]
+                            row["title"] = item["title"]
+                            row["description"] = item["description"]
+                            row["url"] = item["url"]
+                            row["breadcrumb"] = item["breadcrumb"]
+
+                            with open(filename,'a',newline='') as file:
+                                writer = csv.DictWriter(file, fieldnames=fields, delimiter=";")
+                                writer.writerow(row)
+                                file.close()
+
+            print("Batch done.")
+            time.sleep(args.delay)
+        else:
+            next_batch = False
+            print("error. Code: %d Message: %s" % (response["status_code"], response["status_message"]))

tasks_post.py

@@ -0,0 +1,117 @@
+####
+## DATAFORSEO SERPS API
+##
+## Post tasks
+####
+
+import csv
+import configparser
+import argparse
+import time
+import datetime
+from client import RestClient
+
+def range_limited_float_type(arg):
+    """ Type function for argparse - a float within some predefined bounds """
+    try:
+        f = int(arg)
+    except ValueError:    
+        raise argparse.ArgumentTypeError("Must be an integer")
+    if f < 1 or f > 100:
+        raise argparse.ArgumentTypeError("Argument must be < " + str(1) + "and > " + str(100))
+    return f
+
+
+if __name__ == '__main__': 
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', default="config.ini",
+            type=str, help='Global config file (default: "config.ini")')
+    parser.add_argument('--input', required=True,
+            type=str, help='List of keywords to request')
+    parser.add_argument('--output', default="keyword-requests",
+            type=str, help='Output basename (default: "keyword-requests")')
+    parser.add_argument('--language_code', default="fr",
+            type=str, help='Language code for requests (default: "fr")')
+    parser.add_argument('--location_code', default=2250,
+            type=int, help='Location code for requests (default: "2250" for France, get other codes on <https://api.dataforseo.com/v3/serp/google/locations>)')
+    parser.add_argument('--nb_results', default=10,
+            type=int, help='Number of results (default: 10)')
+    parser.add_argument('--device', choices=['desktop','mobile'], default="desktop",
+            help='Device type (default:"desktop")')
+    parser.add_argument('--priority', choices=['high','low'], default='low',
+            help='Priority queue (default: "low")')
+    parser.add_argument('--batch', default=100,
+            type=range_limited_float_type, help='Max number of tasks per batch (default: 100)')
+    parser.add_argument('--delay', default=10,
+            type=float, help='Delay in seconds between batches of requests (default: 10)')
+    parser.add_argument('--sep', default=";",
+            type=str, help='CSV file separator (default: ";")')
+    args = parser.parse_args()
+
+    # Read list of requests
+    with open(args.input,'r') as file:
+        kws = list()
+        for line in file.readlines():
+            kws.append(str.strip(line))
+        file.close()
+
+    # Output headers
+    fields=['request','status','id','tag']
+    # Output name
+    timestr = time.strftime("%Y%m%d-%H%M%S")
+    tag = args.output + "-" + timestr
+    filename = tag + ".csv"
+
+    print('Requests are tagged: {}'.format(tag))
+
+    # Set priority queue
+    priority = {0:0,'low':1,'high':2}
+    priority = priority[args.priority]
+
+    conf = configparser.ConfigParser()
+    conf.read(args.config)
+    user = conf['general']['user']
+    password = conf['general']['password']
+
+    # Send requests
+    with open(filename,'w',newline='') as file:
+        writer = csv.DictWriter(file, fieldnames=fields,delimiter=";")
+        writer.writeheader()
+
+        client = RestClient(user, password)
+
+        # We need to send batches of max 100 tasks
+        i = 0
+        j = args.batch
+
+        # Cut the kws list in batches
+        while j < len(kws)+args.batch:  
+            post_data = dict()          
+            for kw in kws[i:j]:
+                post_data[len(post_data)] = dict(
+                    language_code=args.language_code,
+                    location_code=args.location_code,
+                    keyword=kw,
+                    priority=priority,
+                    depth=args.nb_results,
+                    device=args.device,
+                    tag=tag,
+                )
+
+            response = client.post("/v3/serp/google/organic/task_post", post_data)
+            if response["status_code"] == 20000:
+                for task in response["tasks"]:
+                    data = dict()
+                    data["request"] = task["data"]["keyword"]
+                    data["status"] = task["status_message"]
+                    data["id"] = task["id"]
+                    data["tag"] = task["data"]["tag"]
+                    writer.writerow(data)
+                print("Batch {} done.".format(int((i/args.batch) + 1)))
+            else:
+                print("Error. Code: %d Message: %s" % (response["status_code"], response["status_message"]))
+
+            i = j
+            j += args.batch
+            time.sleep(args.delay)
+

tasks_ready.py

@@ -0,0 +1,33 @@
+####
+## DATAFORSEO SERPS API
+##
+## Shows number of tasks ready for download
+####
+
+import csv
+import configparser
+import argparse
+import time
+import datetime
+from client import RestClient
+
+
+if __name__ == '__main__': 
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--config', default="config.ini",
+            type=str, help='Global config file (default: "config.ini")')
+    args = parser.parse_args()
+
+    conf = configparser.ConfigParser()
+    conf.read(args.config)
+    user = conf['general']['user']
+    password = conf['general']['password']
+
+    client = RestClient(user,password)
+
+    response = client.get("/v3/serp/google/organic/tasks_ready")
+    if response["status_code"] == 20000:
+        tasks_available = response["tasks"][0]["result_count"]
+        print("{} tasks available".format(tasks_available))
+    else:
+        print("error. Code: %d Message: %s" % (response["status_code"], response["status_message"]))