Add Tornado-based web-server. Support REST API methods. Implement task manager.

.dockerignore

@@ -0,0 +1,6 @@
+*
+.*
+!src
+!server.py
+!requirements.txt
+!docker/server/wait-for-it.sh

.gitignore

@@ -1,7 +1,7 @@
 # Custom
 .idea/
 .DS_Store
-results/*
+results/*.json
 
 # Byte-compiled / optimized / DLL files
 __pycache__/

Makefile

@@ -0,0 +1,13 @@
+.PHONY: up up_build test clean
+
+up:
+	docker-compose up -d
+
+up_build:
+	docker-compose up -d --build --force-recreate
+
+tests:
+	python3 -W ignore:ResourceWarning -m unittest discover -v -b
+
+clean:
+	docker-compose down

README.md

@@ -1,27 +1,60 @@
-# osint-framework
-:eyes: All-in-one OSINT/RECON Swiss Knife
+# OSINT Framework
+
+[![Required OS](https://img.shields.io/badge/OS-Linux%20based-blue)](https://en.wikipedia.org/wiki/Linux)
+[![Python3 Version](https://img.shields.io/badge/python-3.7%2B-blue)](https://www.python.org/downloads/)
+[![License](https://img.shields.io/badge/license-GPL--2.0-blue)](/LICENSE)
+[![Code Style](https://img.shields.io/badge/code%20style-black-000000)](https://github.com/psf/black)
+[![Last Commit](https://img.shields.io/github/last-commit/osint-dev-team/osint-framework)](https://github.com/osint-dev-team/osint-framework)
+
+<p align="center">
+    :fork_and_knife: All-in-one OSINT/RECON Swiss Knife
+</p>
+
+<p align="center">
+  <img src="/assets/screenshots/logo.png?raw=true" alt="OSINT Framework Logo" width="50%" height="50%" />
+</p> 
+
+## Screenshots
+
+<div align="center">
+  <img src="/assets/screenshots/cli.png?raw=true" alt="OSINT Framework CLI interface">
+  <p align="center"><i>CLI interface</i></p>
+</div> 
+
 
 ## Installing
 ```bash
-virtualenv -p python3 venv
-    (or python3 -m venv venv)
+virtualenv -p python3 venv (or python3 -m venv venv)
+source venv/bin/activate
 pip3 install -r requirements.txt
 ```
 
-## Running as a framework
-_First of all: provide some arguments in the `main.py` file to collect information based on your data (WIP now, will be improved later)_  
-
-To run the framework:
+## Testing
 ```bash
-python3 main.py example_scenario.yaml
+make tests
 ```
-To run the tests:
+
+## Running
+### As a framework
+To run the framework with a command-line interface:
 ```bash
-chmod +x run_tests.sh
-./run_tests.sh
+python3 cli.py -h
 ```
+### As a REST API web service
+
+<div align="center">
+  <img src="/assets/screenshots/docker.png?raw=true" alt="OSINT Framework Docker usage">
+</div> 
 
-## Running as a separated module
+To run the framework as a web service via docker and docker-compose:  
+```bash
+make up
+```
+or
+```bash
+docker-compose up
+```
+## As a separated module
 Basic:
 ```python3
 python3 -m src.scripts.<category>.<name> any_arguments_here
@@ -39,6 +72,111 @@ Example output:
 
 ```
 
+## REST API web service usage
+1. Create the task:
+```http
+POST /api/tasks/create HTTP/1.1
+Host: localhost:8888
+Content-Type: application/json
+
+[
+    {
+        "case": "base",
+        "name": "testname-profile",
+        "description": "Base example for 'testname' user profile",
+        "kwargs": {
+            "username": "testname",
+            "email": "testmail@gmail.com",
+            "fullname": "Test Name"
+        }
+    },
+    {
+        "case": "osint",
+        "name": "johndoe-profile",
+        "description": "Osint example for 'johndoe' user profile",
+        "kwargs": {
+            "username": "johndoe",
+            "email": "johndoe@gmail.com",
+            "fullname": "John Doe"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "facebook-website",
+        "description": "Recon example for 'facebook.com' website",
+        "kwargs": {
+            "url": "https://facebook.com"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "vk-website",
+        "description": "Recon example for 'vk.com' website",
+        "kwargs": {
+            "url": "https://vk.com"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "mail-website",
+        "description": "Recon example for 'mail.ru' website",
+        "kwargs": {
+            "url": "https://mail.ru"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "8-8-8-8-host",
+        "description": "Recon example for '8.8.8.8' host",
+        "kwargs": {
+            "ip": "8.8.8.8"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "92-63-64-162-host",
+        "description": "Recon example for '92.63.64.162' host",
+        "kwargs": {
+            "ip": "92.63.64.162"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "13-91-95-74-host",
+        "description": "Recon example for '13.91.95.74' host",
+        "kwargs": {
+            "ip": "13.91.95.74"
+        }
+    },
+    {
+        "case": "recon",
+        "name": "87-240-190-78-host",
+        "description": "Recon example for '87.240.190.78' host",
+        "kwargs": {
+            "ip": "87.240.190.78"
+        }
+    },
+    {
+        "case": "osint",
+        "name": "phone-check",
+        "description": "check information about the phone number",
+        "kwargs": {
+            "phone": 89138111111
+        }
+    }
+]
+```
+2. Check tasks status:
+```http
+GET /api/tasks/list HTTP/1.1
+Host: localhost:8888
+```
+3. Get the results when the task is done:
+```http
+GET /api/results?task_id=<YOUR_TASK_ID> HTTP/1.1
+Host: localhost:8888
+```
+
 ## Create your own script
 Use the following structure:  
 1. Create your own module directory in the following way:

cli.py

@@ -0,0 +1,38 @@
+#!/usr/bin/env python3
+
+"""
+Main runner.
+"""
+
+from logging import basicConfig, INFO
+
+from src.cli.handlers.files import FileManager
+from src.cli.interface.arguments import parse_args
+from src.cli.interface.validators import check_arg_length, check_py_version
+from src.cli.interface.opener import show_opener
+from src.core.runner.manager import CaseManager
+from pathlib import Path
+
+
+basicConfig(level=INFO)
+
+
+if __name__ == "__main__":
+    # fmt: off
+
+    show_opener()
+    check_py_version()
+    check_arg_length()
+    scenario = str(parse_args().scenario)
+
+    if scenario.endswith("json"):
+        cases = FileManager.load_json_scenario(scenario)
+    else:
+        cases = FileManager.load_yaml_scenario(scenario)
+
+    manager = CaseManager(cases)
+    results = list(manager.multi_case_runner())
+
+    FileManager.save_results(results, name=Path(scenario).stem)
+
+    # fmt: on

docker-compose.yml

@@ -0,0 +1,51 @@
+version: "3.8"
+
+services:
+  db:
+    container_name: osint-framework-db
+    image: postgres:alpine
+    environment:
+      POSTGRES_HOST_AUTH_METHOD: trust
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
+      POSTGRES_USER: ${POSTGRES_USER:-osint_framework}
+      PGDATA: /data/postgres
+    healthcheck:
+      test: pg_isready -U osint_framework
+      interval: 30s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - postgres:/data/postgres
+    networks:
+      - postgres
+    ports:
+      - "5432:5432"
+  server:
+    container_name: osint-framework-server
+    image: osint-framework-server:1.0
+    depends_on:
+      - db
+    environment:
+      POSTGRES_DATABASE: ${POSTGRES_DATABASE:-osint}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-osint_framework}
+      POSTGRES_USER: ${POSTGRES_USER:-osint_framework}
+      POSTGRES_HOST: ${POSTGRES_HOST:-osint-framework-db}
+      POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+    build:
+      context: .
+      target: osint-framework-server
+      dockerfile: docker/server/Dockerfile
+    healthcheck:
+      test: curl --fail -s http://localhost:8888/api/health || exit 1
+      interval: 30s
+      timeout: 5s
+      retries: 5
+    ports:
+      - "8888:8888"
+    networks:
+      - postgres
+networks:
+  postgres:
+    driver: bridge
+volumes:
+    postgres:

docker/server/Dockerfile

@@ -0,0 +1,17 @@
+FROM python:3.8-alpine as osint-framework-server
+
+LABEL org.label-schema.name="OSINT Framework Server" \
+      org.label-schema.description="OSINT Framework Server" \
+      org.label-schema.license="GPL-2.0"
+
+COPY . /app/
+COPY docker/server/wait-for-it.sh /app/wait-for-it.sh
+WORKDIR /app
+RUN apk add --no-cache --virtual .build_deps build-base libffi-dev gcc musl-dev && \
+    apk add --no-cache postgresql-dev bash curl && \
+    pip install --no-cache-dir -r requirements.txt && \
+    apk del .build_deps && \
+    chmod +x wait-for-it.sh
+
+EXPOSE 8888
+ENTRYPOINT ["./wait-for-it.sh", "-t", "5", "osint-framework-db:5432", "--", "python3", "server.py"]