Skip to content

Commit a5b0308

Browse files
pritam001pritam001
authored
Merge pull request #22 from algotradeX/nec-api-and-model
NSE apis and model
2 parents 6760867 + 6e5d324 commit a5b0308

19 files changed

+668
-16
lines changed

‎atx-data-processor.postman_collection.json

Lines changed: 162 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -67,17 +67,177 @@
6767
}
6868
},
6969
"response": []
70+
},
71+
{
72+
"name": "post nse/data",
73+
"event": [
74+
{
75+
"listen": "prerequest",
76+
"script": {
77+
"id": "9c74e120-8216-44d4-80d3-b9ef99a6a87a",
78+
"exec": [
79+
""
80+
],
81+
"type": "text/javascript"
82+
}
83+
}
84+
],
85+
"request": {
86+
"method": "POST",
87+
"header": [],
88+
"body": {
89+
"mode": "raw",
90+
"raw": "{\n\t\"symbol\": \"HDFC\",\n\t\"series\": \"EQ\",\n\t\"date\": \"28-Feb-2020\",\n\t\"prev_close\": 2272.2,\n\t\"open\": 2228.55,\n\t\"close\": 2228.55,\n\t\"high\": 2228.55,\n\t\"low\": 2175.75,\n\t\"average\": 2191.99,\n\t\"last\": 2186,\n\t\"deliverable_qty\": 5267783,\n\t\"no_of_trades\": 263271,\n\t\"percent_daily_qty_to_traded_qty\": 68.28,\n\t\"total_traded_qty\": 7715143,\n\t\"turnover\": 16911538106.8\n}",
91+
"options": {
92+
"raw": {
93+
"language": "json"
94+
}
95+
}
96+
},
97+
"url": {
98+
"raw": "{{atx-data-processor-url}}/{{atx-data-processor-api}}/nse/data",
99+
"host": [
100+
"{{atx-data-processor-url}}"
101+
],
102+
"path": [
103+
"{{atx-data-processor-api}}",
104+
"nse",
105+
"data"
106+
]
107+
}
108+
},
109+
"response": []
110+
},
111+
{
112+
"name": "put nse/data",
113+
"event": [
114+
{
115+
"listen": "prerequest",
116+
"script": {
117+
"id": "9c74e120-8216-44d4-80d3-b9ef99a6a87a",
118+
"exec": [
119+
""
120+
],
121+
"type": "text/javascript"
122+
}
123+
}
124+
],
125+
"request": {
126+
"method": "PUT",
127+
"header": [],
128+
"body": {
129+
"mode": "raw",
130+
"raw": "{\n\t\"symbol\": \"HDFC\",\n\t\"series\": \"EQ\",\n\t\"date\": \"28-Feb-2020\",\n\t\"prev_close\": 2272.2,\n\t\"open\": 2228.55,\n\t\"close\": 2228.55,\n\t\"high\": 2228.55,\n\t\"low\": 2175.75,\n\t\"average\": 2191.99,\n\t\"last\": 2186,\n\t\"deliverable_qty\": 5267783,\n\t\"no_of_trades\": 263271,\n\t\"percent_daily_qty_to_traded_qty\": 68.28,\n\t\"total_traded_qty\": 7715143,\n\t\"turnover\": 16911538106.8\n}",
131+
"options": {
132+
"raw": {
133+
"language": "json"
134+
}
135+
}
136+
},
137+
"url": {
138+
"raw": "{{atx-data-processor-url}}/{{atx-data-processor-api}}/nse/data",
139+
"host": [
140+
"{{atx-data-processor-url}}"
141+
],
142+
"path": [
143+
"{{atx-data-processor-api}}",
144+
"nse",
145+
"data"
146+
]
147+
}
148+
},
149+
"response": []
150+
},
151+
{
152+
"name": "delete nse/data",
153+
"event": [
154+
{
155+
"listen": "prerequest",
156+
"script": {
157+
"id": "9c74e120-8216-44d4-80d3-b9ef99a6a87a",
158+
"exec": [
159+
""
160+
],
161+
"type": "text/javascript"
162+
}
163+
}
164+
],
165+
"request": {
166+
"method": "DELETE",
167+
"header": [],
168+
"body": {
169+
"mode": "raw",
170+
"raw": "{\n\t\"date\": \"28-Feb-2020\"\n}",
171+
"options": {
172+
"raw": {
173+
"language": "json"
174+
}
175+
}
176+
},
177+
"url": {
178+
"raw": "{{atx-data-processor-url}}/{{atx-data-processor-api}}/nse/data",
179+
"host": [
180+
"{{atx-data-processor-url}}"
181+
],
182+
"path": [
183+
"{{atx-data-processor-api}}",
184+
"nse",
185+
"data"
186+
]
187+
}
188+
},
189+
"response": []
190+
},
191+
{
192+
"name": "post nse/parse_csv",
193+
"event": [
194+
{
195+
"listen": "prerequest",
196+
"script": {
197+
"id": "9c74e120-8216-44d4-80d3-b9ef99a6a87a",
198+
"exec": [
199+
""
200+
],
201+
"type": "text/javascript"
202+
}
203+
}
204+
],
205+
"request": {
206+
"method": "POST",
207+
"header": [],
208+
"body": {
209+
"mode": "raw",
210+
"raw": "{\n\t\"localCsvUrl\": \"\"\n}",
211+
"options": {
212+
"raw": {
213+
"language": "json"
214+
}
215+
}
216+
},
217+
"url": {
218+
"raw": "{{atx-data-processor-url}}/{{atx-data-processor-api}}/nse/parse_csv",
219+
"host": [
220+
"{{atx-data-processor-url}}"
221+
],
222+
"path": [
223+
"{{atx-data-processor-api}}",
224+
"nse",
225+
"parse_csv"
226+
]
227+
}
228+
},
229+
"response": []
70230
}
71231
],
72232
"variable": [
73233
{
74-
"id": "c7e4cfc9-f938-485e-bef4-e8f7bc76f8e3",
234+
"id": "0bdadc2a-d41f-4719-b8a7-a56457a51edf",
75235
"key": "atx-data-processor-api",
76236
"value": "",
77237
"type": "string"
78238
},
79239
{
80-
"id": "68672d83-073e-4ded-af37-ad59afddfc4f",
240+
"id": "e65d6dda-bc16-48b7-bb78-a1d0bcd18312",
81241
"key": "atx-data-processor-url",
82242
"value": "",
83243
"type": "string"

‎console.sql

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,22 @@ CREATE EXTENSION IF NOT EXISTS timescaledb CASCADE;
55

66
-- We start by creating a regular SQL table
77

8-
CREATE TABLE nec_data (
9-
time TIMESTAMPTZ NOT NULL,
8+
CREATE TABLE nse_data_daily (
9+
timestamp TIMESTAMP PRIMARY KEY NOT NULL,
1010
open DOUBLE PRECISION NOT NULL,
1111
close DOUBLE PRECISION NOT NULL,
1212
high DOUBLE PRECISION NOT NULL,
1313
low DOUBLE PRECISION NOT NULL,
14-
volume DOUBLE PRECISION NOT NULL
14+
volume DOUBLE PRECISION NOT NULL,
15+
symbol VARCHAR(20) NOT NULL,
16+
created_time TIMESTAMP NOT NULL,
17+
updated_time TIMESTAMP NOT NULL
1518
);
1619

1720
-- This creates a hypertable that is partitioned by time
1821
-- using the values in the `time` column.
1922

20-
SELECT create_hypertable('nec_data', 'time');
23+
SELECT create_hypertable('nse_data_daily', 'timestamp');
24+
25+
DROP TABLE "nse_data_daily";
2126

‎requirements.txt

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,9 @@ marshmallow==3.5.0
3232
mccabe==0.6.1
3333
more-itertools==8.2.0
3434
nodeenv==1.3.5
35+
numpy==1.18.2
3536
packaging==20.1
37+
pandas==1.0.3
3638
pathspec==0.7.0
3739
pluggy==0.13.1
3840
pprofile==2.0.2
@@ -47,6 +49,7 @@ python-box==3.4.6
4749
python-dateutil==2.8.1
4850
python-dotenv==0.12.0
4951
python-editor==1.0.4
52+
pytz==2019.3
5053
PyYAML==5.3
5154
redis==3.4.1
5255
regex==2020.2.20

‎src/db/postgres.py

Lines changed: 44 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
import sqlalchemy as alchemist
1+
from sqlalchemy import create_engine
22
from dynaconf import settings
3+
from sqlalchemy.ext.declarative import declarative_base
4+
from sqlalchemy import orm
35

46
from src.common import Singleton, Logger
57

@@ -12,14 +14,51 @@ class Postgres(metaclass=Singleton):
1214
"""
1315

1416
def __init__(self, pg_settings=settings["POSTGRES"]):
15-
self.engine = alchemist.create_engine(
17+
# creates a create_engine instance at the bottom of the file
18+
engine = create_engine(
1619
pg_settings["uri"] + "/" + pg_settings["db"], pool_pre_ping=True
1720
)
18-
self.connection = self.engine.connect()
21+
# constructs a base class for the declarative class definition and assigns it to the base variable
22+
base = declarative_base()
23+
# The last step in our configuration is to add Base.metadata.create_all(engine).
24+
# It will add the classes (we’ll write them in a bit) as new tables in the database we just created.
25+
base.metadata.create_all(engine)
26+
# Bind the engine to the metadata of the Base class so that the
27+
# declaratives can be accessed through a DBSession instance
28+
base.metadata.bind = engine
29+
30+
# The sessionmaker function returns an object for building the particular session you want.
31+
# To understand the options passed to sessionmaker you need to know some terminology:
32+
# flushing is the process of updating the database with the objects you have been working with,
33+
# committing is the process of sending a COMMIT statement to the database to make those flushes permanent.
34+
db_sm = orm.sessionmaker(
35+
bind=engine, autoflush=True, autocommit=False, expire_on_commit=True
36+
)
37+
# bind=engine: this binds the session to the engine, the session will automatically create the connections it needs.
38+
#
39+
# autoflush=True: if you commit your changes to the database before they have been flushed, this option tells
40+
# SQLAlchemy to flush them before the commit is gone.
41+
#
42+
# autocommit=False: this tells SQLAlchemy to wrap all changes between commits in a transaction.
43+
# If autocommit=True is specified, SQLAlchemy automatically commits any changes after each flush; this is undesired in most cases.
44+
#
45+
# expire_on_commit=True: this means that all instances attached to the session will be fully expired after each
46+
# commit so that all attribute/object access subsequent to a completed transaction will load from the most recent database state.
47+
48+
# The scoped_session() object ensures that a different session is used for each thread so that every request
49+
# can have its own access to the database.
50+
session = orm.scoped_session(db_sm)
51+
52+
self.engine = engine
53+
self.base = base
54+
self.session = session
1955
log.info(f"Postgres Instance created")
2056

2157
def get_engine(self):
2258
return self.engine
2359

24-
def get_connection(self):
25-
return self.connection
60+
def get_base(self):
61+
return self.base
62+
63+
def get_session(self):
64+
return self.session

‎src/exception/__init__.py

Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
"""Application error handlers."""
2+
import json
3+
import traceback
4+
5+
from flask import Blueprint, jsonify
6+
from marshmallow import ValidationError
7+
from sqlalchemy import exc
8+
9+
from src.common import Logger
10+
11+
errors = Blueprint("errors", __name__)
12+
log = Logger()
13+
14+
15+
@errors.app_errorhandler(exc.IntegrityError)
16+
@errors.app_errorhandler(exc.OperationalError)
17+
def handle_db_exception(error):
18+
try:
19+
log.error(f"handle_db_exception : {error}")
20+
response = {
21+
"success": False,
22+
"error": {
23+
"type": error.__class__.__name__,
24+
"message": f"psycopg2 error code: {error.orig.pgcode}",
25+
"data": str(error.orig.pgerror),
26+
},
27+
}
28+
status_code = 400
29+
return jsonify(response), status_code
30+
except AttributeError:
31+
handle_error(error)
32+
except Exception:
33+
handle_error(error)
34+
35+
36+
@errors.app_errorhandler(Exception)
37+
def handle_error(error):
38+
if isinstance(error, ValidationError):
39+
response = {
40+
"success": False,
41+
"error": {
42+
"type": error.__class__.__name__,
43+
"message": str(error),
44+
"data": error.messages,
45+
},
46+
}
47+
status_code = 400
48+
else:
49+
message = ""
50+
try:
51+
message = error.description
52+
except Exception:
53+
message = str(error)
54+
55+
data = None
56+
try:
57+
if error.data["messages"] is not None:
58+
data = json.loads(json.dumps(error.data["messages"]))
59+
except Exception:
60+
data = None
61+
62+
try:
63+
status_code = error.code
64+
except Exception:
65+
status_code = 500
66+
67+
response = {
68+
"success": False,
69+
"error": {
70+
"type": error.__class__.__name__,
71+
"message": message,
72+
"data": data,
73+
},
74+
}
75+
76+
print(traceback.print_stack())
77+
print(traceback.print_exc())
78+
79+
return jsonify(response), status_code

‎src/model/__init__.py

Whitespace-only changes.

0 commit comments

Comments
 (0)