Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Bug/memory issue #142

Open
wants to merge 140 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
140 commits
Select commit Hold shift + click to select a range
152392b
Merge pull request #59 from fileverse/dev
nadeem-fileverse Apr 10, 2024
9e21410
Merge pull request #61 from fileverse/dev
nadeem-fileverse Apr 12, 2024
3fa90cb
Merge pull request #62 from fileverse/eng-140
anubhavitis Apr 16, 2024
d6936f6
Merge pull request #64 from fileverse/dev
anubhavitis Apr 18, 2024
7206349
Merge branch 'dev' of github.com:fileverse/fileverse-storage into sto…
anubhavitis Apr 29, 2024
14c5867
Merge pull request #65 from fileverse/storage-sdk-dev
Joshua-onwuzu Apr 29, 2024
b5a4904
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis Apr 29, 2024
6bba423
Merge pull request #66 from fileverse/storage-sdk-dev
Joshua-onwuzu Apr 29, 2024
88d23bb
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis Apr 29, 2024
c753b22
added ucan auth for temp users
Joshua-onwuzu Apr 29, 2024
5fe599f
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis Apr 29, 2024
d33ccd5
Merge branch 'storage-sdk-dev' of github.com:fileverse/fileverse-stor…
anubhavitis Apr 29, 2024
b3ac6be
Merge branch 'dev' of github.com:fileverse/fileverse-storage into sto…
anubhavitis Apr 29, 2024
444ed8a
Merge pull request #67 from fileverse/storage-sdk-dev
anubhavitis Apr 29, 2024
4c49523
allowing all origins manually
anubhavitis Apr 29, 2024
212a256
Merge pull request #68 from fileverse/storage-sdk-dev
anubhavitis Apr 29, 2024
5ea905f
revert: back to same cors syntax
anubhavitis Apr 29, 2024
45e5566
Merge pull request #69 from fileverse/storage-sdk-dev
anubhavitis Apr 29, 2024
03b4ea1
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis Apr 29, 2024
6213609
Merge pull request #70 from fileverse/storage-sdk-dev
anubhavitis Apr 29, 2024
d960624
revert: src/infra/ucan.js to last working checkpoint
anubhavitis Apr 29, 2024
7a16be7
Merge pull request #71 from fileverse/storage-sdk-dev
anubhavitis Apr 29, 2024
1a767bc
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 2, 2024
52ef5ea
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 2, 2024
c398fff
Merge pull request #73 from fileverse/storage-sdk-dev
anubhavitis May 2, 2024
7d42b03
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 2, 2024
2622cc2
Merge branch 'dev' of github.com:fileverse/fileverse-storage into sto…
anubhavitis May 2, 2024
f9799e2
fix: ucan verify function tested and fixed with minor issues
anubhavitis May 2, 2024
9011097
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 2, 2024
6cf7668
Merge pull request #74 from fileverse/storage-sdk-dev
Joshua-onwuzu May 2, 2024
94fa77c
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 2, 2024
19cb1c2
Merge pull request #75 from fileverse/storage-sdk-dev
anubhavitis May 2, 2024
42fb2cc
updated ucan
Joshua-onwuzu May 3, 2024
5af1ff3
Merge pull request #77 from fileverse/minor-storage-fix
Joshua-onwuzu May 3, 2024
1cd6cfb
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 6, 2024
f9f8e21
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 6, 2024
7e4d2e5
Merge pull request #78 from fileverse/storage-sdk-dev
Joshua-onwuzu May 6, 2024
4b53226
slack error reported formatted
anubhavitis May 6, 2024
435e3a9
Merge pull request #79 from fileverse/slack_message
anubhavitis May 6, 2024
2b966eb
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 7, 2024
39df4ff
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 7, 2024
d591843
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 7, 2024
ced8a24
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 7, 2024
65175cb
Merge branch 'dev' of github.com:fileverse/fileverse-storage into sto…
anubhavitis May 7, 2024
6dbc129
Merge branch 'dev' of github.com:fileverse/fileverse-storage into pub…
anubhavitis May 9, 2024
3e82f9c
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 9, 2024
0704cdb
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 9, 2024
4029d63
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 10, 2024
25306e6
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 10, 2024
8b8e018
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 13, 2024
e84d338
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 13, 2024
0f551e8
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 13, 2024
0aadf8c
Merge pull request #81 from fileverse/storage-sdk-dev
anubhavitis May 13, 2024
a744350
Revert "added token validation for namespace & namespace getting save…
anubhavitis May 14, 2024
08325d3
Merge pull request #83 from fileverse/revert-81-storage-sdk-dev
anubhavitis May 14, 2024
3a9276f
Merge branch 'storage-sdk' of github.com:fileverse/fileverse-storage …
anubhavitis May 14, 2024
26754af
Revert "Revert "added token validation for namespace & namespace gett…
anubhavitis May 14, 2024
d216ba4
Merge pull request #84 from fileverse/revert-83-revert-81-storage-sdk…
anubhavitis May 14, 2024
7ff2847
Merge pull request #85 from fileverse/storage-sdk-dev
anubhavitis May 14, 2024
ae1fa64
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 14, 2024
70f2aaa
Merge branch 'dev' of github.com:fileverse/fileverse-storage into pub…
anubhavitis May 14, 2024
0a576c7
Merge pull request #82 from fileverse/public-portal-dev
anubhavitis May 14, 2024
5daab9c
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 14, 2024
644e89a
Merge pull request #86 from fileverse/public-portal-dev
anubhavitis May 14, 2024
48ae83e
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 15, 2024
d0188a4
Merge pull request #87 from fileverse/public-portal-dev
anubhavitis May 15, 2024
b7da1ce
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 16, 2024
df32296
added jobQueue data in /portal/file endpoints
anubhavitis May 16, 2024
8a60c58
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 16, 2024
e0358b2
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 16, 2024
448b549
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 16, 2024
45a68a0
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 16, 2024
3f2823a
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 16, 2024
3169eeb
Merge pull request #88 from fileverse/public-portal-dev
anubhavitis May 16, 2024
fc8665e
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 17, 2024
f1a1b2b
fix: task type set to CLIENT
anubhavitis May 17, 2024
3c91659
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 17, 2024
fd7f98b
Merge pull request #89 from fileverse/public-portal-dev
anubhavitis May 17, 2024
00c10ed
Merge branch 'staging' of github.com:fileverse/fileverse-storage into…
anubhavitis May 20, 2024
20aec87
Merge pull request #90 from fileverse/public-portal-stage
anubhavitis May 20, 2024
eef73de
Merge pull request #80 from fileverse/slack_message
anubhavitis May 20, 2024
08e7865
Merge pull request #92 from fileverse/public-portal
anubhavitis May 28, 2024
70cbea3
Merge pull request #93 from fileverse/public-portal
anubhavitis May 28, 2024
5c650f5
Merge pull request #96 from fileverse/public-portal
anubhavitis May 28, 2024
40e73f2
Merge pull request #97 from fileverse/public-portal
anubhavitis May 28, 2024
0913375
testing Limits presave function
anubhavitis May 30, 2024
d635336
Merge pull request #99 from fileverse/public-portal
anubhavitis May 30, 2024
b48069c
Revert "Public portal"
anubhavitis May 30, 2024
00f2ba1
Merge pull request #100 from fileverse/revert-99-public-portal
anubhavitis May 30, 2024
5f9e203
tested the non-contract flow
anubhavitis May 30, 2024
7c5345a
Merge branch 'public-portal' of github.com:fileverse/fileverse-storag…
anubhavitis May 30, 2024
1dc3c3c
Merge pull request #101 from fileverse/public-portal-dev
anubhavitis May 30, 2024
89be85f
Merge pull request #104 from fileverse/revert_storage_sdk
vijaykrishnavanshi Jun 10, 2024
46fdb43
Revert "Public portal dev"
anubhavitis Jun 11, 2024
74c8653
Merge pull request #106 from fileverse/revert-101-public-portal-dev
anubhavitis Jun 11, 2024
4016dec
Merge pull request #107 from fileverse/staging
anubhavitis Jun 11, 2024
b9b2f2e
new api to count comments made on portal
anubhavitis Jun 11, 2024
6263082
Merge pull request #108 from fileverse/quests
anubhavitis Jun 11, 2024
0573a30
changed to in condition
anubhavitis Jun 11, 2024
ea099e6
Merge pull request #110 from fileverse/quests
anubhavitis Jun 11, 2024
1d33689
Merge pull request #112 from fileverse/upload-api
anubhavitis Jun 30, 2024
559d700
Merge pull request #113 from fileverse/upload-api
anubhavitis Jul 1, 2024
57e0801
Merge pull request #114 from fileverse/upload-api
anubhavitis Jul 1, 2024
63a1baa
Merge pull request #115 from fileverse/upload-api
anubhavitis Jul 2, 2024
b929f5c
Merge branch 'dev' of github.com:fileverse/fileverse-storage into ddo…
anubhavitis Jul 5, 2024
acc5294
Merge pull request #116 from fileverse/ddoc_dev
anubhavitis Jul 5, 2024
ee8b8d2
Merge branch 'ddoc' of github.com:fileverse/fileverse-storage into dd…
anubhavitis Jul 9, 2024
38087c2
Merge pull request #121 from fileverse/ddoc_dev
anubhavitis Jul 9, 2024
7f4c21b
ddoc endpoint disabling
anubhavitis Jul 10, 2024
768f35c
Merge branch 'ddoc' of github.com:fileverse/fileverse-storage into dd…
anubhavitis Jul 10, 2024
8787674
Merge pull request #125 from fileverse/ddoc_dev
anubhavitis Jul 10, 2024
09789f5
added redis stream publisher
anubhavitis Jul 16, 2024
c38ae87
Merge pull request #126 from fileverse/bug/memory-issue
anubhavitis Jul 16, 2024
c9a36e3
Revert "Bug/memory issue"
anubhavitis Jul 16, 2024
635d0e8
Merge pull request #127 from fileverse/revert-126-bug/memory-issue
anubhavitis Jul 16, 2024
c3c0928
Merge pull request #128 from fileverse/bug/memory-issue
anubhavitis Jul 16, 2024
c981694
added elastic apm support
anubhavitis Jul 17, 2024
8fa3862
Merge pull request #130 from fileverse/bug/memory-issue
anubhavitis Jul 17, 2024
12d24d2
Merge branch 'main' of github.com:fileverse/fileverse-storage into fe…
anubhavitis Jul 19, 2024
5c8c684
revere: testing setup for /portal/
anubhavitis Jul 19, 2024
3a125ea
Merge pull request #133 from fileverse/feat/publisher
anubhavitis Jul 21, 2024
bf0da40
feat: added password protection in redis
anubhavitis Jul 21, 2024
736428f
Merge pull request #134 from fileverse/feat/publisher
anubhavitis Jul 21, 2024
7fdf701
removed redis publishing, connected to f-producer endpoint
anubhavitis Jul 23, 2024
354ed9f
updated .example.env
anubhavitis Jul 23, 2024
3d0b1b5
Merge branch 'dev' of github.com:fileverse/fileverse-storage into pub…
anubhavitis Jul 23, 2024
4b6e3b7
Merge pull request #135 from fileverse/publisher-dev
anubhavitis Jul 23, 2024
ac56551
req limit added
anubhavitis Jul 25, 2024
f902bee
Merge branch 'main' of github.com:fileverse/fileverse-storage into bu…
anubhavitis Jul 25, 2024
88f382e
removed elastic
anubhavitis Jul 25, 2024
687e44a
env example fixed
anubhavitis Jul 25, 2024
d997a17
Merge branch 'dev' into bug/memory-issue
anubhavitis Jul 25, 2024
59680bd
unwanted imports removed
anubhavitis Jul 25, 2024
b10910a
Merge branch 'bug/memory-issue' of github.com:fileverse/fileverse-sto…
anubhavitis Jul 25, 2024
6d1d072
content type not available bug
anubhavitis Jul 25, 2024
fdba571
bad request status added for limit failure
anubhavitis Jul 25, 2024
0363533
encoding content type fixed
anubhavitis Jul 25, 2024
5fa82b3
increased limit to 400mb
anubhavitis Jul 25, 2024
70b4ef3
uninstalled elastic apm
anubhavitis Jul 25, 2024
9ad6015
removed req limit
anubhavitis Jul 25, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ typings/
# dotenv environment variables file
production.env
development.env
.env
*.env

# next.js build output
.next
Expand Down
9 changes: 8 additions & 1 deletion config/.env.example
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
HOST=
DEFAULT_IPFS_SERVICE=
DEFAULT_STORAGE_LIMIT=
ETH_GOERLI_RPC_URL=
ETH_MAINNET_RPC_URL=
ETH_SEPOLIA_RPC_URL=
FILEVERSE_TESTNET_RPC_URL=
GNOSIS_MAINNET_RPC_URL=
GNOSIS_TESTNET_RPC_URL=
FILEVERSE_TESTNET_RPC_URL=
GITCOIN_API_KEY=
MONGOURI=
NETWORK_PROVIDER_URL=
NODE_ENV=
Expand All @@ -13,4 +16,8 @@ PINATA_SECRET_KEY=
SERVICE_DID=
SLACK_REPORTER=
WEB3STORAGE_TOKEN=
FILEBASE_ACCESS_KEY=
FILEBASE_SECRET=
FILEBASE_BUCKET_NAME=
GITCOIN_API_KEY=
X_API_KEY=
14 changes: 8 additions & 6 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion src/domain/log/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,5 +2,6 @@ const create = require('./create');
const getByFile = require('./getByFile');
const getByContract = require('./getByContract');
const { ddocCreate, ddocSignup } = require('./ddoc');
const upsertComment = require('./upsertComment');

module.exports = { create, getByFile, getByContract, ddocCreate, ddocSignup };
module.exports = { create, getByFile, getByContract, upsertComment, ddocCreate, ddocSignup };
31 changes: 31 additions & 0 deletions src/domain/log/upsertComment.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
const { Log } = require('../../infra/database/models');
const completeMultipleTask = require('../task/completeMultipleTask')

async function upsertComment({ contractAddress, invokerAddress, file }) {
let resp = {
success: false,
task_completed: false
}
const portalLogs = await Log.findOneAndUpdate({
eventName: "comment",
contractAddress,
invokerAddress,
}, {
"$addToSet": { "tags": file }
}, {
"new": true, "upsert": true
});


resp.success = true;


if (portalLogs.tags.length >= 10) {
await completeMultipleTask({ contractAddress, invokerAddress, taskIds: ["COMMENT_ON_10_FILES"] });
resp.task_completed = true;
}

return resp;
}

module.exports = upsertComment;
16 changes: 13 additions & 3 deletions src/infra/reporter.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,19 @@ const config = require('../../config');
const axios = require('axios');

async function reportError(message) {
if (config.SLACK_REPORTER) {
await axios.post(config.SLACK_REPORTER, {
text: message,
env = config.HOST ? config.HOST : 'local';
slackHook = config.SLACK_REPORTER;
if (env !== 'local') {
await axios.post(slackHook, {
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "*Error* ```" + message + "```\n*Environment:* " + env.toUpperCase()
}
}
]
});
}
}
Expand Down
8 changes: 5 additions & 3 deletions src/interface/log/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -5,14 +5,16 @@ const {
const express = require('express');
const router = express.Router();

const canView = require('../middleware/canView');
const { canView, isAuthenticated } = require('../middleware');

const logComments = require('./logComments');
const create = require('./create');
const ddocCreate = require('./ddoc');
const ddocSignUp = require('./ddocSignup');

router.post('/create', asyncHandler(canView), asyncHandlerArray(create));
router.get('/create/ddoc', asyncHandlerArray(ddocCreate));
router.post('/signup/ddoc', asyncHandlerArray(ddocSignUp));
router.put('/comment', asyncHandler(isAuthenticated), asyncHandlerArray(logComments));
// router.get('/create/ddoc', asyncHandlerArray(ddocCreate));
// router.post('/signup/ddoc', asyncHandlerArray(ddocSignUp));

module.exports = router;
29 changes: 29 additions & 0 deletions src/interface/log/logComments.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
const { log } = require("../../domain");
const { validator } = require("../middleware");
const { Joi, validate } = validator;

const logCommentValidation = {
headers: Joi.object({
contract: Joi.string().required(),
invoker: Joi.string().required(),
}).unknown(true)
};

async function logComments(req, res) {
const { contractAddress, invokerAddress } = req;
const { file } = req.query;

try {
const data = await log.upsertComment({ contractAddress, invokerAddress, file });
res.json({
success: data.success,
task_completed: data.task_completed
});
}
catch (error) {
console.log(error);
res.status(500).json({ error: "Internal Server Error" });
}
}

module.exports = [validate(logCommentValidation), logComments];
2 changes: 1 addition & 1 deletion src/interface/middleware/canUpload.js
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ async function canUpload(req, res, next) {

const storageLimitBreached = await checkStorageLimit(contractAddress, invokerAddress);
if (storageLimitBreached) {
const statusCode = 507;
const statusCode = 400;
const message = `Storage for ${contractAddress} is full, please either claim more storage or contact us on twitter @fileverse`;
return ErrorHandler.throwError({ code: statusCode, message, req });
}
Expand Down
2 changes: 2 additions & 0 deletions src/interface/portal/indexPortal/enable.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
const Job = require('../../../domain/jobs');
const constants = require('../../../domain/contants');
const Task = require('../../../domain/task')
const publish = require('./publish');

async function enablePortalHadler(req, res) {
let resp = { ...constants.Response.GenericResp };
Expand All @@ -13,6 +14,7 @@ async function enablePortalHadler(req, res) {
}

try {
await publish({ contractAddress, chainId, publicLayoutFileId });
const newJob = await Job.createJob(constants.JobConst.Type.PublicPortal, jobBody, contractAddress);
resp.message = constants.Response.RespMsg.SUCCESS;
resp.data = {
Expand Down
43 changes: 43 additions & 0 deletions src/interface/portal/indexPortal/publish.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
const axios = require('axios');
const url = process.env.PRODUCER_HOST;
const apiKey = process.env.PRODUCER_API_KEY;

// Custom serializer for BigInt values
function customStringify(obj) {
return JSON.stringify(obj, (key, value) =>
typeof value === 'bigint' ? value.toString() : value,
2
);
}


async function publish(respObj) {

const message = customStringify(respObj);
if (url === undefined || apiKey === undefined) {
throw new Error('PRODUCER_HOST is not defined');
}

const endpoint = url + '/api/v1/producer/public-portal';

let config = {
method: 'post',
maxBodyLength: Infinity,
url: endpoint,
headers: {
'x-api-key': apiKey
},
data: message
};

const resp = await axios.request(config)

if (resp.status !== 200) {
throw new Error('Error in creating job: ' + resp.data);
} else {
console.log('Job created successfully');
}

}

module.exports = publish;