From 34a9f22be9f4a728d89c4b42b234d19081a1a681 Mon Sep 17 00:00:00 2001 From: Kucing Basah 737 Date: Thu, 30 Nov 2023 16:04:27 +0700 Subject: [PATCH] Unique visitor id, close #7 Fill req.cookies.uvid with uuid.v1. --- lib/insert-hit.js | 4 +- lib/webserver/index.js | 16 ++++++ lib/webserver/routers/dump-request/index.js | 1 + lib/webserver/target-lookup.js | 12 ++++- .../20231130083559-table-hits-field-uvid.js | 53 +++++++++++++++++++ ...31130083559-table-hits-field-uvid-down.sql | 5 ++ ...0231130083559-table-hits-field-uvid-up.sql | 17 ++++++ package-lock.json | 21 ++++++++ package.json | 1 + 9 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 migrations/20231130083559-table-hits-field-uvid.js create mode 100644 migrations/sqls/20231130083559-table-hits-field-uvid-down.sql create mode 100644 migrations/sqls/20231130083559-table-hits-field-uvid-up.sql diff --git a/lib/insert-hit.js b/lib/insert-hit.js index ec906e1..8c240fb 100644 --- a/lib/insert-hit.js +++ b/lib/insert-hit.js @@ -14,9 +14,10 @@ const mysql = require('./mysql'); * @param {string} [userAgent] * @param {string} [referrer] * @param {string} [headers] + * @param {string} [uvid=ip] * @returns {string} new uuid */ -module.exports = async (xid, targetUuid, ip, userAgent, referrer, headers) => { +module.exports = async (xid, targetUuid, ip, userAgent, referrer, headers, uvid) => { const query = ` INSERT INTO ?? SET @@ -35,6 +36,7 @@ module.exports = async (xid, targetUuid, ip, userAgent, referrer, headers) => { referrer: referrer || null, headers: (typeof headers === 'string' ? headers : JSON.stringify(headers)) || null, + uvid: uvid || ip, }, ]; diff --git a/lib/webserver/index.js b/lib/webserver/index.js index e37bd80..e2032e1 100644 --- a/lib/webserver/index.js +++ b/lib/webserver/index.js @@ -8,8 +8,10 @@ const uniqid = require('uniqid'); const { default: RedisStore } = require('connect-redis'); const session = require('express-session'); const redis = require('redis'); +const cookieParser = require('cookie-parser'); const dayjs = require('dayjs'); const { IP, IPv4 } = require('ip-toolkit'); +const { v1: uuidv1 } = require('uuid'); const logger = require('../logger'); const accessLogger = require('./access-logger'); @@ -42,6 +44,8 @@ module.exports = async () => { const app = express(); + app.use(cookieParser()); + app.locals.appName = process.env.APP_NAME || 'REDIRECTOR'; app.locals.appVersion = global.appVersion; app.locals.mainPage = '/dashboard'; @@ -107,6 +111,18 @@ module.exports = async () => { res.locals.ip = ip; + // unique visitor id + if (!req.cookies.uvid) { + res.locals.uvid = uuidv1(); + logger.debug(`${MODULE_NAME} 3CEB38EE: New visitor detected, setting uivd cookie`, { + xid, + ip: req.ip, + uvid: res.locals.uvid, + }); + + req.cookies.uvid = res.locals.uvid; + } + next(); }); diff --git a/lib/webserver/routers/dump-request/index.js b/lib/webserver/routers/dump-request/index.js index b1e590b..4534384 100644 --- a/lib/webserver/routers/dump-request/index.js +++ b/lib/webserver/routers/dump-request/index.js @@ -24,6 +24,7 @@ const pageMain = (req, res) => { request: { clientIp: req.ip, clientIps: req.ips, + uvid: req.cookies.uvid, method: req.method, host: req.hostname, url: req.originalUrl, diff --git a/lib/webserver/target-lookup.js b/lib/webserver/target-lookup.js index 89a1507..6d6d272 100644 --- a/lib/webserver/target-lookup.js +++ b/lib/webserver/target-lookup.js @@ -102,8 +102,18 @@ module.exports = async (req, res, next) => { ? IPv4.toIPv6Format(ip).mapped : ip; - const hitUuid = await insertHit(xid, target.uuid, ipv6, req.get('user-agent'), req.get('referer'), req.headers); + const hitUuid = await insertHit( + xid, + target.uuid, + ipv6, + req.get('user-agent'), + req.get('referer'), + req.headers, + req.cookies?.uvid || null, + ); + await incrementHit(xid, target.uuid, hitUuid); + await touchGeoIp(xid, ipv6); } catch (e) { logger.warn(`${MODULE_NAME} 9E4990CC: Exception`, { diff --git a/migrations/20231130083559-table-hits-field-uvid.js b/migrations/20231130083559-table-hits-field-uvid.js new file mode 100644 index 0000000..69d6c30 --- /dev/null +++ b/migrations/20231130083559-table-hits-field-uvid.js @@ -0,0 +1,53 @@ +'use strict'; + +var dbm; +var type; +var seed; +var fs = require('fs'); +var path = require('path'); +var Promise; + +/** + * We receive the dbmigrate dependency from dbmigrate initially. + * This enables us to not have to rely on NODE_PATH. + */ +exports.setup = function(options, seedLink) { + dbm = options.dbmigrate; + type = dbm.dataType; + seed = seedLink; + Promise = options.Promise; +}; + +exports.up = function(db) { + var filePath = path.join(__dirname, 'sqls', '20231130083559-table-hits-field-uvid-up.sql'); + return new Promise( function( resolve, reject ) { + fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ + if (err) return reject(err); + console.log('received data: ' + data); + + resolve(data); + }); + }) + .then(function(data) { + return db.runSql(data); + }); +}; + +exports.down = function(db) { + var filePath = path.join(__dirname, 'sqls', '20231130083559-table-hits-field-uvid-down.sql'); + return new Promise( function( resolve, reject ) { + fs.readFile(filePath, {encoding: 'utf-8'}, function(err,data){ + if (err) return reject(err); + console.log('received data: ' + data); + + resolve(data); + }); + }) + .then(function(data) { + return db.runSql(data); + }); +}; + +exports._meta = { + "version": 1 +}; diff --git a/migrations/sqls/20231130083559-table-hits-field-uvid-down.sql b/migrations/sqls/20231130083559-table-hits-field-uvid-down.sql new file mode 100644 index 0000000..66abe60 --- /dev/null +++ b/migrations/sqls/20231130083559-table-hits-field-uvid-down.sql @@ -0,0 +1,5 @@ +ALTER TABLE `hits` DROP IF EXISTS `uvid`; + +ALTER TABLE `hits` DROP INDEX IF EXISTS `uvid`; + +ALTER TABLE `hits` DROP INDEX IF EXISTS `target_uvid`; diff --git a/migrations/sqls/20231130083559-table-hits-field-uvid-up.sql b/migrations/sqls/20231130083559-table-hits-field-uvid-up.sql new file mode 100644 index 0000000..270ef54 --- /dev/null +++ b/migrations/sqls/20231130083559-table-hits-field-uvid-up.sql @@ -0,0 +1,17 @@ +ALTER TABLE `hits` ADD IF NOT EXISTS `uvid` VARCHAR(64) NULL DEFAULT NULL AFTER `ip`; + +UPDATE `hits` + SET uvid = ip + WHERE uvid IS NULL; + +ALTER TABLE `hits` ADD INDEX IF NOT EXISTS `uvid` (`uvid`, `created`); + +ALTER TABLE `hits` + DROP INDEX IF EXISTS `created_date`, + ADD INDEX `created_date` (`created_date`, `target_uuid`, `uvid`) USING BTREE; + +ALTER TABLE `hits` + DROP INDEX IF EXISTS `created_date_hour`, + ADD INDEX `created_date_hour` (`created_date_hour`, `target_uuid`, `uvid`) USING BTREE; + +ALTER TABLE `hits` ADD INDEX IF NOT EXISTS `target_uvid` (`target_uuid`, `uvid`); diff --git a/package-lock.json b/package-lock.json index 1914455..103c156 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "bcrypt": "^5.1.1", "bootstrap": "^5.3.2", "connect-redis": "^7.1.0", + "cookie-parser": "^1.4.6", "dayjs": "^1.11.10", "dotenv": "^16.3.1", "express": "^4.18.2", @@ -1046,6 +1047,26 @@ "node": ">= 0.6" } }, + "node_modules/cookie-parser": { + "version": "1.4.6", + "resolved": "https://registry.npmjs.org/cookie-parser/-/cookie-parser-1.4.6.tgz", + "integrity": "sha512-z3IzaNjdwUC2olLIB5/ITd0/setiaFMLYiZJle7xg5Fe9KWAceil7xszYfHHBtDFYLSgJduS2Ty0P1uJdPDJeA==", + "dependencies": { + "cookie": "0.4.1", + "cookie-signature": "1.0.6" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/cookie-parser/node_modules/cookie": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/cookie/-/cookie-0.4.1.tgz", + "integrity": "sha512-ZwrFkGJxUR3EIoXtO+yVE69Eb7KlixbaeAWfBQB9vVsNn/o+Yw69gBWSSDK825hQNdN+wF8zELf3dFNl/kxkUA==", + "engines": { + "node": ">= 0.6" + } + }, "node_modules/cookie-signature": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz", diff --git a/package.json b/package.json index ea8dbe5..970b292 100644 --- a/package.json +++ b/package.json @@ -53,6 +53,7 @@ "bcrypt": "^5.1.1", "bootstrap": "^5.3.2", "connect-redis": "^7.1.0", + "cookie-parser": "^1.4.6", "dayjs": "^1.11.10", "dotenv": "^16.3.1", "express": "^4.18.2",