diff --git a/api/src/dns-scan/loaders/index.js b/api/src/dns-scan/loaders/index.js index 9b63653ab8..6d093db6d4 100644 --- a/api/src/dns-scan/loaders/index.js +++ b/api/src/dns-scan/loaders/index.js @@ -1,2 +1,3 @@ export * from './load-dns-by-key' export * from './load-dns-connections-by-domain-id' +export * from './load-mx-record-diff-by-domain-id' diff --git a/api/src/dns-scan/loaders/load-mx-record-diff-by-domain-id.js b/api/src/dns-scan/loaders/load-mx-record-diff-by-domain-id.js new file mode 100644 index 0000000000..a84eeb956d --- /dev/null +++ b/api/src/dns-scan/loaders/load-mx-record-diff-by-domain-id.js @@ -0,0 +1,241 @@ +import { aql } from 'arangojs' +import { t } from '@lingui/macro' + +export const loadMxRecordDiffByDomainId = + ({ query, userKey, cleanseInput, i18n }) => + async ({ limit, domainId, startDate, endDate, after, before, offset, orderBy }) => { + if (limit === undefined) { + console.warn(`User: ${userKey} did not set \`limit\` argument for: loadMxRecordDiffByDomainId.`) + throw new Error(i18n._(t`You must provide a \`limit\` value to properly paginate the \`MXRecord\` connection.`)) + } + + if (limit <= 0 || limit > 100) { + console.warn(`User: ${userKey} set \`limit\` argument outside accepted range: loadMxRecordDiffByDomainId.`) + throw new Error( + i18n._( + t`You must provide a \`limit\` value in the range of 1-100 to properly paginate the \`MXRecord\` connection.`, + ), + ) + } + + const paginationMethodCount = [before, after, offset].reduce( + (paginationMethod, currentValue) => currentValue + (paginationMethod === undefined), + 0, + ) + + if (paginationMethodCount > 1) { + console.warn(`User: ${userKey} set multiple pagination methods for: loadMxRecordDiffByDomainId.`) + throw new Error( + i18n._( + t`You must provide at most one pagination method (\`before\`, \`after\`, \`offset\`) value to properly paginate the \`MXRecord\` connection.`, + ), + ) + } + + before = cleanseInput(before) + after = cleanseInput(after) + + const usingRelayExplicitly = !!(before || after) + + const resolveCursor = (cursor) => { + const cursorString = Buffer.from(cursor, 'base64').toString('utf8').split('|') + + return cursorString.reduce((acc, currentValue) => { + const [type, id] = currentValue.split('::') + acc.push({ type, id }) + return acc + }, []) + } + let relayBeforeTemplate = aql`` + let relayAfterTemplate = aql`` + if (usingRelayExplicitly) { + const cursorList = resolveCursor(after || before) + + if (cursorList.length === 0 || cursorList > 2) { + // TODO: throw error + } + + if (cursorList.at(-1).type !== 'id') { + // id field should always be last property + // TODO: throw error + } + + const orderByDirectionArrow = + orderBy?.direction === 'DESC' ? aql`<` : orderBy?.direction === 'ASC' ? aql`>` : null + const reverseOrderByDirectionArrow = + orderBy?.direction === 'DESC' ? aql`>` : orderBy?.direction === 'ASC' ? aql`<` : null + + relayBeforeTemplate = aql`FILTER TO_NUMBER(dnsScan._key) < TO_NUMBER(${cursorList[0].id})` + relayAfterTemplate = aql`FILTER TO_NUMBER(dnsScan._key) > TO_NUMBER(${cursorList[0].id})` + + if (cursorList.length === 2) { + relayAfterTemplate = aql` + FILTER dnsScan.${cursorList[0].type} ${orderByDirectionArrow || aql`>`} ${cursorList[0].id} + OR (dnsScan.${cursorList[0].type} == ${cursorList[0].id} + AND TO_NUMBER(dnsScan._key) > TO_NUMBER(${cursorList[1].id})) + ` + + relayBeforeTemplate = aql` + FILTER dnsScan.${cursorList[0].type} ${reverseOrderByDirectionArrow || aql`<`} ${cursorList[0].id} + OR (dnsScan.${cursorList[0].type} == ${cursorList[0].id} + AND TO_NUMBER(dnsScan._key) < TO_NUMBER(${cursorList[1].id})) + ` + } + } + + const relayDirectionString = before ? aql`DESC` : aql`ASC` + + let sortTemplate + if (!orderBy) { + sortTemplate = aql`SORT TO_NUMBER(dnsScan._key) ${relayDirectionString}` + } else { + sortTemplate = aql`SORT dnsScan.${orderBy.field} ${orderBy.direction}, TO_NUMBER(dnsScan._key) ${relayDirectionString}` + } + + let startDateFilter = aql`` + if (typeof startDate !== 'undefined') { + startDateFilter = aql` + FILTER DATE_FORMAT(dnsScan.timestamp, '%yyyy-%mm-%dd') >= DATE_FORMAT(${startDate}, '%yyyy-%mm-%dd')` + } + + let endDateFilter = aql`` + if (typeof endDate !== 'undefined') { + endDateFilter = aql` + FILTER DATE_FORMAT(dnsScan.timestamp, '%yyyy-%mm-%dd') <= DATE_FORMAT(${endDate}, '%yyyy-%mm-%dd')` + } + + const removeExtraSliceTemplate = aql`SLICE(dnsScansPlusOne, 0, ${limit})` + const dnsScanQuery = aql` + WITH dns, domains + LET dnsScansPlusOne = ( + FOR dnsScan, e IN 1 OUTBOUND ${domainId} domainsDNS + FILTER dnsScan.mxRecords.diff == true + ${startDateFilter} + ${endDateFilter} + ${before ? relayBeforeTemplate : relayAfterTemplate} + ${sortTemplate} + LIMIT ${limit + 1} + RETURN { id: dnsScan._key, _type: "dnsScan", timestamp: dnsScan.timestamp, mxRecords: dnsScan.mxRecords } + ) + LET hasMoreRelayPage = LENGTH(dnsScansPlusOne) == ${limit} + 1 + LET hasReversePage = ${!usingRelayExplicitly} ? false : (LENGTH( + FOR dnsScan, e IN 1 OUTBOUND ${domainId} domainsDNS + FILTER dnsScan.mxRecords.diff == true + ${startDateFilter} + ${endDateFilter} + ${before ? relayAfterTemplate : relayBeforeTemplate} + LIMIT 1 + RETURN true + ) > 0) ? true : false + LET totalCount = COUNT( + FOR dnsScan, e IN 1 OUTBOUND ${domainId} domainsDNS + FILTER dnsScan.mxRecords.diff == true + ${startDateFilter} + ${endDateFilter} + RETURN true + ) + LET mxRecords = ${removeExtraSliceTemplate} + + RETURN { + "mxRecords": mxRecords, + "hasMoreRelayPage": hasMoreRelayPage, + "hasReversePage": hasReversePage, + "totalCount": totalCount + } + ` + + let mxRecordCursor + try { + mxRecordCursor = await query`${dnsScanQuery}` + } catch (err) { + console.error( + `Database error occurred while user: ${userKey} was trying to get cursor for DNS document with cursor '${ + after || before + }' for domain '${domainId}', error: ${err}`, + ) + throw new Error(i18n._(t`Unable to load DNS scan(s). Please try again.`)) + } + + let mxRecordInfo + try { + mxRecordInfo = await mxRecordCursor.next() + } catch (err) { + console.error( + `Cursor error occurred while user: ${userKey} was trying to get DNS information for ${domainId}, error: ${err}`, + ) + throw new Error(i18n._(t`Unable to load DNS scan(s). Please try again.`)) + } + + const mxRecords = mxRecordInfo.mxRecords + + if (mxRecords.length === 0) { + return { + edges: [], + totalCount: mxRecordInfo.totalCount, + pageInfo: { + hasPreviousPage: !usingRelayExplicitly + ? false + : after + ? mxRecordInfo.hasReversePage + : mxRecordInfo.hasMoreRelayPage, + hasNextPage: after || !usingRelayExplicitly ? mxRecordInfo.hasMoreRelayPage : mxRecordInfo.hasReversePage, + startCursor: null, + endCursor: null, + }, + } + } + + const toCursorString = (cursorObjects) => { + const cursorStringArray = cursorObjects.reduce((acc, cursorObject) => { + if (cursorObject.type === undefined || cursorObject.id === undefined) { + // TODO: throw error + } + acc.push(`${cursorObject.type}::${cursorObject.id}`) + return acc + }, []) + const cursorString = cursorStringArray.join('|') + return Buffer.from(cursorString, 'utf8').toString('base64') + } + + const edges = mxRecords.map((mxRecord) => { + let cursor + if (orderBy) { + cursor = toCursorString([ + { + type: orderBy.field, + id: mxRecord[orderBy.field], + }, + { + type: 'id', + id: mxRecord._key, + }, + ]) + } else { + cursor = toCursorString([ + { + type: 'id', + id: mxRecord._key, + }, + ]) + } + return { + cursor: cursor, + node: mxRecord, + } + }) + + return { + edges: edges, + totalCount: mxRecordInfo.totalCount, + pageInfo: { + hasPreviousPage: !usingRelayExplicitly + ? false + : after + ? mxRecordInfo.hasReversePage + : mxRecordInfo.hasMoreRelayPage, + hasNextPage: after || !usingRelayExplicitly ? mxRecordInfo.hasMoreRelayPage : mxRecordInfo.hasReversePage, + endCursor: edges.length > 0 ? edges.at(-1).cursor : null, + startCursor: edges.length > 0 ? edges[0].cursor : null, + }, + } + } diff --git a/api/src/dns-scan/objects/dns-scan.js b/api/src/dns-scan/objects/dns-scan.js index 28b2a493f5..2f07bdf7b4 100644 --- a/api/src/dns-scan/objects/dns-scan.js +++ b/api/src/dns-scan/objects/dns-scan.js @@ -1,4 +1,4 @@ -import { GraphQLBoolean, GraphQLInt, GraphQLList, GraphQLObjectType, GraphQLString } from 'graphql' +import { GraphQLBoolean, GraphQLList, GraphQLObjectType, GraphQLString } from 'graphql' import { globalIdField } from 'graphql-relay' import { GraphQLDateTime } from 'graphql-scalars' @@ -6,6 +6,7 @@ import { nodeInterface } from '../../node' import { dmarcType } from './dmarc' import { spfType } from './spf' import { dkimType } from './dkim' +import { mxRecordType } from './mx-record' export const dnsScanType = new GraphQLObjectType({ name: 'DNSScan', @@ -62,39 +63,6 @@ export const dnsScanType = new GraphQLObjectType({ description: `Results of DKIM, DMARC, and SPF scans on the given domain.`, }) -export const mxHostType = new GraphQLObjectType({ - name: 'MXHost', - fields: () => ({ - preference: { - type: GraphQLInt, - description: `The preference (or priority) of the host.`, - }, - hostname: { - type: GraphQLString, - description: `The hostname of the given host.`, - }, - addresses: { - type: GraphQLList(GraphQLString), - description: `The IP addresses for the given host.`, - }, - }), - description: `Hosts listed in the domain's MX record.`, -}) - -export const mxRecordType = new GraphQLObjectType({ - name: 'MXRecord', - fields: () => ({ - hosts: { - type: GraphQLList(mxHostType), - description: `Hosts listed in the domain's MX record.`, - }, - warnings: { - type: GraphQLList(GraphQLString), - description: `Additional warning info about the MX record.`, - }, - }), -}) - export const nsRecordType = new GraphQLObjectType({ name: 'NSRecord', fields: () => ({ diff --git a/api/src/dns-scan/objects/index.js b/api/src/dns-scan/objects/index.js index e649f21c81..1308111823 100644 --- a/api/src/dns-scan/objects/index.js +++ b/api/src/dns-scan/objects/index.js @@ -3,4 +3,6 @@ export * from './dkim-selector-result' export * from './dmarc' export * from './dns-scan' export * from './dns-scan-connection' +export * from './mx-record-connection' +export * from './mx-record' export * from './spf' diff --git a/api/src/dns-scan/objects/mx-record-connection.js b/api/src/dns-scan/objects/mx-record-connection.js new file mode 100644 index 0000000000..f33a7e3143 --- /dev/null +++ b/api/src/dns-scan/objects/mx-record-connection.js @@ -0,0 +1,16 @@ +import { GraphQLInt } from 'graphql' +import { connectionDefinitions } from 'graphql-relay' + +import { mxRecordDiffType } from './mx-record' + +export const mxRecordConnection = connectionDefinitions({ + name: 'MXRecordDiff', + nodeType: mxRecordDiffType, + connectionFields: () => ({ + totalCount: { + type: GraphQLInt, + description: 'The total amount of DNS scans related to a given domain.', + resolve: ({ totalCount }) => totalCount, + }, + }), +}) diff --git a/api/src/dns-scan/objects/mx-record.js b/api/src/dns-scan/objects/mx-record.js new file mode 100644 index 0000000000..5f65f6cd81 --- /dev/null +++ b/api/src/dns-scan/objects/mx-record.js @@ -0,0 +1,53 @@ +import { GraphQLInt, GraphQLList, GraphQLObjectType, GraphQLString } from 'graphql' +import { GraphQLDateTime } from 'graphql-scalars' +import { globalIdField } from 'graphql-relay' + +export const mxHostType = new GraphQLObjectType({ + name: 'MXHost', + fields: () => ({ + preference: { + type: GraphQLInt, + description: `The preference (or priority) of the host.`, + }, + hostname: { + type: GraphQLString, + description: `The hostname of the given host.`, + }, + addresses: { + type: GraphQLList(GraphQLString), + description: `The IP addresses for the given host.`, + }, + }), + description: `Hosts listed in the domain's MX record.`, +}) + +export const mxRecordType = new GraphQLObjectType({ + name: 'MXRecord', + fields: () => ({ + hosts: { + type: GraphQLList(mxHostType), + description: `Hosts listed in the domain's MX record.`, + }, + warnings: { + type: GraphQLList(GraphQLString), + description: `Additional warning info about the MX record.`, + }, + }), +}) + +export const mxRecordDiffType = new GraphQLObjectType({ + name: 'MXRecordDiff', + fields: () => ({ + id: globalIdField('dns'), + timestamp: { + type: GraphQLDateTime, + description: `The time when the scan was initiated.`, + resolve: ({ timestamp }) => new Date(timestamp), + }, + mxRecords: { + type: mxRecordType, + description: `The MX records for the domain (if they exist).`, + resolve: ({ mxRecords }) => mxRecords, + }, + }), +}) diff --git a/api/src/domain/objects/domain.js b/api/src/domain/objects/domain.js index fe97e28683..33a34071c4 100644 --- a/api/src/domain/objects/domain.js +++ b/api/src/domain/objects/domain.js @@ -14,6 +14,7 @@ import { organizationConnection } from '../../organization/objects' import { GraphQLDateTime } from 'graphql-scalars' import { dnsOrder } from '../../dns-scan/inputs' import { webOrder } from '../../web-scan/inputs/web-order' +import { mxRecordConnection } from '../../dns-scan/objects/mx-record-connection' export const domainType = new GraphQLObjectType({ name: 'Domain', @@ -160,6 +161,48 @@ export const domainType = new GraphQLObjectType({ }) }, }, + mxRecordDiff: { + type: mxRecordConnection.connectionType, + description: 'List of MX record diffs for a given domain.', + args: { + startDate: { + type: GraphQLDateTime, + description: 'Start date for date filter.', + }, + endDate: { + type: GraphQLDateTime, + description: 'End date for date filter.', + }, + orderBy: { + type: dnsOrder, + description: 'Ordering options for MX connections.', + }, + limit: { + type: GraphQLInt, + description: 'Number of MX scans to retrieve.', + }, + ...connectionArgs, + }, + resolve: async ( + { _id }, + args, + { userKey, auth: { checkDomainPermission, userRequired }, loaders: { loadMxRecordDiffByDomainId } }, + ) => { + await userRequired() + const permitted = await checkDomainPermission({ domainId: _id }) + if (!permitted) { + console.warn( + `User: ${userKey} attempted to access web scan results for ${_id}, but does not have permission.`, + ) + throw new Error(t`Cannot query web scan results without permission.`) + } + + return await loadMxRecordDiffByDomainId({ + domainId: _id, + ...args, + }) + }, + }, web: { type: webConnection.connectionType, description: 'HTTPS, and TLS scan results.', diff --git a/api/src/domain/queries/find-domain-by-domain.js b/api/src/domain/queries/find-domain-by-domain.js index d63a3aeeba..42b2915f22 100644 --- a/api/src/domain/queries/find-domain-by-domain.js +++ b/api/src/domain/queries/find-domain-by-domain.js @@ -1,8 +1,8 @@ -import {GraphQLNonNull} from 'graphql' -import {t} from '@lingui/macro' -import {Domain} from '../../scalars' +import { GraphQLNonNull } from 'graphql' +import { t } from '@lingui/macro' +import { Domain } from '../../scalars' -import {domainType} from '../objects' +import { domainType } from '../objects' export const findDomainByDomain = { type: domainType, @@ -19,20 +19,15 @@ export const findDomainByDomain = { { i18n, userKey, - auth: { - checkDomainPermission, - userRequired, - verifiedRequired, - loginRequiredBool, - }, - loaders: {loadDomainByDomain}, - validators: {cleanseInput}, + auth: { checkDomainPermission, userRequired, verifiedRequired, loginRequiredBool }, + loaders: { loadDomainByDomain }, + validators: { cleanseInput }, }, ) => { if (loginRequiredBool) { // Get User const user = await userRequired() - verifiedRequired({user}) + verifiedRequired({ user }) } // Cleanse input @@ -48,21 +43,17 @@ export const findDomainByDomain = { if (loginRequiredBool) { // Check user permission for domain access - const permitted = await checkDomainPermission({domainId: domain._id}) + const permitted = await checkDomainPermission({ domainId: domain._id }) if (!permitted) { console.warn(`User ${userKey} could not retrieve domain.`) throw new Error( - i18n._( - t`Permission Denied: Please contact organization user for help with retrieving this domain.`, - ), + i18n._(t`Permission Denied: Please contact organization user for help with retrieving this domain.`), ) } } - console.info( - `User ${userKey} successfully retrieved domain ${domain._key}.`, - ) + console.info(`User ${userKey} successfully retrieved domain ${domain._key}.`) return domain }, diff --git a/api/src/initialize-loaders.js b/api/src/initialize-loaders.js index adbcfbd082..d5b65bda04 100644 --- a/api/src/initialize-loaders.js +++ b/api/src/initialize-loaders.js @@ -60,7 +60,7 @@ import { loadVerifiedOrgConnections, } from './verified-organizations/loaders' import { loadChartSummaryByKey } from './summaries/loaders' -import { loadDnsConnectionsByDomainId } from './dns-scan' +import { loadDnsConnectionsByDomainId, loadMxRecordDiffByDomainId } from './dns-scan' export function initializeLoaders({ query, db, userKey, i18n, language, cleanseInput, loginRequiredBool, moment }) { return { @@ -157,6 +157,13 @@ export function initializeLoaders({ query, db, userKey, i18n, language, cleanseI cleanseInput, i18n, }), + loadMxRecordDiffByDomainId: loadMxRecordDiffByDomainId({ + query, + db, + userKey, + cleanseInput, + i18n, + }), loadWebConnectionsByDomainId: loadWebConnectionsByDomainId({ query, db, diff --git a/frontend/src/graphql/queries.js b/frontend/src/graphql/queries.js index 22201ed649..549dbd1822 100644 --- a/frontend/src/graphql/queries.js +++ b/frontend/src/graphql/queries.js @@ -231,6 +231,23 @@ export const DOMAIN_GUIDANCE_PAGE = gql` dmarcPhase hasDMARCReport userHasPermission + # mxRecordDiff(limit: 5, orderBy: { field: TIMESTAMP, direction: DESC }) { + # totalCount + # edges { + # node { + # id + # timestamp + # mxRecords { + # hosts { + # preference + # hostname + # addresses + # } + # warnings + # } + # } + # } + # } dnsScan(limit: 1, orderBy: { field: TIMESTAMP, direction: DESC }) { edges { cursor diff --git a/frontend/src/guidance/EmailGuidance.js b/frontend/src/guidance/EmailGuidance.js index a1c1be2f16..eee68a3632 100644 --- a/frontend/src/guidance/EmailGuidance.js +++ b/frontend/src/guidance/EmailGuidance.js @@ -67,7 +67,7 @@ export function EmailGuidance({ dnsResults, dmarcPhase, status }) { return {step} }) - const { dkim, dmarc, spf, timestamp, mxRecords } = dnsResults + const { dkim, dmarc, spf, timestamp, mxRecords, nsRecords } = dnsResults const emailKeys = ['spf', 'dkim', 'dmarc'] let emailPassCount = 0 let emailInfoCount = 0 @@ -306,11 +306,14 @@ export function EmailGuidance({ dnsResults, dmarcPhase, status }) { - MX + Mail Servers (MX) + + Latest Scan: + {mxRecords.hosts.map(({ preference, hostname, addresses }, idx) => { return ( @@ -350,6 +353,43 @@ export function EmailGuidance({ dnsResults, dmarcPhase, status }) { )} + + + + Name Servers (NS) + + + + + {nsRecords.hostnames.map((hostname, idx) => { + return ( + + + + Hostname: {hostname} + + + + ) + })} + {nsRecords.warnings.length > 0 && ( + + + Warnings: + + {nsRecords.warnings.map((warning, idx) => { + return ( + + + {idx + 1}. {warning} + + + ) + })} + + )} + + ) } diff --git a/scanners/dns-processor/service.py b/scanners/dns-processor/service.py index a87c61c627..33216bcb0c 100644 --- a/scanners/dns-processor/service.py +++ b/scanners/dns-processor/service.py @@ -16,8 +16,11 @@ load_dotenv() -logging.basicConfig(stream=sys.stdout, level=logging.INFO, - format='[%(asctime)s :: %(name)s :: %(levelname)s] %(message)s') +logging.basicConfig( + stream=sys.stdout, + level=logging.INFO, + format="[%(asctime)s :: %(name)s :: %(levelname)s] %(message)s", +) logger = logging.getLogger() NAME = os.getenv("NAME", "dns-processor") @@ -40,13 +43,13 @@ def to_camelcase(string): string = string # remove underscore and uppercase following letter - string = re.sub('_([a-z])', lambda match: match.group(1).upper(), string) + string = re.sub("_([a-z])", lambda match: match.group(1).upper(), string) # keep numbers seperated with hyphen - string = re.sub('([0-9])_([0-9])', r'\1-\2', string) + string = re.sub("([0-9])_([0-9])", r"\1-\2", string) # remove underscore before numbers - string = re.sub('_([0-9])', r'\1', string) + string = re.sub("_([0-9])", r"\1", string) # convert snakecase to camel - string = re.sub('_([a-z])', lambda match: match.group(1).upper(), string) + string = re.sub("_([a-z])", lambda match: match.group(1).upper(), string) return string @@ -56,7 +59,73 @@ def snake_to_camel(d): if isinstance(d, list): return [snake_to_camel(entry) for entry in d] if isinstance(d, dict): - return {to_camelcase(a): snake_to_camel(b) if isinstance(b, (dict, list)) else b for a, b in d.items()} + return { + to_camelcase(a): snake_to_camel(b) if isinstance(b, (dict, list)) else b + for a, b in d.items() + } + + +def mx_record_diff(processed_results): + domain = process_results.get("domain") + new_mx = processed_results.get("mx_records").get("hosts") + mx_record_diff = False + # fetch most recent scan of domain + last_mx = ( + db.aql.execute( + """ + FOR scan IN dns + FILTER scan.domain == @domain + SORT scan.timestamp DESC + LIMIT 1 + RETURN scan + """, + bind_vars={"domain": domain}, + ) + .next() + .get("mx_records") + .get("hosts") + ) + # compare mx_records to most recent scan + # if different, set mx_records_diff to True + # check number of hosts + + if len(new_mx) != len(last_mx): + if len(new_mx) > len(last_mx): + # print("host added") + mx_record_diff = True + else: + # print("host removed") + mx_record_diff = True + else: + # check hostnames + hostnames_new = [] + hostnames_last = [] + for i in range(len(new_mx)): + hostnames_new.append(new_mx[i]["hostname"]) + hostnames_last.append(last_mx[i]["hostname"]) + + if set(hostnames_new) != set(hostnames_last): + # print("host changed") + mx_record_diff = True + else: + # check hostname preferences and addresses + for i in range(len(new_mx)): + # find hostname in last_mx + for j in range(len(last_mx)): + if new_mx[i]["hostname"] == last_mx[j]["hostname"]: + # check preference + if new_mx[i]["preference"] != last_mx[j]["preference"]: + # print("preference changed") + mx_record_diff = True + break + # check addresses + if set(new_mx[i]["addresses"]) != set(last_mx[j]["addresses"]): + # print("addresses changed") + mx_record_diff = True + break + + processed_results["mx_records"].update({"diff": mx_record_diff}) + return processed_results async def run(loop): @@ -95,6 +164,7 @@ async def subscribe_handler(msg): shared_id = payload.get("shared_id") processed_results = process_results(results) + processed_results = mx_record_diff(processed_results) dmarc_status = processed_results.get("dmarc").get("status") spf_status = processed_results.get("spf").get("status") @@ -104,26 +174,30 @@ async def subscribe_handler(msg): if user_key is None: try: - dns_entry = db.collection("dns").insert(snake_to_camel(processed_results)) + dns_entry = db.collection("dns").insert( + snake_to_camel(processed_results) + ) domain = db.collection("domains").get({"_key": domain_key}) db.collection("domainsDNS").insert( { "_from": domain["_id"], "timestamp": processed_results["timestamp"], - "_to": dns_entry["_id"] + "_to": dns_entry["_id"], } ) - web_entry = db.collection("web").insert({ - "timestamp": str(datetime.datetime.now().astimezone()), - "domain": processed_results["domain"] - }) + web_entry = db.collection("web").insert( + { + "timestamp": str(datetime.datetime.now().astimezone()), + "domain": processed_results["domain"], + } + ) db.collection("domainsWeb").insert( { "_from": domain["_id"], "timestamp": processed_results["timestamp"], - "_to": web_entry["_id"] + "_to": web_entry["_id"], } ) @@ -180,14 +254,15 @@ async def subscribe_handler(msg): db.collection("domains").update(domain) for ip in results.get("resolve_ips", None) or []: - web_scan = db.collection("webScan").insert({ - "status": "pending", - "ipAddress": ip - }) - db.collection("webToWebScans").insert({ - "_from": web_entry["_id"], - "_to": web_scan["_id"], - }) + web_scan = db.collection("webScan").insert( + {"status": "pending", "ipAddress": ip} + ) + db.collection("webToWebScans").insert( + { + "_from": web_entry["_id"], + "_to": web_scan["_id"], + } + ) await nc.publish( f"{PUBLISH_TO}.{domain_key}.web", @@ -198,20 +273,20 @@ async def subscribe_handler(msg): "domain_key": domain_key, "shared_id": shared_id, "ip_address": ip, - "web_scan_key": web_scan["_key"] + "web_scan_key": web_scan["_key"], } ).encode(), ) - - except Exception as e: logging.error( f"Inserting processed results: {str(e)} \n\nFull traceback: {traceback.format_exc()}" ) return - logging.info(f"DNS Scans inserted into database: {json.dumps(processed_results)}") + logging.info( + f"DNS Scans inserted into database: {json.dumps(processed_results)}" + ) await nc.subscribe(subject=SUBSCRIBE_TO, queue=QUEUE_GROUP, cb=subscribe_handler) @@ -221,10 +296,10 @@ def ask_exit(sig_name): return loop.create_task(nc.close()) - for signal_name in {'SIGINT', 'SIGTERM'}: + for signal_name in {"SIGINT", "SIGTERM"}: loop.add_signal_handler( - getattr(signal, signal_name), - functools.partial(ask_exit, signal_name)) + getattr(signal, signal_name), functools.partial(ask_exit, signal_name) + ) def main():