Skip to content

Commit

Permalink
Rewrote readUnprocessedTrips, helper functions
Browse files Browse the repository at this point in the history 
- moved functions to timelineHelper
- updated momentJS code to luxon
  • Loading branch information
@the-bay-kay
the-bay-kay committed Nov 2, 2023
1 parent 545d407 commit 7a4b6fb
Show file tree
Hide file tree
Showing 4 changed files with 302 additions and 299 deletions.
4 changes: 2 additions & 2 deletions www/js/diary/LabelTab.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ import LabelListScreen from "./list/LabelListScreen";
import { createStackNavigator } from "@react-navigation/stack";
import LabelScreenDetails from "./details/LabelDetailsScreen";
import { NavigationContainer } from "@react-navigation/native";
import { compositeTrips2TimelineMap, getAllUnprocessedInputs, getLocalUnprocessedInputs, populateCompositeTrips, readAllCompositeTrips } from "./timelineHelper";
import { compositeTrips2TimelineMap, getAllUnprocessedInputs, getLocalUnprocessedInputs, populateCompositeTrips, readAllCompositeTrips, readUnprocessedTrips } from "./timelineHelper";
import { fillLocationNamesOfTrip, resetNominatimLimiter } from "./addressNamesHelper";
import { SurveyOptions } from "../survey/survey";
import { getLabelOptions } from "../survey/multilabel/confirmHelper";
Expand Down Expand Up @@ -209,7 +209,7 @@ const LabelTab = () => {
const lastProcessedTrip = timelineMap && [...timelineMap?.values()].reverse().find(
trip => trip.origin_key.includes('confirmed_trip')
);
readUnprocessedPromise = Timeline.readUnprocessedTrips(pipelineRange.end_ts, nowTs, lastProcessedTrip);
readUnprocessedPromise = readUnprocessedTrips(pipelineRange.end_ts, nowTs, lastProcessedTrip);
} else {
readUnprocessedPromise = Promise.resolve([]);
}
Expand Down
297 changes: 1 addition & 296 deletions www/js/diary/services.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@
import angular from 'angular';
import { SurveyOptions } from '../survey/survey';
import { getConfig } from '../config/dynamicConfig';
import { getRawEntries } from '../commHelper';
import { getUnifiedDataForInterval } from '../unifiedDataLoader'

angular.module('emission.main.diary.services', ['emission.plugin.logger',
'emission.services'])
Expand All @@ -27,304 +25,12 @@ angular.module('emission.main.diary.services', ['emission.plugin.logger',
});
});

/*
* This is going to be a bit tricky. As we can see from
* https://github.com/e-mission/e-mission-phone/issues/214#issuecomment-286279163,
* when we read local transitions, they have a string for the transition
* (e.g. `T_DATA_PUSHED`), while the remote transitions have an integer
* (e.g. `2`).
* See https://github.com/e-mission/e-mission-phone/issues/214#issuecomment-286338606
*
* Also, at least on iOS, it is possible for trip end to be detected way
* after the end of the trip, so the trip end transition of a processed
* trip may actually show up as an unprocessed transition.
* See https://github.com/e-mission/e-mission-phone/issues/214#issuecomment-286279163
*
* Let's abstract this out into our own minor state machine.
*/
var transitions2Trips = function(transitionList) {
var inTrip = false;
var tripList = []
var currStartTransitionIndex = -1;
var currEndTransitionIndex = -1;
var processedUntil = 0;

while(processedUntil < transitionList.length) {
// Logger.log("searching within list = "+JSON.stringify(transitionList.slice(processedUntil)));
if(inTrip == false) {
var foundStartTransitionIndex = transitionList.slice(processedUntil).findIndex(isStartingTransition);
if (foundStartTransitionIndex == -1) {
Logger.log("No further unprocessed trips started, exiting loop");
processedUntil = transitionList.length;
} else {
currStartTransitionIndex = processedUntil + foundStartTransitionIndex;
processedUntil = currStartTransitionIndex;
Logger.log("Unprocessed trip started at "+JSON.stringify(transitionList[currStartTransitionIndex]));
inTrip = true;
}
} else {
// Logger.log("searching within list = "+JSON.stringify(transitionList.slice(processedUntil)));
var foundEndTransitionIndex = transitionList.slice(processedUntil).findIndex(isEndingTransition);
if (foundEndTransitionIndex == -1) {
Logger.log("Can't find end for trip starting at "+JSON.stringify(transitionList[currStartTransitionIndex])+" dropping it");
processedUntil = transitionList.length;
} else {
currEndTransitionIndex = processedUntil + foundEndTransitionIndex;
processedUntil = currEndTransitionIndex;
Logger.log("currEndTransitionIndex = "+currEndTransitionIndex);
Logger.log("Unprocessed trip starting at "+JSON.stringify(transitionList[currStartTransitionIndex])+" ends at "+JSON.stringify(transitionList[currEndTransitionIndex]));
tripList.push([transitionList[currStartTransitionIndex],
transitionList[currEndTransitionIndex]])
inTrip = false;
}
}
}
return tripList;
}

var isStartingTransition = function(transWrapper) {
// Logger.log("isStartingTransition: transWrapper.data.transition = "+transWrapper.data.transition);
if(transWrapper.data.transition == 'local.transition.exited_geofence' ||
transWrapper.data.transition == 'T_EXITED_GEOFENCE' ||
transWrapper.data.transition == 1) {
// Logger.log("Returning true");
return true;
}
// Logger.log("Returning false");
return false;
}

var isEndingTransition = function(transWrapper) {
// Logger.log("isEndingTransition: transWrapper.data.transition = "+transWrapper.data.transition);
if(transWrapper.data.transition == 'T_TRIP_ENDED' ||
transWrapper.data.transition == 'local.transition.stopped_moving' ||
transWrapper.data.transition == 2) {
// Logger.log("Returning true");
return true;
}
// Logger.log("Returning false");
return false;
}

/*
* Fill out place geojson after pulling trip location points.
* Place is only partially filled out because we haven't linked the timeline yet
*/

var moment2localdate = function(currMoment, tz) {
return {
timezone: tz,
year: currMoment.year(),
//the months of the draft trips match the one format needed for
//moment function however now that is modified we need to also
//modify the months value here
month: currMoment.month() + 1,
day: currMoment.date(),
weekday: currMoment.weekday(),
hour: currMoment.hour(),
minute: currMoment.minute(),
second: currMoment.second()
};
}

var points2TripProps = function(locationPoints) {
var startPoint = locationPoints[0];
var endPoint = locationPoints[locationPoints.length - 1];
var tripAndSectionId = "unprocessed_"+startPoint.data.ts+"_"+endPoint.data.ts;
var startMoment = moment.unix(startPoint.data.ts).tz(startPoint.metadata.time_zone);
var endMoment = moment.unix(endPoint.data.ts).tz(endPoint.metadata.time_zone);

const speeds = [], dists = [];
let loc, locLatLng;
locationPoints.forEach((pt) => {
const ptLatLng = L.latLng([pt.data.latitude, pt.data.longitude]);
if (loc) {
const dist = locLatLng.distanceTo(ptLatLng);
const timeDelta = pt.data.ts - loc.data.ts;
dists.push(dist);
speeds.push(dist / timeDelta);
}
loc = pt;
locLatLng = ptLatLng;
});

const locations = locationPoints.map((point, i) => ({
loc: {
coordinates: [point.data.longitude, point.data.latitude]
},
ts: point.data.ts,
speed: speeds[i],
}));

return {
_id: {$oid: tripAndSectionId},
key: "UNPROCESSED_trip",
origin_key: "UNPROCESSED_trip",
additions: [],
confidence_threshold: 0,
distance: dists.reduce((a, b) => a + b, 0),
duration: endPoint.data.ts - startPoint.data.ts,
end_fmt_time: endMoment.format(),
end_local_dt: moment2localdate(endMoment, endPoint.metadata.time_zone),
end_ts: endPoint.data.ts,
expectation: {to_label: true},
inferred_labels: [],
locations: locations,
source: "unprocessed",
start_fmt_time: startMoment.format(),
start_local_dt: moment2localdate(startMoment, startPoint.metadata.time_zone),
start_ts: startPoint.data.ts,
user_input: {},
}
}

var tsEntrySort = function(e1, e2) {
// compare timestamps
return e1.data.ts - e2.data.ts;
}

var transitionTrip2TripObj = function(trip) {
var tripStartTransition = trip[0];
var tripEndTransition = trip[1];
var tq = {key: "write_ts",
startTs: tripStartTransition.data.ts,
endTs: tripEndTransition.data.ts
}
Logger.log("About to pull location data for range "
+ moment.unix(tripStartTransition.data.ts).toString() + " -> "
+ moment.unix(tripEndTransition.data.ts).toString());
const getSensorData = window['cordova'].plugins.BEMUserCache.getSensorDataForInterval;
return getUnifiedDataForInterval("background/filtered_location", tq, getSensorData)
.then(function(locationList) {
if (locationList.length == 0) {
return undefined;
}
var sortedLocationList = locationList.sort(tsEntrySort);
var retainInRange = function(loc) {
return (tripStartTransition.data.ts <= loc.data.ts) &&
(loc.data.ts <= tripEndTransition.data.ts)
}

var filteredLocationList = sortedLocationList.filter(retainInRange);

// Fix for https://github.com/e-mission/e-mission-docs/issues/417
if (filteredLocationList.length == 0) {
return undefined;
}

var tripStartPoint = filteredLocationList[0];
var tripEndPoint = filteredLocationList[filteredLocationList.length-1];
Logger.log("tripStartPoint = "+JSON.stringify(tripStartPoint)+"tripEndPoint = "+JSON.stringify(tripEndPoint));
// if we get a list but our start and end are undefined
// let's print out the complete original list to get a clue
// this should help with debugging
// https://github.com/e-mission/e-mission-docs/issues/417
// if it ever occurs again
if (angular.isUndefined(tripStartPoint) || angular.isUndefined(tripEndPoint)) {
Logger.log("BUG 417 check: locationList = "+JSON.stringify(locationList));
Logger.log("transitions: start = "+JSON.stringify(tripStartTransition.data)
+ " end = "+JSON.stringify(tripEndTransition.data.ts));
}

const tripProps = points2TripProps(filteredLocationList);

return {
...tripProps,
start_loc: {
type: "Point",
coordinates: [tripStartPoint.data.longitude, tripStartPoint.data.latitude]
},
end_loc: {
type: "Point",
coordinates: [tripEndPoint.data.longitude, tripEndPoint.data.latitude],
},
}
});
}

var linkTrips = function(trip1, trip2) {
// complete trip1
trip1.starting_trip = {$oid: trip2.id};
trip1.exit_fmt_time = trip2.enter_fmt_time;
trip1.exit_local_dt = trip2.enter_local_dt;
trip1.exit_ts = trip2.enter_ts;

// start trip2
trip2.ending_trip = {$oid: trip1.id};
trip2.enter_fmt_time = trip1.exit_fmt_time;
trip2.enter_local_dt = trip1.exit_local_dt;
trip2.enter_ts = trip1.exit_ts;
}

timeline.readUnprocessedTrips = function(startTs, endTs, lastProcessedTrip) {
$ionicLoading.show({
template: i18next.t('service.reading-unprocessed-data')
});

var tq = {key: "write_ts",
startTs,
endTs
}
Logger.log("about to query for unprocessed trips from "
+moment.unix(tq.startTs).toString()+" -> "+moment.unix(tq.endTs).toString());

const getMessageMethod = window['cordova'].plugins.BEMUserCache.getMessagesForInterval;
return getUnifiedDataForInterval("statemachine/transition", tq, getMessageMethod)
.then(function(transitionList) {
if (transitionList.length == 0) {
Logger.log("No unprocessed trips. yay!");
$ionicLoading.hide();
return [];
} else {
Logger.log("Found "+transitionList.length+" transitions. yay!");
var sortedTransitionList = transitionList.sort(tsEntrySort);
/*
sortedTransitionList.forEach(function(transition) {
console.log(moment(transition.data.ts * 1000).format()+":" + JSON.stringify(transition.data));
});
*/
var tripsList = transitions2Trips(transitionList);
Logger.log("Mapped into"+tripsList.length+" trips. yay!");
tripsList.forEach(function(trip) {
console.log(JSON.stringify(trip));
});
var tripFillPromises = tripsList.map(transitionTrip2TripObj);
return Promise.all(tripFillPromises).then(function(raw_trip_gj_list) {
// Now we need to link up the trips. linking unprocessed trips
// to one another is fairly simple, but we need to link the
// first unprocessed trip to the last processed trip.
// This might be challenging if we don't have any processed
// trips for the day. I don't want to go back forever until
// I find a trip. So if this is the first trip, we will start a
// new chain for now, since this is with unprocessed data
// anyway.

Logger.log("mapped trips to trip_gj_list of size "+raw_trip_gj_list.length);
/* Filtering: we will keep trips that are 1) defined and 2) have a distance >= 100m or duration >= 5 minutes
https://github.com/e-mission/e-mission-docs/issues/966#issuecomment-1709112578 */
const trip_gj_list = raw_trip_gj_list.filter((trip) =>
trip && (trip.distance >= 100 || trip.duration >= 300)
);
Logger.log("after filtering undefined and distance < 100, trip_gj_list size = "+raw_trip_gj_list.length);
// Link 0th trip to first, first to second, ...
for (var i = 0; i < trip_gj_list.length-1; i++) {
linkTrips(trip_gj_list[i], trip_gj_list[i+1]);
}
Logger.log("finished linking trips for list of size "+trip_gj_list.length);
if (lastProcessedTrip && trip_gj_list.length != 0) {
// Need to link the entire chain above to the processed data
Logger.log("linking unprocessed and processed trip chains");
linkTrips(lastProcessedTrip, trip_gj_list[0]);
}
$ionicLoading.hide();
Logger.log("Returning final list of size "+trip_gj_list.length);
return trip_gj_list;
});
}
});
}

var localCacheReadFn = timeline.updateFromDatabase;
var localCacheReadFn = timeline.updateFromDatabase;

timeline.getTrip = function(tripId) {
return angular.isDefined(timeline.data.tripMap)? timeline.data.tripMap[tripId] : undefined;
Expand All @@ -350,4 +56,3 @@ angular.module('emission.main.diary.services', ['emission.plugin.logger',

return timeline;
})

Loading

0 comments on commit 7a4b6fb

Please sign in to comment.