Skip to content

Commit

Permalink
Merge pull request #18 from TravelMapping/master
Browse files Browse the repository at this point in the history
sync, 2018-11-06
  • Loading branch information
yakra authored Nov 7, 2018
2 parents 0037eb3 + 8015317 commit 12bb234
Show file tree
Hide file tree
Showing 5 changed files with 99 additions and 69 deletions.
67 changes: 14 additions & 53 deletions fpcull/fpcull.cpp
Original file line number Diff line number Diff line change
@@ -1,72 +1,33 @@
// Travel Mapping Project, Eric Bryant, 2017, 2018
#include <fstream>
#include <iostream>
#include <list>
using namespace std;

class entry
{ public:
string text;
bool cull;
entry *prev, *next;

// init alpha
entry() {prev = 0; next = 0; cull = 0;}
// insertions
entry(entry *cursor) // cursor, in this case, is a pointer to the prev entry
{ prev = cursor;
next = 0;
cull = 0;
}
};

bool find(string needle, entry *haystack)
{ while (haystack)
{ if (needle == haystack->text) return 1;
haystack = haystack->next;
}
return 0;
}

int main(int argc, char *argv[])
{ entry *cList = new entry;
entry *fList = new entry;
entry *cursor;
{ string FPline;
if (argc != 4) { cout << "usage: ./fpcull CullFile FullFile OutputFile\n"; return 0; }
// init ifstreams
ifstream cFile(argv[1], ios::in);
if (!cFile.is_open()) { cout << argv[1] << " file not found!\n"; return 0; }
ifstream fFile(argv[2], ios::in);
if (!fFile.is_open()) { cout << argv[2] << " file not found!\n"; return 0; }

// create cull list
cFile >> cList->text; //TODO: enclose in an IF in case hell breaks loose (IE, no text in cFile) //FIXME: see below
cursor = new entry(cList);
while (cFile >> cursor->text) //FIXME: This expects every line to be one text string and every text string to be one line. This will not always be the case.
// For example, see line 6613 of https://github.com/TravelMapping/HighwayData/blob/d33d0f6e93eec8ce7f71f31af94336a473161e5f/datacheckfps.csv
{ cursor->prev->next = cursor;
cursor = new entry(cursor);
}
delete cursor;

// create full list
fFile >> fList->text; //TODO: enclose in an IF in case hell breaks loose (IE, no text in fFile) //FIXME: see below
cursor = new entry(fList);
while (fFile >> cursor->text) //FIXME: This expects every line to be one text string and every text string to be one line. This will not always be the case.
// For example, see line 6613 of https://github.com/TravelMapping/HighwayData/blob/d33d0f6e93eec8ce7f71f31af94336a473161e5f/datacheckfps.csv
{ cursor->prev->next = cursor;
cursor = new entry(cursor);
list<string> fList;
while (getline(fFile, FPline)) fList.push_back(FPline);

// read cull list...
while (getline(cFile, FPline))
{ list<string>::iterator cursor = fList.begin();
// advance thru full FP list until current unmatched entry found...
while (*cursor != FPline && cursor != fList.end()) cursor++;
// ...and delete it.
if (cursor != fList.end()) cursor = fList.erase(cursor);
}
delete cursor;

// flag unmatchedfps as to-be-culled
for (cursor = fList; cursor; cursor = cursor->next)
if (find(cursor->text, cList)) cursor->cull = 1;

// output
ofstream oFile(argv[3]);
cursor = fList;
while (cursor)
{ if (!cursor->cull) oFile << cursor->text << endl;
cursor = cursor->next;
}
for (list<string>::iterator cursor = fList.begin(); cursor != fList.end(); cursor++)
oFile << *cursor << '\n';
}
12 changes: 8 additions & 4 deletions nmpfilter/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,11 @@
**Purpose:**<br>
Splits *tm-master.nmp* into smaller .nmp files filtered by country.

**usage:**<br>
**Compiling:**<br>
C++11 support is required.<br>
With GCC, I use the commandline `g++ nmpbycountry.cpp -std=c++11 -o nmpbycountry`

**Usage:**<br>
`nmpbycountry RgCsvFile MasterNMP OutputDir`, where
* `RgCsvFile` is the path of *regions.csv*
* `MasterNMP` is the path of *tm-master.nmp*
Expand All @@ -18,7 +22,7 @@ The output directory should be empty before running *nmpbycountry*. Files for co
**Purpose:**<br>
Splits *tm-master.nmp* into smaller .nmp files filtered by region.

**usage:**<br>
**Usage:**<br>
`nmpbyregion HwyDataDir MasterNMP OutputDir`, where
* `HwyDataDir` is the path of the *hwy_data* directory in the HighwayData repository, or equivalent
* `MasterNMP` is the path of *tm-master.nmp*
Expand All @@ -31,7 +35,7 @@ Splits *tm-master.nmp* into smaller .nmp files filtered by region.
**Purpose:**<br>
Removes marked false-positive pairs from a specified .nmp file.

**usage:**<br>
**Usage:**<br>
`nmpfpfilter InputFile OutputFile`

---
Expand All @@ -41,5 +45,5 @@ Removes marked false-positive pairs from a specified .nmp file.
**Purpose:**<br>
Removes all NMP pairs from a specified .nmp file except those that look intentional.

**usage:**<br>
**Usage:**<br>
`nmplifilter InputFile OutputFile`
24 changes: 24 additions & 0 deletions nmpfilter/nmpbycountry.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Travel Mapping Project, Eric Bryant, 2018
#include <ctime>
#include <fstream>
#include <iostream>
#include <list>
Expand Down Expand Up @@ -100,6 +101,29 @@ int main(int argc, char *argv[])
{ cout << "usage: nmpbycountry RgCsvFile MasterNMP OutputDir\n";
return 0;
}

// Record execution start time
time_t StartTime = time(0);
char* LocalTime = ctime(&StartTime);

// Attempt to find most recent commit info
string MasterInfo;
string MasterPath = argv[1];
MasterPath.erase(MasterPath.find_last_of("/\\")+1);
MasterPath += ".git/refs/heads/master";
ifstream MasterFile(MasterPath.data());
if (MasterFile)
MasterFile >> MasterInfo;
else MasterInfo = "unknown.";

// nmpbyregion.log
string LogPath = argv[3];
LogPath += "nmpbycountry.log";
ofstream LogFile(LogPath.data());
LogFile << "nmpbycountry executed " << LocalTime;
LogFile << "Most recent commit is " << MasterInfo << '\n';

// The actual filtering
list<string> CoList;
vector<region> RgList;
vector<string> master;
Expand Down
23 changes: 23 additions & 0 deletions nmpfilter/nmpbyregion.cpp
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
// Travel Mapping Project, Eric Bryant, 2018
#include <ctime>
#include <dirent.h>
#include <fstream>
#include <iostream>
Expand Down Expand Up @@ -63,6 +64,28 @@ int main(int argc, char *argv[])
{ cout << "usage: nmpbyregion HwyDataDir MasterNMP OutputDir\n";
return 0;
}

// Record execution start time
time_t StartTime = time(0);
char* LocalTime = ctime(&StartTime);

// Attempt to find most recent commit info
string MasterInfo;
string MasterPath = argv[1];
MasterPath += "../.git/refs/heads/master";
ifstream MasterFile(MasterPath.data());
if (MasterFile)
MasterFile >> MasterInfo;
else MasterInfo = "unknown.";

// nmpbyregion.log
string LogPath = argv[3];
LogPath += "nmpbyregion.log";
ofstream LogFile(LogPath.data());
LogFile << "nmpbyregion executed " << LocalTime;
LogFile << "Most recent commit is " << MasterInfo << '\n';

// The actual filtering
vector<string> RgList;
GetRegions(argv[1], RgList);
filter(RgList, argv[2], argv[3]);
Expand Down
42 changes: 30 additions & 12 deletions siteupdate/python-teresco/siteupdate.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
#!/usr/bin/env python3
# Travel Mapping Project, Jim Teresco, 2015, 2016, 2017
# Travel Mapping Project, Jim Teresco, 2015-2018
"""Python code to read .csv and .wpt files and prepare for
adding to the Travel Mapping Project database.
(c) 2015, 2016, 2017, Jim Teresco
(c) 2015-2018, Jim Teresco
This module defines classes to represent the contents of a
.csv file that lists the highways within a system, and a
Expand Down Expand Up @@ -58,6 +58,7 @@ def __init__(self,min_lat,min_lng,max_lat,max_lng):
self.sw_child = None
self.se_child = None
self.points = []
self.unique_locations = 0

def refine(self):
"""refine a quadtree into 4 sub-quadrants"""
Expand All @@ -71,13 +72,15 @@ def refine(self):
for p in points:
self.insert(p)


def insert(self,w):
"""insert Waypoint w into this quadtree node"""
#print("QTDEBUG: " + str(self) + " insert " + str(w))
if self.points is not None:
if self.waypoint_at_same_point(w) is None:
self.unique_locations += 1
#print("QTDEBUG: " + str(self) + " has " + str(self.unique_locations) + " unique locations")
self.points.append(w)
if len(self.points) > 50: # 50 points max per quadtree node
if self.unique_locations > 50: # 50 unique points max per quadtree node
self.refine()
else:
if w.lat < self.mid_lat:
Expand Down Expand Up @@ -195,8 +198,8 @@ def is_valid(self):

else:
# not refined, but should have no more than 50 points
if len(self.points) > 50:
print("ERROR: WaypointQuadtree.is_valid terminal quadrant has too many points (" + str(len(self.points)) + ")")
if self.unique_locations > 50:
print("ERROR: WaypointQuadtree.is_valid terminal quadrant has too many unique points (" + str(self.unique_locations) + ")")
return False
# not refined, so should not have any children
if self.nw_child is not None:
Expand All @@ -214,6 +217,22 @@ def is_valid(self):

return True

def max_colocated(self):
"""return the maximum number of waypoints colocated at any one location"""
max_col = 1
for p in self.point_list():
if max_col < p.num_colocated():
max_col = p.num_colocated()
print("Largest colocate count = " + str(max_col))
return max_col

def total_nodes(self):
if self.points is not None:
# not refined, no children, return 1 for self
return 1
else:
return 1 + self.nw_child.total_nodes() + self.ne_child.total_nodes() + self.sw_child.total_nodes() + self.se_child.total_nodes()

class Waypoint:
"""This class encapsulates the information about a single waypoint
from a .wpt file.
Expand Down Expand Up @@ -1035,8 +1054,8 @@ class DatacheckEntry:
form a sharp angle)
code is the error code string, one of SHARP_ANGLE, BAD_ANGLE,
DUPLICATE_LABELS, DUPLICATE_COORDS, LABEL_SELFREF,
LABEL_INVALID_CHAR, LONG_SEGMENT, LABEL_NO_VALID,
DUPLICATE_LABEL, DUPLICATE_COORDS, LABEL_SELFREF,
LABEL_INVALID_CHAR, LONG_SEGMENT,
LABEL_UNDERSCORES, VISIBLE_DISTANCE, LABEL_PARENS, LACKS_GENERIC,
BUS_WITH_I, NONTERMINAL_UNDERSCORE,
LONG_UNDERSCORE, LABEL_SLASHES, US_BANNER, VISIBLE_HIDDEN_COLOC,
Expand Down Expand Up @@ -3592,12 +3611,13 @@ def run(self):
str(points) + " points and " + str(segments) + " segments.")
if points != all_waypoints.size():
print("MISMATCH: all_waypoints contains " + str(all_waypoints.size()) + " waypoints!")
print("WaypointQuadtree contains " + str(all_waypoints.total_nodes()) + " total nodes.")

if not args.errorcheck:
# compute colocation of waypoints stats
print(et.et() + "Computing waypoint colocation stats, reporting all with 8 or more colocations:")
colocate_counts = [0]*50
largest_colocate_count = 1
largest_colocate_count = all_waypoints.max_colocated()
colocate_counts = [0]*(largest_colocate_count+1)
big_colocate_locations = dict()
for w in all_waypoints.point_list():
c = w.num_colocated()
Expand All @@ -3614,8 +3634,6 @@ def run(self):
big_colocate_locations[point] = the_list
#print(str(w) + " with " + str(c) + " other points.")
colocate_counts[c] += 1
if c > largest_colocate_count:
largest_colocate_count = c
for place in big_colocate_locations:
the_list = big_colocate_locations[place]
print(str(place) + " is occupied by " + str(len(the_list)) + " waypoints: " + str(the_list))
Expand Down

0 comments on commit 12bb234

Please sign in to comment.