Skip to content

Commit

Permalink
precompress database files in tar1090-db
Browse files Browse the repository at this point in the history 
load regCache as a json rather than constructing it from the database
  • Loading branch information
@wiedehopf
wiedehopf committed Oct 22, 2020
1 parent f530027 commit 2c0822e
Show file tree
Hide file tree
Showing 8 changed files with 118 additions and 53 deletions.
23 changes: 11 additions & 12 deletions 88-tar1090.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,33 +39,26 @@ $HTTP["url"] =~ "^/INSTANCE/data/aircraft\.json$" {
)
}
$HTTP["url"] =~ "^/INSTANCE/data/globe_.*" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=0",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/data/traces/" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=10",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/globe_history/" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=3600",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/data/icao_.*\.json$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=10",
)
}
$HTTP["url"] =~ "^/INSTANCE/db.*\.json$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=1209600",
)
}
$HTTP["url"] =~ "^/INSTANCE/.*\.png$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=2419200",
Expand All @@ -76,12 +69,18 @@ $HTTP["url"] =~ "^/INSTANCE/config.js$" {
"Cache-Control" => "public, must-revalidate",
)
}

$HTTP["url"] =~ "^/INSTANCE/.*_.*\.js$" {
$HTTP["url"] =~ "^/INSTANCE/[^/]*\.js$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=1209600",
)
}
$HTTP["url"] =~ "^/INSTANCE/db-.*\.js$" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=1209600",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/ol/.*$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=1209600",
Expand Down
38 changes: 30 additions & 8 deletions 95-tar1090-otherport.conf
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,14 @@
# serve tar1090 directly on port 8504
$SERVER["socket"] == ":8504" {
# redirect the slash-less URL
url.redirect += (
"^/INSTANCE$" => "/INSTANCE/"
)

alias.url += (
"/INSTANCE/data/" => "SOURCE_DIR/",
"/INSTANCE/chunks/" => "/run/SERVICE/",
"/INSTANCE/globe_history/" => "/var/globe_history/",
"/INSTANCE/" => "HTMLPATH/"
)

Expand Down Expand Up @@ -33,40 +39,56 @@ $HTTP["url"] =~ "^/INSTANCE/data/aircraft\.json$" {
)
}
$HTTP["url"] =~ "^/INSTANCE/data/globe_.*" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=0",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/data/traces/" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=10",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/data/icao_.*\.json$" {
$HTTP["url"] =~ "^/INSTANCE/globe_history/" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=10",
"Cache-Control" => "public, max-age=3600",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/.*\.png$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=2419200",
)
}
$HTTP["url"] =~ "^/INSTANCE/db.*\.json$" {
$HTTP["url"] =~ "^/INSTANCE/config.js$" {
setenv.add-response-header += (
"Cache-Control" => "public, must-revalidate",
)
}
$HTTP["url"] =~ "^/INSTANCE/[^/]*\.js$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=1209600",
)
}
$HTTP["url"] =~ "^/INSTANCE/.*\.png$" {
$HTTP["url"] =~ "^/INSTANCE/db-.*\.js$" {
compress.filetype = ()
setenv.add-response-header += (
"Cache-Control" => "public, max-age=2419200",
"Cache-Control" => "public, max-age=1209600",
"Content-Encoding" => "gzip",
)
}
$HTTP["url"] =~ "^/INSTANCE/.*\.js$" {
$HTTP["url"] =~ "^/INSTANCE/ol/.*$" {
setenv.add-response-header += (
"Cache-Control" => "public, max-age=1209600",
)
}
$HTTP["url"] =~ "^/INSTANCE/config.js$" {
$HTTP["url"] =~ "^/INSTANCE/jquery/.*$" {
setenv.add-response-header += (
"Cache-Control" => "must-revalidate",
"Cache-Control" => "public, max-age=1209600",
)
}
$HTTP["url"] =~ "^/INSTANCE/.*\.css$" {
Expand Down
33 changes: 29 additions & 4 deletions html/dbloader.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,12 @@
let _aircraft_cache = {};
let _airport_coords_cache = null;

let _request_count = 0;
let _request_queue = [];
let _request_cache = {};

let regCache = null;

function getAircraftData(icao) {
let defer = $.Deferred();
if (icao.charAt(0) == '~') {
Expand Down Expand Up @@ -107,10 +113,6 @@ function lookupIcaoAircraftType(aircraftData, defer) {
defer.resolve(aircraftData);
}

let _request_count = 0;
let _request_queue = [];
let _request_cache = {};

function db_ajax(bkey) {
let req;

Expand Down Expand Up @@ -143,6 +145,7 @@ function db_ajax_request_complete() {
dataType : 'json' });
ajaxreq.done(function(data) {
req.resolve(data);
/*
const keys = Object.keys(data);
for (let i in keys) {
if (keys[i] == 'children')
Expand All @@ -152,6 +155,7 @@ function db_ajax_request_complete() {
//console.log(reg);
regCache[reg] = icao;
}
*/
});
ajaxreq.fail(function(jqxhr, status, error) {
if (status == 'timeout') {
Expand All @@ -166,3 +170,24 @@ function db_ajax_request_complete() {
});
}
}
/*
function dbLoadAll() {
return $.getJSON(databaseFolder + "/files.js")
.done(function(data) {
for (let i in data) {
const icao = data[i].padEnd(6, 0);
//console.log(icao);
let req = getAircraftData(icao);
req.icao = icao;
req.fail(function(jqXHR,textStatus,errorThrown) {
if (textStatus == 'timeout') {
getAircraftData(this.icao);
console.log('Database load timeout:' + this.icao);
} else {
console.log(this.icao + ': Database load error: ' + textStatus + ' at URL: ' + jqXHR.url);
}
});
}
});
}
*/
2 changes: 1 addition & 1 deletion html/early.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ let historyTimeout = 60;
let globeIndex = 0;
let binCraft = false;
let dbServer = false;
let regCache = {};
let l3harris = false;
let heatmap = false;
let heatLoaded = 0;
Expand All @@ -30,6 +29,7 @@ let pTracks = false;
let lastTraceGet = 0;
let traceRate = 0;
let _aircraft_type_cache = null;
let dbLoadedAll = false;

let databaseFolder = "db2";

Expand Down
11 changes: 9 additions & 2 deletions html/planeObject.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,9 @@ function PlaneObject(icao) {
this.wtc = null;


this.regLoaded = false;
// request metadata
if (!dbServer)
if (!dbServer && !this.regLoaded)
this.getAircraftData();

// military icao ranges
Expand Down Expand Up @@ -1955,6 +1956,7 @@ PlaneObject.prototype.getAircraftData = function() {
let req = getAircraftData(this.icao);

req.done(function(data) {
this.regLoaded = true;
if (data == null) {
//console.log(this.icao + ': Not found in database!');
return;
Expand Down Expand Up @@ -2480,11 +2482,16 @@ PlaneObject.prototype.setTypeData = function() {
this.wtc = typeData.wtc;
}
PlaneObject.prototype.checkForDB = function(t) {
if (!t)
if (!t) {
return;
}

if (t.desc) this.typeLong = t.desc;
if (t.r) this.registration = t.r;

if (!t.r && !dbServer && !this.regLoaded)
this.getAircraftData();

if (t.t) {
this.icaoType = t.t;
this.setTypeData();
Expand Down
58 changes: 34 additions & 24 deletions html/script.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -686,26 +686,6 @@ function initialize() {

// Wait for history item downloads and append them to the buffer
push_history();
// this will be needed later
if (!dbServer && !onMobile && !hideButtons && !heatmap) {
$.getJSON(databaseFolder + "/files.js")
.done(function(data) {
for (let i in data) {
const icao = data[i].padEnd(6, 0);
//console.log(icao);
let req = getAircraftData(icao);
req.icao = icao;
req.fail(function(jqXHR,textStatus,errorThrown) {
if (textStatus == 'timeout') {
getAircraftData(this.icao);
console.log('Database load timeout:' + this.icao);
} else {
console.log(this.icao + ': Database load error: ' + textStatus + ' at URL: ' + jqXHR.url);
}
});
}
});
}
});

let coll = document.getElementsByClassName("collapseButton");
Expand Down Expand Up @@ -3180,9 +3160,20 @@ function onSearch(e) {
$("#search_input").val("");
$("#search_input").blur();
if (searchTerm)
findPlanes(searchTerm, true, true, true, true);
findPlanes(searchTerm, "byIcao", "byCallsign", "byReg", "byType");
return false;
}
/*
function onSearchReg(e) {
e.preventDefault();
const searchTerm = $("#search_reg_input").val().trim();
$("#search_reg_input").val("");
$("#search_reg_input").blur();
if (searchTerm)
findPlanes(searchTerm, false, false, "byReg", false);
return false;
}
*/

function onResetCallsignFilter(e) {
$("#callsign_filter").val("");
Expand Down Expand Up @@ -3688,9 +3679,27 @@ function findPlanes(query, byIcao, byCallsign, byReg, byType) {
return;
query = query.toLowerCase();
let results = [];
if (byReg && regCache[query.toUpperCase()]) {
selectPlaneByHex(regCache[query.toUpperCase()].toLowerCase(), {follow: true});
return;
if (byReg) {
if (regCache) {
if (regCache[query.toUpperCase()]) {
selectPlaneByHex(regCache[query.toUpperCase()].toLowerCase(), {follow: true});
return;
}
} else {
let req_url = databaseFolder + "/regIcao.js";
let req = $.ajax({ url: req_url,
cache: true,
timeout: 10000,
dataType : 'json'
});
req.done(function(data) {
regCache = data;
if (regCache[query.toUpperCase()]) {
selectPlaneByHex(regCache[query.toUpperCase()].toLowerCase(), {follow: true});
return;
}
});
}
}
for (let i in PlanesOrdered) {
const plane = PlanesOrdered[i];
Expand Down Expand Up @@ -3963,6 +3972,7 @@ function toggleLargeMode() {

function toggleShowTrace() {
if (!showTrace) {
dbServer = false; // temporary until i get the traces supplemented with registration data
toggleMultiSelect("off");
showTrace = true;
toggleFollow(false);
Expand Down
2 changes: 1 addition & 1 deletion install.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ fi

{ [[ "$1" == "test" ]] && cd "$ipath/git-db" && git rev-parse; } ||
{ cd "$ipath/git-db" &>/dev/null && git fetch --depth 1 origin master && git reset --hard FETCH_HEAD; } ||
{ rm -rf "$ipath/git-db" && git clone --depth 1 "$db_repo" "$ipath/git-db"; }
{ cd && rm -rf "$ipath/git-db" && git clone --depth 1 "$db_repo" "$ipath/git-db"; }

if ! cd $ipath/git-db || ! git rev-parse
then
Expand Down
4 changes: 3 additions & 1 deletion nginx.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,10 @@ location /INSTANCE {

gzip_static on;

location ~ db.*\.json$ {
location ~ db-.*\.js$ {
gzip off;
add_header Cache-Control "public, max-age=1209600";
add_header Content-Encoding "gzip";
}
location ~ .*\.png$ {
add_header Cache-Control "public, max-age=1209600";
Expand Down

0 comments on commit 2c0822e

Please sign in to comment.