From b07cd7bad3295ba8ba2888a090832607ccd12c3a Mon Sep 17 00:00:00 2001 From: Roland Olbricht Date: Thu, 22 Jan 2009 23:11:17 +0000 Subject: [PATCH] Abendlicher Zwischenstand --- Makefile | 43 ++++++++--- add_rule.c | 143 ++++++++++++++++++++++++++++++++++ expat_justparse_interface.h | 2 +- html/testseite.html | 18 ++++- init_database | 26 +------ init_derived_data | 49 ++++++++++++ osmy_vigilance.c | 141 +++++++++++++++++++++++++++++++++ script-interpreter.c | 1 + script_tools.c | 4 + script_tools.h | 5 ++ update_rule.c | 150 ++++++++++++++++++++++++++++++++++++ user_interface.c | 4 +- vigilance_control.c | 122 +++++++++++++++++++++++++++++ vigilance_control.h | 26 +++++++ 14 files changed, 696 insertions(+), 38 deletions(-) create mode 100644 add_rule.c create mode 100755 init_derived_data create mode 100644 osmy_vigilance.c create mode 100644 update_rule.c create mode 100644 vigilance_control.c create mode 100644 vigilance_control.h diff --git a/Makefile b/Makefile index 726378c1e..573ca03d7 100644 --- a/Makefile +++ b/Makefile @@ -12,17 +12,30 @@ conflict_$(suffix) \ report_$(suffix) \ detect_odd_nodes_$(suffix) -objects = expat_justparse_interface.o cgi-helper.o user_interface.o script_queries.o script_tools.o script-interpreter.o vigilance_control.o $(stmts) - +objects = expat_justparse_interface.o cgi-helper.o user_interface.o script_queries.o script_tools.o vigilance_control.o $(stmts) +executables_objects = script-interpreter.o add_rule.o update_rule.o osm2load_infile.o +executables = cgi-bin/interpreter cgi-bin/add_rule cgi-bin/update_rule import_osm osmy_vigilance tool_headers = expat_justparse_interface.h script_datatypes.h script_queries.h script_tools.h user_interface.h +main: $(executables) + suffix = statement.o -main: suffix = statement.o -main: $(objects) - g++ -o cgi-bin/input-interpreter -O3 -Wall -lexpat $(objects) `mysql_config --libs` +cgi-bin/interpreter: suffix = statement.o +cgi-bin/interpreter: $(objects) script-interpreter.o + g++ -o cgi-bin/interpreter -O3 -Wall -lexpat $(objects) script-interpreter.o `mysql_config --libs` -init_db: expat_justparse_interface.o cgi-helper.o user_interface.o osm2load_infile.o - g++ -o osm2load_infile -O3 -Wall -lexpat expat_justparse_interface.o cgi-helper.o user_interface.o osm2load_infile.o `mysql_config --libs` +suffix = statement.o +cgi-bin/add_rule: suffix = statement.o +cgi-bin/add_rule: $(objects) add_rule.o + g++ -o cgi-bin/add_rule -O3 -Wall -lexpat $(objects) add_rule.o `mysql_config --libs` + +suffix = statement.o +cgi-bin/update_rule: suffix = statement.o +cgi-bin/update_rule: $(objects) update_rule.o + g++ -o cgi-bin/update_rule -O3 -Wall -lexpat $(objects) update_rule.o `mysql_config --libs` + +import_osm: expat_justparse_interface.o cgi-helper.o user_interface.o osm2load_infile.o + g++ -o import_osm -O3 -Wall -lexpat expat_justparse_interface.o cgi-helper.o user_interface.o osm2load_infile.o `mysql_config --libs` osmy_vigilance: osmy_vigilance.c g++ -o osmy_vigilance -O3 -Wall `mysql_config --include` osmy_vigilance.c `mysql_config --libs` @@ -45,16 +58,24 @@ script_tools.o: script_tools.c $(tool_headers) vigilance_control.h %_statement.o: %_statement.c $(tool_headers) %_statement.h g++ -c -O3 -Wall `mysql_config --include` $< +vigilance_control.o: vigilance_control.c vigilance_control.h script_datatypes.h expat_justparse_interface.h cgi-helper.h user_interface.h + g++ -c -O3 -Wall vigilance_control.c + suffix = statement.h script-interpreter.o: script-interpreter.c expat_justparse_interface.h user_interface.h script_datatypes.h script_queries.h script_tools.h $(stmts) statement_factory.h g++ -c -O3 -Wall `mysql_config --include` script-interpreter.c +suffix = statement.h +add_rule.o: add_rule.c expat_justparse_interface.h user_interface.h script_datatypes.h script_queries.h script_tools.h $(stmts) statement_factory.h + g++ -c -O3 -Wall `mysql_config --include` add_rule.c + +suffix = statement.h +update_rule.o: update_rule.c expat_justparse_interface.h user_interface.h script_datatypes.h script_queries.h script_tools.h $(stmts) statement_factory.h + g++ -c -O3 -Wall `mysql_config --include` update_rule.c + osm2load_infile.o: osm2load_infile.c script_datatypes.h expat_justparse_interface.h cgi-helper.h user_interface.h g++ -c -O3 -Wall `mysql_config --include` osm2load_infile.c -vigilance_control.o: vigilance_control.c vigilance_control.h script_datatypes.h expat_justparse_interface.h cgi-helper.h user_interface.h osmy_vigilance - g++ -c -O3 -Wall vigilance_control.c - clean: suffix = statement.o clean: - rm -f $(objects) + rm -f $(objects) $(executable_objects) diff --git a/add_rule.c b/add_rule.c new file mode 100644 index 000000000..aafeccada --- /dev/null +++ b/add_rule.c @@ -0,0 +1,143 @@ +#include +#include +#include +#include +#include +#include +#include +#include "expat_justparse_interface.h" +#include "script_datatypes.h" +#include "script_queries.h" +#include "script_tools.h" +#include "statement_factory.h" +#include "user_interface.h" + +#include + +using namespace std; + +static int output_mode(HTML); + +MYSQL* mysql(NULL); + +vector< Statement* > statement_stack; +vector< string > text_stack; + +void start(const char *el, const char **attr) +{ + Statement* statement(generate_statement(el)); + if (statement) + { + statement->set_attributes(attr); + statement_stack.push_back(statement); + text_stack.push_back(get_parsed_text()); + reset_parsed_text(); + } +} + +void end(const char *el) +{ + if ((is_known_element(el)) && (statement_stack.size() > 1)) + { + Statement* statement(statement_stack.back()); + statement->add_final_text(get_parsed_text()); + reset_parsed_text(); + //Include an end-control to catch e.g. empty query-statements? + statement_stack.pop_back(); + statement_stack.back()->add_statement(statement, text_stack.back()); + text_stack.pop_back(); + } + else if ((is_known_element(el)) && (statement_stack.size() == 1)) + statement_stack.front()->add_final_text(get_parsed_text()); +} + +int main(int argc, char *argv[]) +{ + string xml_raw(get_xml_raw()); + if (display_encoding_errors(cout)) + return 0; + if (display_parse_errors(cout, xml_raw)) + return 0; + + parse(xml_raw, start, end); + if (display_parse_errors(cout, xml_raw)) + return 0; + // getting special information for rules + string rule_name(""); + int rule_replace(0); + Root_Statement* root_statement(dynamic_cast< Root_Statement* >(statement_stack.front())); + if (root_statement) + { + rule_name = root_statement->get_rule_name(); + rule_replace = root_statement->get_rule_replace(); + } + if (rule_name == "") + add_static_error("Adding a rule requires the name of the rule."); + if (rule_replace) + add_static_error("Providing a version-id while adding a rule is not allowed."); + + xml_raw = xml_raw.substr(xml_raw.find("') + 1); + if (xml_raw.find("") != string::npos) + xml_raw = xml_raw.substr(0, xml_raw.find("")); + else + add_static_error("No content between start and end of the root-tag."); + + if (display_static_errors(cout, xml_raw)) + return 0; + + //Sanity-Check + + mysql = mysql_init(NULL); + + if (!mysql_real_connect(mysql, "localhost", "osm", "osm", "osm", 0, NULL, + CLIENT_LOCAL_FILES)) + { + runtime_error("Connection to database failed.\n", cout); + return 0; + } + + out_header(cout, output_mode); + + int body_id(int_query(mysql, "select max(id) from rule_bodys")+1); + + ostringstream temp; + temp<<"select id from rule_names where name = '"; + escape_insert(temp, rule_name); + temp<<'\''; + int name_id(int_query(mysql, temp.str())); + + if (name_id) + { + temp.str(""); + temp<<"Rule '"< - inline Ostream& escape_insert(Ostream& out, string s) +inline Ostream& escape_insert(Ostream& out, string s) { unsigned int i(0); while (i < s.size()) diff --git a/html/testseite.html b/html/testseite.html index 1f083f323..22aaf9ce1 100644 --- a/html/testseite.html +++ b/html/testseite.html @@ -6,9 +6,23 @@ -

Eingabe für Input-Interpreter

+

Eingabe für Interpreter

-
+ + + +
+ +

Eingabe für "Update Rule"

+ +
+ + +
+ +

Eingabe für "Add Rule"

+ +
diff --git a/init_database b/init_database index de80ea291..b12f86e03 100755 --- a/init_database +++ b/init_database @@ -79,18 +79,7 @@ mysql -u osm -posm osm -e "create table key_s (id int unsigned, key_ varchar(218 mysql -u osm -posm osm -e "create table value_s (id int unsigned, value_ varchar(21844))" -mysql -u osm -posm osm -e "create table conflicts (id int, message varchar(21844))" -mysql -u osm -posm osm -e "create table node_conflicts (id int, conflict int)" -mysql -u osm -posm osm -e "create table way_conflicts (id int, conflict int)" -mysql -u osm -posm osm -e "create table relation_conflicts (id int, conflict int)" - - -mysql -u osm -posm osm -e "create table areas (id int)" -mysql -u osm -posm osm -e "create table area_segments (id int, lat_idx int, min_lat int, min_lon int, max_lat int, max_lon int)" -mysql -u osm -posm osm -e "create table area_tags (id int, key_ int unsigned, value_ int unsigned)" - - -bunzip2 <$1 | osm2load_infile --savemem --limit-node-tags --split-tables +bunzip2 <$1 | import_osm --savemem --limit-node-tags --split-tables echo "Collecting statements ... " @@ -151,16 +140,6 @@ echo "alter table member_roles add unique key(id);" >>init_database.sql echo "alter table key_s add unique key(id), add index(key_);" >>init_database.sql echo "alter table value_s add unique key(id), add index(value_);" >>init_database.sql -echo "alter table conflicts add unique key(id);" >>init_database.sql -echo "insert conflicts values (0, '');" >>init_database.sql -echo "alter table node_conflicts add key(id);" >>init_database.sql -echo "alter table way_conflicts add key(id);" >>init_database.sql -echo "alter table relation_conflicts add key(id);" >>init_database.sql -echo "alter table areas add unique key(id);" >>init_database.sql -echo "insert areas values (0);" >>init_database.sql -echo "alter table area_segments add key(id), add index(lat_idx, min_lon);" >>init_database.sql -echo "alter table area_tags add key(id);" >>init_database.sql - stmt="nodes_0" i=1 while [[ $i -lt $numberofsubtables ]]; @@ -214,3 +193,6 @@ echo "create table way_tags (id int unsigned, key_ int unsigned, value_ int unsi echo "Creating indices and merged tables ... " mysql -u osm -posm osm init_derived_data.sql + +echo "create table rule_names (id int, name varchar(21844));" >>init_derived_data.sql + +echo "create table rule_bodys (id int, rule int, source varchar(21844));" >>init_derived_data.sql + +echo "create table conflicts (id int, message varchar(21844));" >>init_derived_data.sql +echo "create table node_conflicts (id int, conflict int);" >>init_derived_data.sql +echo "create table way_conflicts (id int, conflict int);" >>init_derived_data.sql +echo "create table relation_conflicts (id int, conflict int);" >>init_derived_data.sql + +echo "create table areas (id int);" >>init_derived_data.sql +echo "create table area_segments (id int, lat_idx int, min_lat int, min_lon int, max_lat int, max_lon int);" >>init_derived_data.sql +echo "create table area_tags (id int, key_ int unsigned, value_ int unsigned);" >>init_derived_data.sql + +echo "alter table rule_names add unique key(id), add key(name);" >>init_derived_data.sql +echo "alter table rule_bodys add key(id), add key(rule);" >>init_derived_data.sql +echo "alter table conflicts add unique key(id);" >>init_derived_data.sql +echo "insert conflicts values (0, '');" >>init_derived_data.sql +echo "alter table node_conflicts add key(id);" >>init_derived_data.sql +echo "alter table way_conflicts add key(id);" >>init_derived_data.sql +echo "alter table relation_conflicts add key(id);" >>init_derived_data.sql +echo "alter table areas add unique key(id);" >>init_derived_data.sql +echo "insert areas values (0);" >>init_derived_data.sql +echo "alter table area_segments add key(id), add index(lat_idx, min_lon);" >>init_derived_data.sql +echo "alter table area_tags add key(id);" >>init_derived_data.sql + + +echo "Creating tables for derived data ... " + +mysql -u osm -posm osm +#include +#include +#include + +#include +#include +#include + +#include + +using namespace std; + +const char* FIFO_FILE = "/tmp/osmy_watchdog.pipe"; + +int main(int argc, char *argv[]) +{ + if ((argc == 2) && (!strcmp(argv[1], "shutdown"))) + { + //perform only shutdown, no server operation + int fd = open(FIFO_FILE, O_WRONLY|O_NONBLOCK); + write(fd, " shutdown ", 10); + + return 0; + } + + // prepare connection to mysql + MYSQL* mysql(NULL); + mysql = mysql_init(NULL); + if (!mysql_real_connect(mysql, "localhost", "osm", "osm", NULL, 0, NULL, + CLIENT_LOCAL_FILES)) + { + remove(FIFO_FILE); + return 0; + } + + multimap< unsigned long long, unsigned long long > event_queue; + + // create named pipe + umask(0); + mknod(FIFO_FILE, S_IFIFO|0666, 0); + + int fd = open(FIFO_FILE, O_RDONLY|O_NONBLOCK); + + bool shutdown_req(false); + while (!shutdown_req) + { + //sleep for a second + struct timeval timeout; + timeout.tv_sec = 1; + timeout.tv_usec = 0; + select (FD_SETSIZE, NULL, NULL, NULL, &timeout); + + //get data if any + string input; + char buf[80]; + int count(read(fd, buf, 79)); + while (count) + { + buf[count] = '\0'; + input += buf; + count = read(fd, buf, 79); + } + + unsigned long long current_time((uintmax_t)time(NULL)); + + // process data + unsigned int pos(0); + while (pos < input.size()) + { + unsigned long long pid(0), timeout(0); + while ((pos < input.size()) && (isspace(input[pos]))) + ++pos; + while ((pos < input.size()) && (isdigit(input[pos]))) + pid = 10*pid + (unsigned char)(input[pos++] - '0'); + while ((pos < input.size()) && (isspace(input[pos]))) + ++pos; + while ((pos < input.size()) && (isdigit(input[pos]))) + timeout = 10*timeout + (unsigned char)(input[pos++] - '0'); + //cerr<<'['< 0) && (timeout > 0)) + { +/* cerr< + (timeout + current_time, pid)); + } + else if ((pid > 0) && (input.substr(pos, 12) == "accomplished")) + { + pos += 12; +// cerr<::iterator it(event_queue.begin()); + while ((it != event_queue.end()) && (it->second != pid)) + ++it; + if (it != event_queue.end()) + event_queue.erase(it); + } + else if (input.substr(pos, 8) == "shutdown") + { +/* cerr<::iterator it(event_queue.begin()); + while ((it != event_queue.end()) && (it->first <= current_time)) + { + // kill runaway osm script + mysql_ping(mysql); + ostringstream temp; + temp<<"kill "<second; + mysql_query(mysql, temp.str().c_str()); + + cerr< attributes; attributes["timeout"] = "0"; + attributes["name"] = ""; + attributes["replace"] = "0"; eval_cstr_array(get_name(), attributes, attr); + name = attributes["name"]; + replace = atoi(attributes["replace"].c_str()); timeout = atoi(attributes["timeout"].c_str()); } diff --git a/script_tools.h b/script_tools.h index a11fe49fb..df59344e9 100644 --- a/script_tools.h +++ b/script_tools.h @@ -48,9 +48,14 @@ class Root_Statement : public Statement virtual void execute(MYSQL* mysql, map< string, Set >& maps); virtual ~Root_Statement() {} + string get_rule_name() { return name; } + int get_rule_replace() { return replace; } + private: vector< Statement* > substatements; int timeout; + string name; + int replace; }; #endif diff --git a/update_rule.c b/update_rule.c new file mode 100644 index 000000000..d9daf68f4 --- /dev/null +++ b/update_rule.c @@ -0,0 +1,150 @@ +#include +#include +#include +#include +#include +#include +#include +#include "expat_justparse_interface.h" +#include "script_datatypes.h" +#include "script_queries.h" +#include "script_tools.h" +#include "statement_factory.h" +#include "user_interface.h" + +#include + +using namespace std; + +static int output_mode(HTML); + +MYSQL* mysql(NULL); + +vector< Statement* > statement_stack; +vector< string > text_stack; + +void start(const char *el, const char **attr) +{ + Statement* statement(generate_statement(el)); + if (statement) + { + statement->set_attributes(attr); + statement_stack.push_back(statement); + text_stack.push_back(get_parsed_text()); + reset_parsed_text(); + } +} + +void end(const char *el) +{ + if ((is_known_element(el)) && (statement_stack.size() > 1)) + { + Statement* statement(statement_stack.back()); + statement->add_final_text(get_parsed_text()); + reset_parsed_text(); + //Include an end-control to catch e.g. empty query-statements? + statement_stack.pop_back(); + statement_stack.back()->add_statement(statement, text_stack.back()); + text_stack.pop_back(); + } + else if ((is_known_element(el)) && (statement_stack.size() == 1)) + statement_stack.front()->add_final_text(get_parsed_text()); +} + +int main(int argc, char *argv[]) +{ + string xml_raw(get_xml_raw()); + if (display_encoding_errors(cout)) + return 0; + if (display_parse_errors(cout, xml_raw)) + return 0; + + parse(xml_raw, start, end); + if (display_parse_errors(cout, xml_raw)) + return 0; + // getting special information for rules + string rule_name(""); + int rule_replace(0); + Root_Statement* root_statement(dynamic_cast< Root_Statement* >(statement_stack.front())); + if (root_statement) + { + rule_name = root_statement->get_rule_name(); + rule_replace = root_statement->get_rule_replace(); + } + if (rule_name == "") + add_static_error("Updating a rule requires the name of the rule."); + if (!rule_replace) + add_static_error("Updating a rule requires providing its last version-id."); + + xml_raw = xml_raw.substr(xml_raw.find("') + 1); + if (xml_raw.find("") != string::npos) + xml_raw = xml_raw.substr(0, xml_raw.find("")); + else + add_static_error("No content between start and end of the root-tag."); + + if (display_static_errors(cout, xml_raw)) + return 0; + + //Sanity-Check + + mysql = mysql_init(NULL); + + if (!mysql_real_connect(mysql, "localhost", "osm", "osm", "osm", 0, NULL, + CLIENT_LOCAL_FILES)) + { + runtime_error("Connection to database failed.\n", cout); + out_footer(cout, output_mode); + return 0; + } + + out_header(cout, output_mode); + + ostringstream temp; + temp<<"select id from rule_names where name = '"; + escape_insert(temp, rule_name); + temp<<'\''; + int name_id(int_query(mysql, temp.str())); + + if (!name_id) + { + temp.str(""); + temp<<"Rule '"<\n\n\n"; header_state = WRITTEN_XML; } - else if (type == HTML) + else if (type == HTML) { header_state = WRITTEN_HTML; out<<"Content-Type: text/html; charset=utf-8\n\n" @@ -501,7 +501,7 @@ void runtime_remark(const string& error, ostream& out) { if (header_state == 0) { - out_error_header(out, "Runtime Error"); + out_error_header(out, "Runtime Remark(s)"); out<<"

Runtime Error

\n"; out<<"

The following error occured while running your script:

\n"; diff --git a/vigilance_control.c b/vigilance_control.c new file mode 100644 index 000000000..3b0c78dd8 --- /dev/null +++ b/vigilance_control.c @@ -0,0 +1,122 @@ +#include +#include +#include + +#include +#include +#include + +#include "user_interface.h" +#include "vigilance_control.h" + +using namespace std; + +const char* FIFO_FILE = "/tmp/osmy_watchdog.pipe"; + +static unsigned long timeout_time(0); +static unsigned long pid_(0); + +// register the process for a penalty brake on a given timeout +// It starts the vigilance daemon if necesscary. +int add_timeout(int pid, int timeout) +{ + // automatic start of osmy_vigilance does somehow prevent the query from returning + // it's by the way also a security issue + /* int fd = open(FIFO_FILE, O_WRONLY|O_NONBLOCK); + + if (fd == -1) + { + if (errno == ENOENT) + { + int child = fork(); + if (child) + { + if (child == -1) + runtime_error("Something's wrong with vigilance control - can't start vigilance process.", cout); + + // number of retries for opening the pipe + // -- maybe the just started + int retries(10); + fd = open(FIFO_FILE, O_WRONLY|O_NONBLOCK); + while ((fd == -1) && (--retries >= 0)) + { + //sleep for a second + struct timeval timeout; + timeout.tv_sec = 1; + timeout.tv_usec = 0; + select (FD_SETSIZE, NULL, NULL, NULL, &timeout); + + fd = open(FIFO_FILE, O_WRONLY|O_NONBLOCK); + } + if (fd == -1) + runtime_error("Something's wrong with vigilance control - can't open timeout pipe.", cout); + } + else + execl("../osmy_vigilance", "osmy_vigilance", (char*) NULL); + } + else + runtime_error("Something's wrong with vigilance control - can't open timeout pipe.", cout); + }*/ + + // store current time. + timeout_time = (uintmax_t)time(NULL) + timeout; + pid_ = pid; + + ostringstream temp; + temp<<' '< 0) && ((uintmax_t)time(NULL) >= timeout_time)); +} + +// int main(int argc, char *argv[]) +// { +// int pid, timeout; +// cout<<"Id: "; +// cin>>pid; +// cout<<" timeout: "; +// cin>>timeout; +// +// if (timeout > 0) +// add_timeout(pid, timeout); +// else +// remove_timeout(pid); +// +// return 0; +// } diff --git a/vigilance_control.h b/vigilance_control.h new file mode 100644 index 000000000..b0c0e63b7 --- /dev/null +++ b/vigilance_control.h @@ -0,0 +1,26 @@ +#ifndef VIGILANCE_CONTROL +#define VIGILANCE_CONTROL + +#include +#include +#include + +#include +#include +#include + +#include "user_interface.h" + +using namespace std; + +// register the process for a penalty brake on a given timeout +// It starts the vigilance daemon if necesscary. +int add_timeout(int pid, int timeout); + +// unregister the process' timeout +int remove_timeout(); + +// unregister the process' timeout +bool is_timed_out(); + +#endif