From 6febcc350dd0f9593c1a3f27067ec5ff2be9db4e Mon Sep 17 00:00:00 2001 From: Andrew Bettison Date: Tue, 2 Jun 2015 02:18:14 +0930 Subject: [PATCH] Replace RESTful newsince polling with triggers Remove the 'api.restful.newsince_poll_ms' config option, no longer needed. --- conf_schema.h | 1 - http_server.h | 1 + httpd.c | 54 ++++++++++++++++++++++++++++++++++++++------ httpd.h | 9 ++++++++ meshms_restful.c | 16 +++++++++---- rhizome_restful.c | 13 +++++++---- tests/meshmsjava | 4 +--- tests/meshmsrestful | 6 ++--- tests/rhizomejava | 1 - tests/rhizomerestful | 3 +-- 10 files changed, 81 insertions(+), 27 deletions(-) diff --git a/conf_schema.h b/conf_schema.h index f84e29d1..47da87f3 100644 --- a/conf_schema.h +++ b/conf_schema.h @@ -482,7 +482,6 @@ END_ARRAY(10) STRUCT(api_restful) SUB_STRUCT(userlist, users,) ATOM(uint32_t, newsince_timeout, 60, uint32_time_interval,, "Time to block while reporting new bundles") -ATOM(uint32_t, newsince_poll_ms, 2000, uint32_nonzero,, "Database poll interval while blocked reporting new bundles") END_STRUCT STRUCT(api) diff --git a/http_server.h b/http_server.h index 0b9d79c2..7889ffe6 100644 --- a/http_server.h +++ b/http_server.h @@ -163,6 +163,7 @@ void http_request_free_response_buffer(struct http_request *r); int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz); void http_request_finalise(struct http_request *r); void http_request_pause_response(struct http_request *r, time_ms_t until); +void http_request_resume_response(struct http_request *r); void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes); void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *); void http_request_simple_response(struct http_request *r, uint16_t result, const char *body); diff --git a/httpd.c b/httpd.c index 70e97faf..3b914fe7 100644 --- a/httpd.c +++ b/httpd.c @@ -105,7 +105,6 @@ struct profile_total server_stats = { }; uint16_t httpd_server_port = 0; -unsigned int httpd_request_count = 0; static int httpd_server_socket = -1; static time_ms_t httpd_server_last_start_attempt = -1; @@ -233,9 +232,37 @@ success: return 0; } +static int httpd_dispatch(struct http_request *); + +static unsigned int http_request_uuid_counter = 0; +static httpd_request * current_httpd_requests = NULL; +unsigned int httpd_request_count = 0; + static void httpd_server_finalise_http_request(struct http_request *hr) { httpd_request *r = (httpd_request *) hr; + if (config.debug.httpd) + DEBUGF("httpd_request_count=%u current_httpd_requests=%p r=%p r->next=%p r->prev=%p", httpd_request_count, current_httpd_requests, r, r->next, r->prev); + if (r->next) { + assert(httpd_request_count >= 2); + assert(r->next->prev == r); + r->next->prev = r->prev; + } + if (r->prev) { + assert(httpd_request_count >= 2); + assert(r->prev->next == r); + r->prev->next = r->next; + } + else { + assert(current_httpd_requests == r); + current_httpd_requests = r->next; + } + r->next = r->prev = NULL; + assert(httpd_request_count > 0); + --httpd_request_count; + if (current_httpd_requests == NULL) { + assert(httpd_request_count == 0); + } if (r->manifest) { rhizome_manifest_free(r->manifest); r->manifest = NULL; @@ -244,14 +271,8 @@ static void httpd_server_finalise_http_request(struct http_request *hr) r->finalise_union(r); r->finalise_union = NULL; } - if (httpd_request_count) - --httpd_request_count; } -static int httpd_dispatch(struct http_request *); - -static unsigned int http_request_uuid_counter = 0; - void httpd_server_poll(struct sched_ent *alarm) { if (alarm->poll.revents & (POLLIN | POLLOUT)) { @@ -282,6 +303,13 @@ void httpd_server_poll(struct sched_ent *alarm) WHY("Cannot respond to HTTP request, out of memory"); close(sock); } else { + request->next = current_httpd_requests; + request->prev = NULL; + if (current_httpd_requests) { + assert(httpd_request_count > 0); + current_httpd_requests->prev = request; + } + current_httpd_requests = request; ++httpd_request_count; request->uuid = http_request_uuid_counter++; request->payload_status = INVALID_RHIZOME_PAYLOAD_STATUS; @@ -303,6 +331,18 @@ void httpd_server_poll(struct sched_ent *alarm) } } +static void trigger_rhizome_bundle_added(rhizome_manifest *m) +{ + httpd_request *r; + for (r = current_httpd_requests; r; r = r->next) { + if (r->trigger_rhizome_bundle_added) { + (*r->trigger_rhizome_bundle_added)(r, m); + } + } +} + +DEFINE_TRIGGER(rhizome_bundle_added, trigger_rhizome_bundle_added) + int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call) { IN(); diff --git a/httpd.h b/httpd.h index 1d4fc404..d0d98dfb 100644 --- a/httpd.h +++ b/httpd.h @@ -52,6 +52,11 @@ typedef struct httpd_request { struct http_request http; // MUST BE FIRST ELEMENT + /* Doubly-linked list of current requests. Used to pass triggers to requests. + */ + struct httpd_request *next; + struct httpd_request *prev; + /* Identify request from others being run. Monotonic counter feeds it. Only * used for debugging when we write post-.log files for multi-part form * requests. @@ -77,6 +82,10 @@ typedef struct httpd_request */ uint64_t ui64; + /* Trigger function for Rhizome bundle added. + */ + void (*trigger_rhizome_bundle_added)(struct httpd_request *, rhizome_manifest *); + /* Finaliser for union contents (below). */ void (*finalise_union)(struct httpd_request *); diff --git a/meshms_restful.c b/meshms_restful.c index 9c04325e..a3298bb6 100644 --- a/meshms_restful.c +++ b/meshms_restful.c @@ -22,6 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "httpd.h" #include "strbuf_helpers.h" +static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *m); + static void finalise_union_meshms_conversationlist(httpd_request *r) { meshms_free_conversations(r->u.mclist.conv); @@ -327,6 +329,7 @@ static int restful_meshms_newsince_messagelist_json(httpd_request *r, const char return 404; assert(r->finalise_union == NULL); r->finalise_union = finalise_union_meshms_messagelist; + r->trigger_rhizome_bundle_added = on_rhizome_bundle_added; r->u.msglist.rowcount = 0; r->u.msglist.phase = LIST_HEADER; enum meshms_status status; @@ -347,6 +350,14 @@ static int restful_meshms_newsince_messagelist_json(httpd_request *r, const char return 1; } +static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *m) +{ + // TODO select on sender and/or recipient fields + if (strcmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) { + http_request_resume_response(&r->http); + } +} + static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_meshms_messagelist_json_content_chunk; static int restful_meshms_messagelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result) @@ -411,10 +422,7 @@ static int restful_meshms_messagelist_json_content_chunk(struct http_request *hr ++r->u.msglist.rowcount; r->u.msglist.token = r->u.msglist.latest; meshms_message_iterator_close(&r->u.msglist.iter); - time_ms_t wake_at = now + config.api.restful.newsince_poll_ms; - if (wake_at > r->u.msglist.end_time) - wake_at = r->u.msglist.end_time; - http_request_pause_response(&r->http, wake_at); + http_request_pause_response(&r->http, r->u.msglist.end_time); return 0; } r->u.msglist.phase = LIST_END; diff --git a/rhizome_restful.c b/rhizome_restful.c index 5033192d..d3d28d73 100644 --- a/rhizome_restful.c +++ b/rhizome_restful.c @@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. #include "strbuf_helpers.h" static HTTP_RENDERER render_manifest_headers; +static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *m); static void finalise_union_read_state(httpd_request *r) { @@ -199,10 +200,16 @@ int restful_rhizome_newsince(httpd_request *r, const char *remainder) bzero(&r->u.rhlist.cursor, sizeof r->u.rhlist.cursor); r->u.rhlist.cursor.rowid_since = rowid; r->u.rhlist.end_time = gettime_ms() + config.api.restful.newsince_timeout * 1000; + r->trigger_rhizome_bundle_added = on_rhizome_bundle_added; http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_rhizome_bundlelist_json_content); return 1; } +static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *UNUSED(m)) +{ + http_request_resume_response(&r->http); +} + static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr, strbuf b) { httpd_request *r = (httpd_request *) hr; @@ -235,7 +242,6 @@ static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr if (!strbuf_overrun(b)) r->u.rhlist.phase = LIST_ROWS; return 1; - case LIST_FIRST: case LIST_ROWS: { @@ -248,10 +254,7 @@ static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr r->u.rhlist.phase = LIST_END; return 1; } - time_ms_t wake_at = now + config.api.restful.newsince_poll_ms; - if (wake_at > r->u.rhlist.end_time) - wake_at = r->u.rhlist.end_time; - http_request_pause_response(&r->http, wake_at); + http_request_pause_response(&r->http, r->u.rhlist.end_time); return 0; } rhizome_manifest *m = r->u.rhlist.cursor.manifest; diff --git a/tests/meshmsjava b/tests/meshmsjava index fc1078b9..48c641d0 100755 --- a/tests/meshmsjava +++ b/tests/meshmsjava @@ -146,8 +146,7 @@ test_MeshmsListMessages() { doc_MeshmsListMessagesNewSince="Java API list MeshMS messages in one conversation since token" setup_MeshmsListMessagesNewSince() { set_extra_config() { - executeOk_servald config set api.restful.newsince_timeout 1s \ - set api.restful.newsince_poll_ms 500 + executeOk_servald config set api.restful.newsince_timeout 1s } setup meshms_add_messages $SIDA1 $SIDA2 '><>>A>A<>><><><>>>A>A><<<><>>A<<>' @@ -199,7 +198,6 @@ doc_MeshmsListMessagesNewSinceArrival="Java API list newly arriving MeshMS messa setup_MeshmsListMessagesNewSinceArrival() { set_extra_config() { executeOk_servald config set api.restful.newsince_timeout 360s \ - set api.restful.newsince_poll_ms 1000 \ set api.restful.users.harry.password potter } setup diff --git a/tests/meshmsrestful b/tests/meshmsrestful index f231ec35..f47b86f4 100755 --- a/tests/meshmsrestful +++ b/tests/meshmsrestful @@ -290,8 +290,7 @@ doc_MeshmsListMessagesNewSince="HTTP RESTful list MeshMS messages in one convers setup_MeshmsListMessagesNewSince() { IDENTITY_COUNT=2 set_extra_config() { - executeOk_servald config set api.restful.newsince_timeout 1s \ - set api.restful.newsince_poll_ms 500 + executeOk_servald config set api.restful.newsince_timeout 1s } setup meshms_add_messages $SIDA1 $SIDA2 '><>>A>A<>><><><>>>A>A><<<><>>A<<>' @@ -345,8 +344,7 @@ doc_MeshmsListMessagesNewSinceArrival="HTTP RESTful list newly arriving MeshMS m setup_MeshmsListMessagesNewSinceArrival() { IDENTITY_COUNT=2 set_extra_config() { - executeOk_servald config set api.restful.newsince_timeout 360s \ - set api.restful.newsince_poll_ms 1000 + executeOk_servald config set api.restful.newsince_timeout 60s } setup # Use REST interface to send messages, not CLI, in order to avoid a database diff --git a/tests/rhizomejava b/tests/rhizomejava index c7a12a05..dc054d53 100755 --- a/tests/rhizomejava +++ b/tests/rhizomejava @@ -151,7 +151,6 @@ doc_RhizomeListNewSince="Java API list Rhizome bundles since token" setup_RhizomeListNewSince() { set_extra_config() { executeOk_servald config set api.restful.newsince_timeout 60s \ - set api.restful.newsince_poll_ms 1000 \ set api.restful.users.harry.password potter } setup diff --git a/tests/rhizomerestful b/tests/rhizomerestful index dc568b4a..7d0dea59 100755 --- a/tests/rhizomerestful +++ b/tests/rhizomerestful @@ -230,8 +230,7 @@ test_RhizomeList() { doc_RhizomeListNewSince="HTTP RESTful list Rhizome bundles since token as JSON" setup_RhizomeListNewSince() { set_extra_config() { - executeOk_servald config set api.restful.newsince_timeout 60s \ - set api.restful.newsince_poll_ms 1000 + executeOk_servald config set api.restful.newsince_timeout 60s } setup # Use REST interface to add bundles, not CLI, in order to avoid a database