Replace RESTful newsince polling with triggers

Remove the 'api.restful.newsince_poll_ms' config option, no longer
needed.

conf_schema.h

@@ -482,7 +482,6 @@ END_ARRAY(10)
 STRUCT(api_restful)
 SUB_STRUCT(userlist,        users,)
 ATOM(uint32_t,              newsince_timeout,       60, uint32_time_interval,, "Time to block while reporting new bundles")
-ATOM(uint32_t,              newsince_poll_ms,       2000, uint32_nonzero,, "Database poll interval while blocked reporting new bundles")
 END_STRUCT
 
 STRUCT(api)

http_server.h

@@ -163,6 +163,7 @@ void http_request_free_response_buffer(struct http_request *r);
 int http_request_set_response_bufsize(struct http_request *r, size_t bufsiz);
 void http_request_finalise(struct http_request *r);
 void http_request_pause_response(struct http_request *r, time_ms_t until);
+void http_request_resume_response(struct http_request *r);
 void http_request_response_static(struct http_request *r, int result, const char *mime_type, const char *body, uint64_t bytes);
 void http_request_response_generated(struct http_request *r, int result, const char *mime_type, HTTP_CONTENT_GENERATOR *);
 void http_request_simple_response(struct http_request *r, uint16_t result, const char *body);

httpd.c

@@ -105,7 +105,6 @@ struct profile_total server_stats = {
 };
 
 uint16_t httpd_server_port = 0;
-unsigned int httpd_request_count = 0;
 
 static int httpd_server_socket = -1;
 static time_ms_t httpd_server_last_start_attempt = -1;
@@ -233,9 +232,37 @@ success:
   return 0;
 }
 
+static int httpd_dispatch(struct http_request *);
+
+static unsigned int http_request_uuid_counter = 0;
+static httpd_request * current_httpd_requests = NULL;
+unsigned int httpd_request_count = 0;
+
 static void httpd_server_finalise_http_request(struct http_request *hr)
 {
   httpd_request *r = (httpd_request *) hr;
+  if (config.debug.httpd)
+    DEBUGF("httpd_request_count=%u current_httpd_requests=%p r=%p r->next=%p r->prev=%p", httpd_request_count, current_httpd_requests, r, r->next, r->prev);
+  if (r->next) {
+    assert(httpd_request_count >= 2);
+    assert(r->next->prev == r);
+    r->next->prev = r->prev;
+  }
+  if (r->prev) {
+    assert(httpd_request_count >= 2);
+    assert(r->prev->next == r);
+    r->prev->next = r->next;
+  }
+  else {
+    assert(current_httpd_requests == r);
+    current_httpd_requests = r->next;
+  }
+  r->next = r->prev = NULL;
+  assert(httpd_request_count > 0);
+  --httpd_request_count;
+  if (current_httpd_requests == NULL) {
+    assert(httpd_request_count == 0);
+  }
   if (r->manifest) {
     rhizome_manifest_free(r->manifest);
     r->manifest = NULL;
@@ -244,14 +271,8 @@ static void httpd_server_finalise_http_request(struct http_request *hr)
     r->finalise_union(r);
     r->finalise_union = NULL;
   }
-  if (httpd_request_count)
-    --httpd_request_count;
 }
 
-static int httpd_dispatch(struct http_request *);
-
-static unsigned int http_request_uuid_counter = 0;
-
 void httpd_server_poll(struct sched_ent *alarm)
 {
   if (alarm->poll.revents & (POLLIN | POLLOUT)) {
@@ -282,6 +303,13 @@ void httpd_server_poll(struct sched_ent *alarm)
 	WHY("Cannot respond to HTTP request, out of memory");
 	close(sock);
       } else {
+	request->next = current_httpd_requests;
+	request->prev = NULL;
+	if (current_httpd_requests) {
+	  assert(httpd_request_count > 0);
+	  current_httpd_requests->prev = request;
+	}
+	current_httpd_requests = request;
 	++httpd_request_count;
 	request->uuid = http_request_uuid_counter++;
 	request->payload_status = INVALID_RHIZOME_PAYLOAD_STATUS;
@@ -303,6 +331,18 @@ void httpd_server_poll(struct sched_ent *alarm)
   }
 }
 
+static void trigger_rhizome_bundle_added(rhizome_manifest *m)
+{
+  httpd_request *r;
+  for (r = current_httpd_requests; r; r = r->next) {
+    if (r->trigger_rhizome_bundle_added) {
+      (*r->trigger_rhizome_bundle_added)(r, m);
+    }
+  }
+}
+
+DEFINE_TRIGGER(rhizome_bundle_added, trigger_rhizome_bundle_added)
+
 int is_http_header_complete(const char *buf, size_t len, size_t read_since_last_call)
 {
   IN();

httpd.h

@@ -52,6 +52,11 @@ typedef struct httpd_request
 {
   struct http_request http; // MUST BE FIRST ELEMENT
 
+  /* Doubly-linked list of current requests.  Used to pass triggers to requests.
+   */
+  struct httpd_request *next;
+  struct httpd_request *prev;
+
   /* Identify request from others being run.  Monotonic counter feeds it.  Only
    * used for debugging when we write post-<uuid>.log files for multi-part form
    * requests.
@@ -77,6 +82,10 @@ typedef struct httpd_request
    */
   uint64_t ui64;
 
+  /* Trigger function for Rhizome bundle added.
+   */
+  void (*trigger_rhizome_bundle_added)(struct httpd_request *, rhizome_manifest *);
+
   /* Finaliser for union contents (below).
    */
   void (*finalise_union)(struct httpd_request *);

meshms_restful.c

@@ -22,6 +22,8 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "httpd.h"
 #include "strbuf_helpers.h"
 
+static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *m);
+
 static void finalise_union_meshms_conversationlist(httpd_request *r)
 {
   meshms_free_conversations(r->u.mclist.conv);
@@ -327,6 +329,7 @@ static int restful_meshms_newsince_messagelist_json(httpd_request *r, const char
     return 404;
   assert(r->finalise_union == NULL);
   r->finalise_union = finalise_union_meshms_messagelist;
+  r->trigger_rhizome_bundle_added = on_rhizome_bundle_added;
   r->u.msglist.rowcount = 0;
   r->u.msglist.phase = LIST_HEADER;
   enum meshms_status status;
@@ -347,6 +350,14 @@ static int restful_meshms_newsince_messagelist_json(httpd_request *r, const char
   return 1;
 }
 
+static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *m)
+{
+  // TODO select on sender and/or recipient fields
+  if (strcmp(m->service, RHIZOME_SERVICE_MESHMS2) == 0) {
+    http_request_resume_response(&r->http);
+  }
+}
+
 static HTTP_CONTENT_GENERATOR_STRBUF_CHUNKER restful_meshms_messagelist_json_content_chunk;
 
 static int restful_meshms_messagelist_json_content(struct http_request *hr, unsigned char *buf, size_t bufsz, struct http_content_generator_result *result)
@@ -411,10 +422,7 @@ static int restful_meshms_messagelist_json_content_chunk(struct http_request *hr
 	      ++r->u.msglist.rowcount;
 	    r->u.msglist.token = r->u.msglist.latest;
 	    meshms_message_iterator_close(&r->u.msglist.iter);
-	    time_ms_t wake_at = now + config.api.restful.newsince_poll_ms;
-	    if (wake_at > r->u.msglist.end_time)
-	      wake_at = r->u.msglist.end_time;
-	    http_request_pause_response(&r->http, wake_at);
+	    http_request_pause_response(&r->http, r->u.msglist.end_time);
 	    return 0;
 	  }
 	  r->u.msglist.phase = LIST_END;

rhizome_restful.c

@@ -23,6 +23,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 #include "strbuf_helpers.h"
 
 static HTTP_RENDERER render_manifest_headers;
+static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *m);
 
 static void finalise_union_read_state(httpd_request *r)
 {
@@ -199,10 +200,16 @@ int restful_rhizome_newsince(httpd_request *r, const char *remainder)
   bzero(&r->u.rhlist.cursor, sizeof r->u.rhlist.cursor);
   r->u.rhlist.cursor.rowid_since = rowid;
   r->u.rhlist.end_time = gettime_ms() + config.api.restful.newsince_timeout * 1000;
+  r->trigger_rhizome_bundle_added = on_rhizome_bundle_added;
   http_request_response_generated(&r->http, 200, CONTENT_TYPE_JSON, restful_rhizome_bundlelist_json_content);
   return 1;
 }
 
+static void on_rhizome_bundle_added(httpd_request *r, rhizome_manifest *UNUSED(m))
+{
+  http_request_resume_response(&r->http);
+}
+
 static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr, strbuf b)
 {
   httpd_request *r = (httpd_request *) hr;
@@ -235,7 +242,6 @@ static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr
       if (!strbuf_overrun(b))
 	r->u.rhlist.phase = LIST_ROWS;
       return 1;
-      
     case LIST_FIRST:
     case LIST_ROWS:
       {
@@ -248,10 +254,7 @@ static int restful_rhizome_bundlelist_json_content_chunk(struct http_request *hr
 	    r->u.rhlist.phase = LIST_END;
 	    return 1;
 	  }
-	  time_ms_t wake_at = now + config.api.restful.newsince_poll_ms;
-	  if (wake_at > r->u.rhlist.end_time)
-	    wake_at = r->u.rhlist.end_time;
-	  http_request_pause_response(&r->http, wake_at);
+	  http_request_pause_response(&r->http, r->u.rhlist.end_time);
 	  return 0;
 	}
 	rhizome_manifest *m = r->u.rhlist.cursor.manifest;

tests/meshmsjava

@@ -146,8 +146,7 @@ test_MeshmsListMessages() {
 doc_MeshmsListMessagesNewSince="Java API list MeshMS messages in one conversation since token"
 setup_MeshmsListMessagesNewSince() {
    set_extra_config() {
-      executeOk_servald config set api.restful.newsince_timeout 1s \
-                               set api.restful.newsince_poll_ms 500
+      executeOk_servald config set api.restful.newsince_timeout 1s
    }
    setup
    meshms_add_messages $SIDA1 $SIDA2 '><>>A>A<>><><><>>>A>A><<<<A<>><>>A<<>'
@@ -199,7 +198,6 @@ doc_MeshmsListMessagesNewSinceArrival="Java API list newly arriving MeshMS messa
 setup_MeshmsListMessagesNewSinceArrival() {
    set_extra_config() {
       executeOk_servald config set api.restful.newsince_timeout 360s \
-                               set api.restful.newsince_poll_ms 1000 \
                                set api.restful.users.harry.password potter
    }
    setup

tests/meshmsrestful

@@ -290,8 +290,7 @@ doc_MeshmsListMessagesNewSince="HTTP RESTful list MeshMS messages in one convers
 setup_MeshmsListMessagesNewSince() {
    IDENTITY_COUNT=2
    set_extra_config() {
-      executeOk_servald config set api.restful.newsince_timeout 1s \
-                               set api.restful.newsince_poll_ms 500
+      executeOk_servald config set api.restful.newsince_timeout 1s
    }
    setup
    meshms_add_messages $SIDA1 $SIDA2 '><>>A>A<>><><><>>>A>A><<<<A<>><>>A<<>'
@@ -345,8 +344,7 @@ doc_MeshmsListMessagesNewSinceArrival="HTTP RESTful list newly arriving MeshMS m
 setup_MeshmsListMessagesNewSinceArrival() {
    IDENTITY_COUNT=2
    set_extra_config() {
-      executeOk_servald config set api.restful.newsince_timeout 360s \
-                               set api.restful.newsince_poll_ms 1000
+      executeOk_servald config set api.restful.newsince_timeout 60s
    }
    setup
    # Use REST interface to send messages, not CLI, in order to avoid a database

tests/rhizomejava

@@ -151,7 +151,6 @@ doc_RhizomeListNewSince="Java API list Rhizome bundles since token"
 setup_RhizomeListNewSince() {
    set_extra_config() {
       executeOk_servald config set api.restful.newsince_timeout 60s \
-                               set api.restful.newsince_poll_ms 1000 \
                                set api.restful.users.harry.password potter
    }
    setup

tests/rhizomerestful

@@ -230,8 +230,7 @@ test_RhizomeList() {
 doc_RhizomeListNewSince="HTTP RESTful list Rhizome bundles since token as JSON"
 setup_RhizomeListNewSince() {
    set_extra_config() {
-      executeOk_servald config set api.restful.newsince_timeout 60s \
-                               set api.restful.newsince_poll_ms 1000
+      executeOk_servald config set api.restful.newsince_timeout 60s
    }
    setup
    # Use REST interface to add bundles, not CLI, in order to avoid a database