Implement http_parser_pause().

Summary:
- Add http_parser_pause() API. A callback may invoke this at any time.
  This will cause http_parser_parse() to return indicating that it
  parsed less than the number of requested bytes and set an error to
  HBE_PAUSED. A paused parser with fail with HBE_PAUSED until it is
  un-paused with http_parser_pause().
- Stop using 'state', 'header_state', 'index', and 'nread' shadow
  variables and then updating their http_parser fields when we're done.
  Instead, update the live values as we go. This will make it possible
  to return from anywhere in the parser (say, due to EPAUSED) and have
  valid/expected state.
- Update state before making callbacks so that if the want to pause,
  we'll know the correct state already.
- Make sure that every callback has a state that uniquely identifies the
  next step so that we can resume in the right place if we were suppoed
  to be paused.
- Clean and re-factor up CALLBACK() macros.
- Use CALLBACK() macros for (almost) all callbacks; on_headers_complete
  is still a special case. This includes on_body which we used to invoke
  manually with a long run of bytes. We now use a 'body' mark and hit
  its callback just like every other data callback.
- Clean up (most) gotos and replace with real states.
- Add some unit tests.

Fixes #70

http_parser.h

@@ -177,6 +177,7 @@ enum flags
   XX(INVALID_CONSTANT, "invalid constant string")                    \
   XX(INVALID_INTERNAL_STATE, "encountered unexpected internal state")\
   XX(STRICT, "strict mode assertion failed")                         \
+  XX(PAUSED, "parser is paused")                                     \
   XX(UNKNOWN, "an unknown error occurred")
 
 
@@ -201,20 +202,20 @@ enum http_errno {
 
 struct http_parser {
   /** PRIVATE **/
-  unsigned char type : 2;
+  unsigned char type : 2;     /* enum http_parser_type */
-  unsigned char flags : 6; /* F_* values from 'flags' enum; semi-public */
+  unsigned char flags : 6;    /* F_* values from 'flags' enum; semi-public */
-  unsigned char state;
+  unsigned char state;        /* enum state from http_parser.c */
-  unsigned char header_state;
+  unsigned char header_state; /* enum header_state from http_parser.c */
-  unsigned char index;
+  unsigned char index;        /* index into current matcher */
 
-  uint32_t nread;
+  uint32_t nread;          /* # bytes read in various scenarios */
-  int64_t content_length;
+  int64_t content_length;  /* # bytes in body (0 if no Content-Length header) */
 
   /** READ-ONLY **/
   unsigned short http_major;
   unsigned short http_minor;
   unsigned short status_code; /* responses only */
-  unsigned char method;    /* requests only */
+  unsigned char method;       /* requests only */
   unsigned char http_errno : 7;
 
   /* 1 = Upgrade header was present and the parser has exited because of that.
@@ -304,6 +305,9 @@ int http_parser_parse_url(const char *buf, size_t buflen,
                           int is_connect,
                           struct http_parser_url *u);
 
+/* Pause or un-pause the parser; a nonzero value pauses */
+void http_parser_pause(http_parser *parser, int paused);
+
 #ifdef __cplusplus
 }
 #endif

test.c

@@ -71,6 +71,7 @@ static int currently_parsing_eof;
 
 static struct message messages[5];
 static int num_messages;
+static http_parser_settings *current_pause_parser;
 
 /* * R E Q U E S T S * */
 const struct message requests[] =
@@ -1290,6 +1291,146 @@ message_complete_cb (http_parser *p)
   return 0;
 }
 
+/* These dontcall_* callbacks exist so that we can verify that when we're
+ * paused, no additional callbacks are invoked */
+int
+dontcall_message_begin_cb (http_parser *p)
+{
+  if (p) { } // gcc
+  fprintf(stderr, "\n\n*** on_message_begin() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_header_field_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_header_field() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_header_value_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_header_value() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_request_url_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_request_url() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_body_cb (http_parser *p, const char *buf, size_t len)
+{
+  if (p || buf || len) { } // gcc
+  fprintf(stderr, "\n\n*** on_body_cb() called on paused parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_headers_complete_cb (http_parser *p)
+{
+  if (p) { } // gcc
+  fprintf(stderr, "\n\n*** on_headers_complete() called on paused "
+                  "parser ***\n\n");
+  exit(1);
+}
+
+int
+dontcall_message_complete_cb (http_parser *p)
+{
+  if (p) { } // gcc
+  fprintf(stderr, "\n\n*** on_message_complete() called on paused "
+                  "parser ***\n\n");
+  exit(1);
+}
+
+static http_parser_settings settings_dontcall =
+  {.on_message_begin = dontcall_message_begin_cb
+  ,.on_header_field = dontcall_header_field_cb
+  ,.on_header_value = dontcall_header_value_cb
+  ,.on_url = dontcall_request_url_cb
+  ,.on_body = dontcall_body_cb
+  ,.on_headers_complete = dontcall_headers_complete_cb
+  ,.on_message_complete = dontcall_message_complete_cb
+  };
+
+/* These pause_* callbacks always pause the parser and just invoke the regular
+ * callback that tracks content. Before returning, we overwrite the parser
+ * settings to point to the _dontcall variety so that we can verify that
+ * the pause actually did, you know, pause. */
+int
+pause_message_begin_cb (http_parser *p)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return message_begin_cb(p);
+}
+
+int
+pause_header_field_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return header_field_cb(p, buf, len);
+}
+
+int
+pause_header_value_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return header_value_cb(p, buf, len);
+}
+
+int
+pause_request_url_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return request_url_cb(p, buf, len);
+}
+
+int
+pause_body_cb (http_parser *p, const char *buf, size_t len)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return body_cb(p, buf, len);
+}
+
+int
+pause_headers_complete_cb (http_parser *p)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return headers_complete_cb(p);
+}
+
+int
+pause_message_complete_cb (http_parser *p)
+{
+  http_parser_pause(p, 1);
+  *current_pause_parser = settings_dontcall;
+  return message_complete_cb(p);
+}
+
+static http_parser_settings settings_pause =
+  {.on_message_begin = pause_message_begin_cb
+  ,.on_header_field = pause_header_field_cb
+  ,.on_header_value = pause_header_value_cb
+  ,.on_url = pause_request_url_cb
+  ,.on_body = pause_body_cb
+  ,.on_headers_complete = pause_headers_complete_cb
+  ,.on_message_complete = pause_message_complete_cb
+  };
+
 static http_parser_settings settings =
   {.on_message_begin = message_begin_cb
   ,.on_header_field = header_field_cb
@@ -1359,6 +1500,17 @@ size_t parse_count_body (const char *buf, size_t len)
   return nparsed;
 }
 
+size_t parse_pause (const char *buf, size_t len)
+{
+  size_t nparsed;
+  http_parser_settings s = settings_pause;
+
+  currently_parsing_eof = (len == 0);
+  current_pause_parser = &s;
+  nparsed = http_parser_execute(parser, current_pause_parser, buf, len);
+  return nparsed;
+}
+
 static inline int
 check_str_eq (const struct message *m,
               const char *prop,
@@ -1982,6 +2134,58 @@ create_large_chunked_message (int body_size_in_kb, const char* headers)
   return buf;
 }
 
+/* Verify that we can pause parsing at any of the bytes in the
+ * message and still get the result that we're expecting. */
+void
+test_message_pause (const struct message *msg)
+{
+  char *buf = (char*) msg->raw;
+  size_t buflen = strlen(msg->raw);
+  size_t nread;
+
+  parser_init(msg->type);
+
+  do {
+    nread = parse_pause(buf, buflen);
+
+    // We can only set the upgrade buffer once we've gotten our message
+    // completion callback.
+    if (messages[0].message_complete_cb_called &&
+        msg->upgrade &&
+        parser->upgrade) {
+      messages[0].upgrade = buf + nread;
+      goto test;
+    }
+
+    if (nread < buflen) {
+
+      // Not much do to if we failed a strict-mode check
+      if (HTTP_PARSER_ERRNO(parser) == HPE_STRICT) {
+        parser_free();
+        return;
+      }
+
+      assert (HTTP_PARSER_ERRNO(parser) == HPE_PAUSED);
+    }
+
+    buf += nread;
+    buflen -= nread;
+    http_parser_pause(parser, 0);
+  } while (buflen > 0);
+
+  nread = parse_pause(NULL, 0);
+  assert (nread == 0);
+
+test:
+  if (num_messages != 1) {
+    printf("\n*** num_messages != 1 after testing '%s' ***\n\n", msg->name);
+    exit(1);
+  }
+
+  if(!message_eq(0, msg)) exit(1);
+
+  parser_free();
+}
 
 int
 main (void)
@@ -2012,6 +2216,10 @@ main (void)
     test_message(&responses[i]);
   }
 
+  for (i = 0; i < response_count; i++) {
+    test_message_pause(&responses[i]);
+  }
+
   for (i = 0; i < response_count; i++) {
     if (!responses[i].should_keep_alive) continue;
     for (j = 0; j < response_count; j++) {
@@ -2187,7 +2395,9 @@ main (void)
     test_message(&requests[i]);
   }
 
-
+  for (i = 0; i < request_count; i++) {
+    test_message_pause(&requests[i]);
+  }
 
   for (i = 0; i < request_count; i++) {
     if (!requests[i].should_keep_alive) continue;