Sean Cribbs avatar Sean Cribbs committed 57d4a53

Allow streaming of keys over the raw interface.

Streaming is done using chunked-encoding where each chunk is a JSON
object containing a number of keys. To invoke streaming, set the "keys"
query parameter to "stream". The first chunk is the bucket props unless you
set the "props" parameter to "false", in which case it will be an
empty JSON object.

Comments (0)

Files changed (2)


 -define(Q_PROPS, "props").
 -define(Q_KEYS,  "keys").
 -define(Q_FALSE, "false").
+-define(Q_STREAM, "stream").
 -define(Q_VTAG,  "vtag").


 %%      Includes the bucket props unless the "props=false" query param
 %%      is specified.
 %%      Includes the keys of the documents in the bucket unless the
-%%      "keys=false" query param is specified.
+%%      "keys=false" query param is specified. If "keys=stream" query param
+%%      is specified, keys will be streamed back to the client in JSON chunks
+%%      like so: {"keys":[Key1, Key2,...]}.
 %%      A Link header will also be added to the response by this function
 %%      if the keys are included in the JSON object.  The Link header
 %%      will include links to all keys in the bucket, with the property
     {KeyPart, KeyRD} =
         case wrq:get_qs_value(?Q_KEYS, RD) of
             ?Q_FALSE -> {[], RD};
+            ?Q_STREAM -> {stream, RD};
             _ ->
                 {ok, KeyList} = C:list_keys(B),
                 {[{?Q_KEYS, KeyList}],
                    RD, KeyList)}
-    {mochijson2:encode({struct, SchemaPart++KeyPart}), KeyRD, Ctx}.
+    case KeyPart of
+        stream -> {{stream, {mochijson2:encode({struct, SchemaPart}),
+                             fun() ->
+                                     {ok, ReqId} = C:stream_list_keys(B),
+                                     stream_keys(ReqId)
+                             end}},
+                   KeyRD,
+                   Ctx};
+       _ ->
+            {mochijson2:encode({struct, SchemaPart++KeyPart}), KeyRD, Ctx}
+    end.
+stream_keys(ReqId) ->
+    receive
+        {ReqId, {keys, Keys}} ->
+            {mochijson2:encode({struct, [{<<"keys">>, Keys}]}), fun() -> stream_keys(ReqId) end};                                                                     
+        {ReqId, done} -> {mochijson2:encode({struct, [{<<"keys">>, []}]}), done}
+    end.
 %% @spec accept_bucket_body(reqdata(), context()) -> {true, reqdata(), context()}
 %% @doc Modify the bucket properties according to the body of the
Tip: Filter by directory path e.g. /media app.js to search for public/media/app.js.
Tip: Use camelCasing e.g. ProjME to search for
Tip: Filter by extension type e.g. /repo .js to search for all .js files in the /repo directory.
Tip: Separate your search with spaces e.g. /ssh pom.xml to search for src/ssh/pom.xml.
Tip: Use ↑ and ↓ arrow keys to navigate and return to view the file.
Tip: You can also navigate files with Ctrl+j (next) and Ctrl+k (previous) and view the file with Ctrl+o.
Tip: You can also navigate files with Alt+j (next) and Alt+k (previous) and view the file with Alt+o.