Commits

Anonymous committed 79b63e3 Merge

merge

Comments (0)

Files changed (41)

 .*/.eunit/.*
 ^rel/riak/.*
 ^dev/.*
+include/.*_pb.hrl$
-                                Welcome to Riak.
-                                ================
+                           Welcome to Riak.
+                           ================
 
 
 
    Assuming you have a working Erlang (R13B03 or later) installation,
    building Riak should be as simple as:
 
-   $ cd $RIAK
-   $ make all rel
+  $ cd $RIAK
+  $ make all rel
+
 
 2.2 Starting Riak 
 ==================
    Once you have successfully built Riak, you can start the server with the
    following commands:
 
-   $ cd $RIAK/rel/riak
-   $ bin/riak start
+  $ cd $RIAK/rel/riak
+  $ bin/riak start
+
 
    Now, verify that the server started up cleanly and is working:
 
    Now that you have a functional server, let's try storing some data in
    it. First, start up a erlang node using our embedded version of erlang:
 
-   $ erts-<vsn>/bin/erl -name riaktest -setcookie riak
-   
-   Eshell V5.7.4  (abort with ^G)
-   (riaktest@example.com)1>
+  $ erts-<vsn>/bin/erl -name riaktest -setcookie riak
+  
+  Eshell V5.7.4  (abort with ^G)
+  (riaktest@example.com)1>
+
 
    Now construct the node name of Riak server and make sure we can talk to it:
 
-   (riaktest@example.com)4> RiakNode = riak_util:str_to_node(riak).
+  (riaktest@example.com)4> RiakNode = riak_util:str_to_node(riak).
+  
+  (riaktest@example.com)2> net_adm:ping(RiakNode).
+  pong
+  (riaktest@example.com)2>
 
-   (riaktest@example.com)2> net_adm:ping(RiakNode).
-   pong
-   (riaktest@example.com)2>
    
    We are now ready to start the Riak client:
 
-   (riaktest@example.com)2> {ok, C} = riak:client_connect(RiakNode).
-   {ok,{riak_client,'riak@example.com',4,136,81,151}}
+  (riaktest@example.com)2> {ok, C} = riak:client_connect(RiakNode).
+  {ok,{riak_client,'riak@example.com',<<4,136,81,151>>}}
+
 
    Let's create a shopping list for bread at /groceries/mine:
 
-   (riaktest@example.com)6> O0 = riak_object:new(<<"groceries">>, <<"mine">>, ["bread"]).
-   O0 = riak_object:new(<<"groceries">>, <<"mine">>, ["bread"]).
-   {r_object,<<"groceries">>,<<"mine">>,
-          [{r_content,{dict,0,16,16,8,80,48,
-                            {[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
-                            {{[],[],[],[],[],[],[],[],[],[],[],[],...}}},
-                      ["bread"]}],
-          [],
-          {dict,1,16,16,8,80,48,
-                {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
-                {{[],[],[],[],[],[],[],[],[],[],[],[],[],...}}},
-          undefined}
+  (riaktest@example.com)6> O0 = riak_object:new(<<"groceries">>, <<"mine">>, ["bread"]).
+  O0 = riak_object:new(<<"groceries">>, <<"mine">>, ["bread"]).
+  {r_object,<<"groceries">>,<<"mine">>,
+         [{r_content,{dict,0,16,16,8,80,48,
+                           {[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
+                           {{[],[],[],[],[],[],[],[],[],[],[],[],...}}},
+                     ["bread"]}],
+         [],
+         {dict,1,16,16,8,80,48,
+               {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
+               {{[],[],[],[],[],[],[],[],[],[],[],[],[],...}}},
+         undefined}
+  
+   (riaktest@example.com)3> C:put(O0, 1).
 
-    (riaktest@example.com)3> C:put(O0, 1).
     
     Now, read the list back from the Riak server and extract the value
 
-    (riaktest@example.com)4> {ok, O1} = C:get(<<"groceries">>, <<"mine">>, 1).
-    {ok,{r_object,<<"groceries">>,<<"mine">>,
-              [{r_content,{dict,2,16,16,8,80,48,
-                                {[],[],[],[],[],[],[],[],[],[],[],[],...},
-                                {{[],[],[],[],[],[],
-                                  ["X-Riak-Last-Modified",87|...],
-                                  [],[],[],...}}},
-                          ["bread"]}],
-              [{"20090722191020-riaktest@example.com-riakdemo@example.com-266664",
-                {1,63415509105}}],
-              {dict,0,16,16,8,80,48,
-                    {[],[],[],[],[],[],[],[],[],[],[],[],[],...},
-                    {{[],[],[],[],[],[],[],[],[],[],[],...}}},
-              undefined}}
+  (riaktest@example.com)4> {ok, O1} = C:get(<<"groceries">>, <<"mine">>, 1).
+  {ok,{r_object,<<"groceries">>,<<"mine">>,
+            [{r_content,{dict,2,16,16,8,80,48,
+                              {[],[],[],[],[],[],[],[],[],[],[],[],...},
+                              {{[],[],[],[],[],[],
+                                ["X-Riak-Last-Modified",87|...],
+                                [],[],[],...}}},
+                        ["bread"]}],
+            [{"20090722191020-riaktest@example.com-riakdemo@example.com-266664",
+              {1,63415509105}}],
+            {dict,0,16,16,8,80,48,
+                  {[],[],[],[],[],[],[],[],[],[],[],[],[],...},
+                  {{[],[],[],[],[],[],[],[],[],[],[],...}}},
+            undefined}}
+  
+   (riaktest@example.com)5> %% extract the value
+   (riaktest@example.com)5> V = riak_object:get_value(O1).
+   ["bread"]
 
-     (riaktest@example.com)5> %% extract the value
-     (riaktest@example.com)5> V = riak_object:get_value(O1).
-     ["bread"]
 
      Add milk to our list of groceries and write the new value to Riak:
 
-     (riaktest@example.com)6> %% add milk to the list
-     (riaktest@example.com)6> O2 = riak_object:update_value(O1, ["milk" | V]).
-     {r_object,<<"groceries">>,<<"mine">>,
-          [{r_content,{dict,2,16,16,8,80,48,
-                            {[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
-                            {{[],[],[],[],[],[],
-                              ["X-Riak-Last-Modified",87,101,100|...],
-                              [],[],[],[],[],...}}},
-                      ["bread"]}],
-          [{"20090722191020-riaktest@example.com-riakdemo@example.com-266664",
-            {1,63415509105}}],
-          {dict,0,16,16,8,80,48,
-                {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
-                {{[],[],[],[],[],[],[],[],[],[],[],[],[],...}}},
-          ["milk","bread"]}
+  (riaktest@example.com)6> %% add milk to the list
+  (riaktest@example.com)6> O2 = riak_object:update_value(O1, ["milk" | V]).
+  {r_object,<<"groceries">>,<<"mine">>,
+       [{r_content,{dict,2,16,16,8,80,48,
+                         {[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
+                         {{[],[],[],[],[],[],
+                           ["X-Riak-Last-Modified",87,101,100|...],
+                           [],[],[],[],[],...}}},
+                   ["bread"]}],
+       [{"20090722191020-riaktest@example.com-riakdemo@example.com-266664",
+         {1,63415509105}}],
+       {dict,0,16,16,8,80,48,
+             {[],[],[],[],[],[],[],[],[],[],[],[],[],[],[],...},
+             {{[],[],[],[],[],[],[],[],[],[],[],[],[],...}}},
+       ["milk","bread"]}
+  
+  (riaktest@example.com)7> %% store the new list
+  (riaktest@example.com)7> C:put(O2, 1).
+  ok
 
-     (riaktest@example.com)7> %% store the new list
-     (riaktest@example.com)7> C:put(O2, 1).
-     ok
 
      Finally, see what other keys are available in groceries bucket:
 
-     (riaktest@example.com)8> C:list_keys(<<"groceries">>).
-     {ok,[<<"mine">>]}
+  (riaktest@example.com)8> C:list_keys(<<"groceries">>).
+  {ok,[<<"mine">>]}
 
 
 
-     ["X-Riak-Last-Modified",87|...]: sec-2.3
-     ["X-Riak-Last-Modified",87,101,100|...]: sec-2.3
-
 3 Server Management 
 ~~~~~~~~~~~~~~~~~~~~
 
      This file contains the configuration for the Erlang applications
      that run on the Riak server.
 
+   More information about this files is available in doc/basic-setup.txt.
+
 3.2 Server Control 
 ===================
 
 
     To join a new Riak node to an existing cluster:
 
-    $ bin/riak start # If a local server is not already running
-    $ bin/riak-admin join <node in cluster>
+  $ bin/riak start # If a local server is not already running
+  $ bin/riak-admin join <node in cluster>
+
 
     (Note that you must have a local node already running for this to work)
     
      This file contains the configuration for the Erlang applications
      that run on the Riak server.
 
+   More information about this files is available in doc/basic-setup.txt.
+
 ** Server Control
 *** bin/riak
     This script is the primary interface for starting and stopping the Riak

apps/protobuffs/README.markdown

+# README
+
+This module is a composite of other open source modules and original code to
+make interfacing with the Protocol Buffers protocol easy.
+
+## Encode / Decode
+
+Encoding is simple.
+
+    1> protobuffs:encode(1, 1, uint32).
+    <<8,1>>
+    2> erlang:iolist_to_binary([
+        protobuffs:encode(1, <<"Nick">>, string),
+        protobuffs:encode(2, 25, uint32)
+    ]).
+    <<10,4,78,105,99,107,16,25>>
+
+Decoding is simple too.
+
+    1> protobuffs:decode(<<8, 1>>, uint32).
+    {{1, 1}, <<>>}
+    2> protobuffs:decode(<<10,4,78,105,99,107,16,25>>, bytes).
+    {{1, <<"Nick">>}, <<16,25>>}
+    3> protobuffs:decode(<<16,25>>, bytes).
+    {{2, 25}, <<>>}
+
+## Using .proto Files
+
+The main objective of this module is to allow developers to use .proto files
+easily. This module provides very basic functionality to do so.
+
+Consider the `t/simple.proto` file.
+
+    message Person {
+    	required string name = 1;
+    	required string address = 2;
+    	required string phone_number = 3;
+    	required int32 age = 4;
+    }
+
+From that file we can create an Erlang module that can encode and decode the
+Person message into records.
+
+    1> protobuffs_compile:scan_file("simple.proto").
+    ok
+	2> simple_pb:decode_person(<<10,4,78,105,99,107,18,13,77,111,...>>).
+	{person,<<"Nick">>,<<"Mountain View">>, <<"+1 (000) 555-1234">>,25}
+	3> simple_pb:encode_person({person, <<"Nick">>, <<"Mountain View">>,
+	    <<"+1 (000) 555-1234">>,25}).
+	<<10,4,78,105,99,107,18,13,77,111,117,110,116,97,105,110,32,86,105,...>>
+
+How cool is that? From .proto files, we create modules that export encode and
+decode functions for the messages defined.
+
+## CAVEATS
+
+Support for parsing proto files and creating code from it is volatile and
+should be considered alpha software at best. It currently only supports flat
+messages, simple types (ints, strings, etc) and will break on ENUM types and
+any sort of nesting. Please do not use this in production.
+
+## no_debug_info
+
+The protobuffs_compile module relies on the pokemon_pb module being compiled
+with debug info. This is because pokemon_pb serves as a template for generated
+_pb modules. Running protobuffs_compile:scan_file/1 reads the erlang forms from
+the pokemon_pb.beam file and expands and alters those forms to create the generated
+module.
+
+## CREDITS
+
+Some of the protobuffs.erl module came from code written by Brian Buchanan.
+
+Some of the protobuffs\_compile.erl module came from code written by Tim
+Fletcher.
+
+The rest of it and it's test suite was written by Nick Gerakines. Major
+contributions have been made by Jacob Vorreuter.

apps/protobuffs/ebin/protobuffs.app

+{application, protobuffs,
+ [
+    {description, "Google protobuffs implementation for Erlang."},
+    {vsn, "4"},
+    {modules, [
+		pokemon_pb,
+		protobuffs,
+		protobuffs_compile,
+		protobuffs_parser
+    ]},
+    {registered, []},
+    {applications, [
+                    kernel,
+                    stdlib,
+                    sasl,
+                    crypto
+                   ]}
+]}.

apps/protobuffs/rebar

Binary file added.

apps/protobuffs/rebar.config

+{erl_opts, [debug_info]}.

apps/protobuffs/scripts/mysql.escript

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -sasl errlog_type error -boot start_sasl -noshell
+
+%% ====================================================
+%% Generate proto message from mysql table definition
+%% $> escript mysql.escript host port user pwd db table
+%% ====================================================
+
+main([Host, Port0, User, Password, Database, Table]) ->
+	ok = crypto:start(),
+	Greeting_Fun =
+		fun(_M,_L,_E,_F) -> 
+			%{Format,Args} = F(),
+			%io:format("[~p:~b] ~p~n", [M,L,E]),
+			%io:format(Format ++ "~n", Args),
+			ok
+		end,
+		
+	Port = list_to_integer(Port0),
+	{ok, Pid} = mysql_conn:start(Host, Port, User, Password, Database, Greeting_Fun, undefined, undefined),
+
+	[C|Rest] = Table,
+	MessageName = [string:to_upper(C)|Rest],
+	case mysql_conn:fetch(Pid, list_to_binary(lists:flatten(io_lib:format("DESC ~s", [Table]))), self()) of 
+        {data, MySQLRes} ->
+            Rows = mysql:get_result_rows(MySQLRes),
+			io:format("message ~s {~n", [MessageName]),
+			print_row(Rows, 1),
+			io:format("}~n");
+		[] -> 
+			[];
+		Err -> 
+			error_logger:error_msg("~p~n", [Err])
+    end,
+
+	ok;
+	
+main(_Args) ->
+	io:format("usage: escript mysql.escript host port user password database table~n"),
+	io:format("args: ~p~n", [_Args]).
+	
+print_row([], _) -> ok;
+print_row([[Field, Type, Null, _Key, Default, _Extra]|Tail], Number) ->
+	Null1 = null(Null),
+	Type1 = type(binary_to_list(Type)),
+	Default1 = default(Type1, Default),
+	io:format("    ~s ~s ~s = ~w~s;~n", [Null1, Type1, Field, Number, Default1]),
+
+	print_row(Tail, Number+1).
+		
+null(<<"YES">>) -> "optional";
+null(<<"NO">>) -> "required".
+
+type("tinyint" ++ _) -> "int32";
+type("bigint" ++ _) -> "int64";
+type("serial" ++ _) -> "int64";
+type("mediumint" ++ _) -> "int32";
+type("int" ++ _) -> "int32";
+type("datetime" ++ _) -> "int32";
+type("float" ++ _) -> "float";
+type("double" ++ _) -> "double";
+type("decimal" ++ _) -> "double";
+type("varchar" ++ _) -> "string";
+type("char" ++ _) -> "string";
+type("text" ++ _) -> "string";
+type(Type) -> Type.
+
+
+default(_, <<"NULL">>) -> "";
+default(_, undefined) -> "";
+default("int" ++ _, Default) -> lists:flatten(io_lib:format(" [default = ~w]", [list_to_integer(binary_to_list(Default))]));
+default(_, Default) -> lists:flatten(io_lib:format(" [default = ~p]", [binary_to_list(Default)])).

apps/protobuffs/src/Makefile

+include ../support/include.mk
+
+all: $(EBIN_FILES)
+
+debug:
+	$(MAKE) DEBUG=-DDEBUG
+
+clean:
+	rm -rf $(EBIN_FILES) erl_crash.dump

apps/protobuffs/src/overview.edoc

Empty file added.

apps/protobuffs/src/pokemon_pb.erl

+%% Copyright (c) 2009 
+%% Nick Gerakines <nick@gerakines.net>
+%% Jacob Vorreuter <jacob.vorreuter@gmail.com>
+%%
+%% Permission is hereby granted, free of charge, to any person
+%% obtaining a copy of this software and associated documentation
+%% files (the "Software"), to deal in the Software without
+%% restriction, including without limitation the rights to use,
+%% copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the
+%% Software is furnished to do so, subject to the following
+%% conditions:
+%%
+%% The above copyright notice and this permission notice shall be
+%% included in all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+%% OTHER DEALINGS IN THE SOFTWARE.
+-module(pokemon_pb).
+-export([encode_pikachu/1, decode_pikachu/1]).
+-compile([export_all]).
+-record(pikachu, {abc, def}).
+
+%% ENCODE
+encode_pikachu(Record) when is_record(Record, pikachu) ->
+    encode(pikachu, Record).
+
+encode(pikachu, Record) ->
+    iolist_to_binary(iolist(pikachu, Record)).
+
+iolist(pikachu, Record) ->
+    [pack(1, required, with_default(Record#pikachu.abc, none), string, [])].
+
+with_default(undefined, none) -> undefined;
+with_default(undefined, Default) -> Default;
+with_default(Val, _) -> Val.
+
+pack(_, optional, undefined, _, _) -> [];
+
+pack(_, repeated, undefined, _, _) -> [];
+    
+pack(FNum, required, undefined, Type, _) ->
+    exit({error, {required_field_is_undefined, FNum, Type}});
+
+pack(_, repeated, [], _, Acc) -> 
+    lists:reverse(Acc);
+
+pack(FNum, repeated, [Head|Tail], Type, Acc) ->
+    pack(FNum, repeated, Tail, Type, [pack(FNum, optional, Head, Type, [])|Acc]);
+
+pack(FNum, _, Data, _, _) when is_tuple(Data) ->
+    [RecName|_] = tuple_to_list(Data),
+    protobuffs:encode(FNum, encode(RecName, Data), bytes);
+
+pack(FNum, _, Data, Type, _) ->
+    protobuffs:encode(FNum, Data, Type).
+
+%% DECODE
+decode_pikachu(Bytes) when is_binary(Bytes) ->
+    decode(pikachu, Bytes).
+    
+decode(pikachu, Bytes) when is_binary(Bytes) ->
+    Types = [{1, abc, int32, []}, {2, def, double, []}],
+    Decoded = decode(Bytes, Types, []),
+    to_record(pikachu, Decoded).
+    
+decode(<<>>, _, Acc) -> Acc;
+decode(Bytes, Types, Acc) ->
+    {{FNum, WireType}, Rest} = protobuffs:read_field_num_and_wire_type(Bytes),
+    case lists:keysearch(FNum, 1, Types) of
+        {value, {FNum, Name, Type, Opts}} ->
+            {Value1, Rest1} = 
+                case lists:member(is_record, Opts) of
+                    true ->
+                        {V, R} = protobuffs:decode_value(Rest, WireType, bytes),
+                        RecVal = decode(list_to_atom(string:to_lower(atom_to_list(Type))), V),
+                        {RecVal, R};
+                    false ->
+                        {V, R} = protobuffs:decode_value(Rest, WireType, Type),
+                        {unpack_value(V, Type), R}
+                end,
+            case lists:member(repeated, Opts) of
+                true ->
+                    case lists:keytake(FNum, 1, Acc) of
+                        {value, {FNum, Name, List}, Acc1} ->
+                            decode(Rest1, Types, [{FNum, Name, lists:reverse([Value1|lists:reverse(List)])}|Acc1]);
+                        false ->
+                            decode(Rest1, Types, [{FNum, Name, [Value1]}|Acc])
+                    end;
+                false ->
+                    decode(Rest1, Types, [{FNum, Name, Value1}|Acc])
+            end;
+        false ->
+            exit({error, {unexpected_field_index, FNum}})
+    end.
+    
+unpack_value(Binary, string) when is_binary(Binary) ->
+    binary_to_list(Binary);
+unpack_value(Value, _) -> Value.
+    
+to_record(pikachu, DecodedTuples) ->
+    lists:foldl(
+        fun({_FNum, Name, Val}, Record) ->
+            set_record_field(record_info(fields, pikachu), Record, Name, Val)
+        end, #pikachu{}, DecodedTuples).
+        
+set_record_field(Fields, Record, Field, Value) ->
+    Index = list_index(Field, Fields),
+    erlang:setelement(Index+1, Record, Value).
+    
+list_index(Target, List) -> list_index(Target, List, 1).
+
+list_index(Target, [Target|_], Index) -> Index;
+list_index(Target, [_|Tail], Index) -> list_index(Target, Tail, Index+1);
+list_index(_, [], _) -> 0.

apps/protobuffs/src/protobuffs.erl

+%% Copyright (c) 2009 
+%% Nick Gerakines <nick@gerakines.net>
+%% Jacob Vorreuter <jacob.vorreuter@gmail.com>
+%%
+%% Permission is hereby granted, free of charge, to any person
+%% obtaining a copy of this software and associated documentation
+%% files (the "Software"), to deal in the Software without
+%% restriction, including without limitation the rights to use,
+%% copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the
+%% Software is furnished to do so, subject to the following
+%% conditions:
+%%
+%% The above copyright notice and this permission notice shall be
+%% included in all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+%% OTHER DEALINGS IN THE SOFTWARE.
+%%
+%% @doc A protcol buffers encoding and decoding module.
+-module(protobuffs).
+-export([encode/3, read_field_num_and_wire_type/1, decode/2, decode_value/3]).
+-compile(export_all).
+
+-define(TYPE_VARINT, 0).
+-define(TYPE_64BIT, 1).
+-define(TYPE_STRING, 2).
+-define(TYPE_START_GROUP, 3).
+-define(TYPE_END_GROUP, 4).
+-define(TYPE_32BIT, 5).
+
+%% @spec encode(FieldID, Value, Type) -> Result
+%%       FieldID = integer()
+%%       Value = any()
+%%       Type = bool | enum | int32 | uint32 | int64 | unit64 | sint32 | sint64 | fixed32 | sfixed32 | fixed64 | sfixed64 | string | bytes | float | double 
+%%       Result = list()
+%% @doc Encode an Erlang data structure into a Protocol Buffers value.
+encode(FieldID, Value, Type) ->
+    iolist_to_binary(encode_internal(FieldID, Value, Type)).
+    
+%% @hidden
+encode_internal(FieldID, false, bool) ->
+    encode_internal(FieldID, 0, int32);
+encode_internal(FieldID, true, bool) ->
+    encode_internal(FieldID, 1, int32);
+encode_internal(FieldID, Integer, enum) ->
+    encode_internal(FieldID, Integer, uint32);
+encode_internal(FieldID, Integer, int32) when Integer >= -16#80000000, Integer < 0 ->
+    encode_internal(FieldID, Integer, int64);
+encode_internal(FieldID, Integer, int64) when Integer >= -16#8000000000000000, Integer < 0 ->
+    encode_internal(FieldID, Integer + (1 bsl 64), uint64);
+encode_internal(FieldID, Integer, int32) when Integer >= -16#80000000, Integer =< 16#7fffffff ->
+    encode_varint_field(FieldID, Integer);
+encode_internal(FieldID, Integer, uint32) when Integer band 16#ffffffff =:= Integer ->
+    encode_varint_field(FieldID, Integer);    
+encode_internal(FieldID, Integer, int64) when Integer >= -16#8000000000000000, Integer =< 16#7fffffffffffffff ->
+    encode_varint_field(FieldID, Integer);
+encode_internal(FieldID, Integer, uint64) when Integer band 16#ffffffffffffffff =:= Integer ->
+    encode_varint_field(FieldID, Integer);
+encode_internal(FieldID, Integer, bool) when Integer band 1 =:= 1 ->
+    encode_varint_field(FieldID, Integer);
+encode_internal(FieldID, Integer, sint32) when Integer >= -16#80000000, Integer < 0 ->
+    encode_varint_field(FieldID, bnot (Integer bsl 1));
+encode_internal(FieldID, Integer, sint64) when Integer >= -16#8000000000000000, Integer < 0 ->
+    encode_varint_field(FieldID, bnot (Integer bsl 1));
+encode_internal(FieldID, Integer, sint32) when Integer >= 0, Integer =< 16#7fffffff ->
+    encode_varint_field(FieldID, Integer bsl 1);
+encode_internal(FieldID, Integer, sint64) when Integer >= 0, Integer =< 16#7fffffffffffffff ->
+    encode_varint_field(FieldID, Integer bsl 1);
+encode_internal(FieldID, Integer, fixed32) when Integer band 16#ffffffff =:= Integer ->
+    [encode_field_tag(FieldID, ?TYPE_32BIT), <<Integer:32/little-integer>>];
+encode_internal(FieldID, Integer, sfixed32) when Integer >= -16#80000000, Integer =< 16#7fffffff ->
+    [encode_field_tag(FieldID, ?TYPE_32BIT), <<Integer:32/little-integer>>];
+encode_internal(FieldID, Integer, fixed64) when Integer band 16#ffffffffffffffff =:= Integer ->
+    [encode_field_tag(FieldID, ?TYPE_64BIT), <<Integer:64/little-integer>>];
+encode_internal(FieldID, Integer, sfixed64) when Integer >= -16#8000000000000000, Integer =< 16#7fffffffffffffff ->
+    [encode_field_tag(FieldID, ?TYPE_64BIT), <<Integer:64/little-integer>>];
+encode_internal(FieldID, String, string) when is_list(String) ->
+    encode_internal(FieldID, list_to_binary(String), string);
+encode_internal(FieldID, String, string) when is_binary(String) ->
+    encode_internal(FieldID, String, bytes);
+encode_internal(FieldID, String, bytes) when is_list(String) ->
+    encode_internal(FieldID, list_to_binary(String), bytes);
+encode_internal(FieldID, Bytes, bytes) when is_binary(Bytes) ->
+    [encode_field_tag(FieldID, ?TYPE_STRING), encode_varint(size(Bytes)), Bytes];
+encode_internal(FieldID, String, bytes) when is_list(String) ->
+    encode_internal(FieldID, list_to_binary(String), bytes);
+encode_internal(FieldID, Float, float) when is_integer(Float) ->
+    encode_internal(FieldID, Float + 0.0, float);
+encode_internal(FieldID, Float, float) when is_float(Float) ->
+    [encode_field_tag(FieldID, ?TYPE_32BIT), <<Float:32/little-float>>];
+encode_internal(FieldID, Float, double) when is_integer(Float) ->
+    encode_internal(FieldID, Float + 0.0, double);
+encode_internal(FieldID, Float, double) when is_float(Float) ->
+    [encode_field_tag(FieldID, ?TYPE_64BIT), <<Float:64/little-float>>].
+
+read_field_num_and_wire_type(Bytes) ->
+    {Tag, Rest} = decode_varint(Bytes),
+    FieldID = Tag bsr 3,
+    WireType = Tag band 7,
+    {{FieldID, WireType}, Rest}.
+    
+%% @spec decode(Bytes, ExpectedType) -> Result
+%%       Bytes = binary()
+%%       ExpectedType = bool | enum | int32 | uint32 | int64 | unit64 | sint32 | sint64 | fixed32 | sfixed32 | fixed64 | sfixed64 | string | bytes | float | double 
+%%       Result = {{integer(), any()}, binary()}
+decode(Bytes, ExpectedType) ->
+    {{FieldID, WireType}, Rest} = read_field_num_and_wire_type(Bytes),
+    {Value, Rest1} = decode_value(Rest, WireType, ExpectedType),
+    {{FieldID, Value}, Rest1}.
+
+%% @hidden
+decode_value(Bytes, ?TYPE_VARINT, ExpectedType) ->
+    {Value, Rest} = decode_varint(Bytes),
+    {typecast(Value, ExpectedType), Rest};
+decode_value(Bytes, ?TYPE_STRING, ExpectedType) when ExpectedType =:= string; ExpectedType =:= bytes ->
+    {Length, Rest} = decode_varint(Bytes),
+    split_binary(Rest, Length);
+decode_value(<<Value:64/little-unsigned-integer, Rest/binary>>, ?TYPE_64BIT, fixed64) ->
+    {Value, Rest};
+decode_value(<<Value:32/little-unsigned-integer, _:32, Rest/binary>>, ?TYPE_64BIT, fixed32) ->
+    {Value, Rest};
+decode_value(<<Value:64/little-signed-integer, Rest/binary>>, ?TYPE_64BIT, sfixed64) ->
+    {Value, Rest};
+decode_value(<<Value:32/little-signed-integer, _:32, Rest/binary>>, ?TYPE_64BIT, sfixed32) ->
+    {Value, Rest};
+decode_value(<<Value:32/little-unsigned-integer, Rest/binary>>, ?TYPE_32BIT, Type) when Type =:= fixed32; Type =:= fixed64 ->
+    {Value, Rest};
+decode_value(<<Value:32/little-signed-integer, Rest/binary>>, ?TYPE_32BIT, Type) when Type =:= sfixed32; Type =:= sfixed64 ->
+    {Value, Rest};
+decode_value(<<Value:32/little-float, Rest/binary>>, ?TYPE_32BIT, float) ->
+    {Value + 0.0, Rest};
+decode_value(<<Value:64/little-float, Rest/binary>>, ?TYPE_64BIT, double) ->
+    {Value + 0.0, Rest};
+decode_value(_, WireType, ExpectedType) ->
+    exit({error, {unexpected_value, WireType, ExpectedType}}).
+
+%% @hidden
+typecast(Value, SignedType) when SignedType =:= int32; SignedType =:= int64 ->
+    if
+        Value band 16#8000000000000000 =/= 0 -> Value - 16#10000000000000000;
+        true -> Value
+    end;
+typecast(Value, SignedType) when SignedType =:= sint32; SignedType =:= sint64 ->
+    (Value bsr 1) bxor (-(Value band 1));
+typecast(Value, _) ->
+    Value.
+
+%% @hidden
+encode_field_tag(FieldID, FieldType) when FieldID band 16#3fffffff =:= FieldID ->
+    encode_varint((FieldID bsl 3) bor FieldType).
+
+%% @hidden
+encode_varint_field(FieldID, Integer) ->
+    [encode_field_tag(FieldID, ?TYPE_VARINT), encode_varint(Integer)].
+
+%% @hidden
+encode_varint(I) ->
+    encode_varint(I, []).
+
+%% @hidden
+encode_varint(I, Acc) when I =< 16#7f ->
+    iolist_to_binary(lists:reverse([I | Acc]));
+encode_varint(I, Acc) ->
+    Last_Seven_Bits = (I - ((I bsr 7) bsl 7)),
+    First_X_Bits = (I bsr 7),
+    With_Leading_Bit = Last_Seven_Bits bor 16#80,
+    encode_varint(First_X_Bits, [With_Leading_Bit|Acc]).
+
+%% @hidden
+decode_varint(Bytes) ->
+    decode_varint(Bytes, []).
+decode_varint(<<0:1, I:7, Rest/binary>>, Acc) ->
+    Acc1 = [I|Acc],
+    Result = 
+        lists:foldl(
+            fun(X, Acc0) ->
+                (Acc0 bsl 7 bor X)
+            end, 0, Acc1),
+    {Result, Rest};
+decode_varint(<<1:1, I:7, Rest/binary>>, Acc) ->
+    decode_varint(Rest, [I | Acc]).

apps/protobuffs/src/protobuffs_compile.erl

+%% Copyright (c) 2009 
+%% Nick Gerakines <nick@gerakines.net>
+%% Jacob Vorreuter <jacob.vorreuter@gmail.com>
+%%
+%% Permission is hereby granted, free of charge, to any person
+%% obtaining a copy of this software and associated documentation
+%% files (the "Software"), to deal in the Software without
+%% restriction, including without limitation the rights to use,
+%% copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the
+%% Software is furnished to do so, subject to the following
+%% conditions:
+%%
+%% The above copyright notice and this permission notice shall be
+%% included in all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+%% OTHER DEALINGS IN THE SOFTWARE.
+-module(protobuffs_compile).
+-export([scan_file/1, output/2]).
+
+scan_file(ProtoFile) when is_list(ProtoFile) ->
+    Basename = filename:basename(ProtoFile, ".proto") ++ "_pb",
+    Parsed = protobuffs_parser:parse_file(ProtoFile),
+    Messages = collect_full_messages(Parsed),
+    output(Basename, Messages).
+    
+output(Basename, Messages) ->
+    ok = write_header_include_file(Basename, Messages),
+    BeamFile = filename:dirname(code:which(?MODULE)) ++ "/pokemon_pb.beam",
+    {ok,{_,[{abstract_code,{_,Forms}}]}} = beam_lib:chunks(BeamFile, [abstract_code]),
+    Forms1 = filter_forms(Messages, Forms, Basename, []),
+    {ok, _, Bytes, _} = compile:forms(Forms1, [return]),
+    file:write_file(Basename ++ ".beam", Bytes).
+
+filter_forms(Msgs, [{attribute,L,file,{_,_}}|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [{attribute,L,file,{"src/" ++ Basename ++ ".erl",L}}|Acc]);
+    
+filter_forms(Msgs, [{attribute,L,module,pokemon_pb}|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [{attribute,L,module,list_to_atom(Basename)}|Acc]);
+
+filter_forms(Msgs, [{attribute,L,export,[{encode_pikachu,1},{decode_pikachu,1}]}|Tail], Basename, Acc) ->
+    Exports = lists:foldl(
+        fun({Name,_}, Acc1) ->
+            [{list_to_atom("encode_" ++ string:to_lower(Name)),1},
+             {list_to_atom("decode_" ++ string:to_lower(Name)),1} | Acc1]
+        end, [], Msgs),
+    filter_forms(Msgs, Tail, Basename, [{attribute,L,export,Exports}|Acc]);
+
+filter_forms(Msgs, [{attribute,L,record,{pikachu,_}}|Tail], Basename, Acc) ->
+    Records = [begin
+        OutFields = [string:to_lower(A) || {_, _, _, A, _, _} <- lists:keysort(1, Fields)],
+        Frm_Fields = [{record_field,L,{atom,L,list_to_atom(OutField)}}|| OutField <- OutFields],
+        {attribute, L, record, {atomize(Name), Frm_Fields}}
+     end || {Name, Fields} <- Msgs],
+    filter_forms(Msgs, Tail, Basename, Records ++ Acc);
+
+filter_forms(Msgs, [{function,L,encode_pikachu,1,[Clause]}|Tail], Basename, Acc) ->
+    Functions = [begin
+        {function,L,list_to_atom("encode_" ++ string:to_lower(Name)),1,[replace_atom(Clause, pikachu, atomize(Name))]} 
+    end || {Name, _} <- Msgs],
+    filter_forms(Msgs, Tail, Basename, Functions ++ Acc);
+
+filter_forms(Msgs, [{function,L,encode,2,[Clause]}|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [expand_encode_function(Msgs, L, Clause)|Acc]);
+
+filter_forms(Msgs, [{function,L,iolist,2,[Clause]}|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [expand_iolist_function(Msgs, L, Clause)|Acc]);
+        
+filter_forms(Msgs, [{function,L,decode_pikachu,1,[Clause]}|Tail], Basename, Acc) ->
+    Functions = [begin
+        {function,L,list_to_atom("decode_" ++ string:to_lower(Name)),1,[replace_atom(Clause, pikachu, atomize(Name))]} 
+    end || {Name, _} <- Msgs],
+    filter_forms(Msgs, Tail, Basename, Functions ++ Acc);
+    
+filter_forms(Msgs, [{function,L,decode,2,[Clause]}|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [expand_decode_function(Msgs, L, Clause)|Acc]);
+    
+filter_forms(Msgs, [{function,L,to_record,2,[Clause]}|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [expand_to_record_function(Msgs, L, Clause)|Acc]);
+
+filter_forms(Msgs, [Form|Tail], Basename, Acc) ->
+    filter_forms(Msgs, Tail, Basename, [Form|Acc]);
+
+filter_forms(_, [], _, Acc) -> lists:reverse(Acc).
+
+expand_encode_function(Msgs, Line, Clause) ->
+    {function,Line,encode,2,[filter_encode_clause(Msg, Clause) || Msg <- Msgs]}.
+    
+filter_encode_clause({MsgName, Fields}, {clause,L,_Args,Guards,_Content}) ->
+    ToBin = {call,L,{atom,L,iolist_to_binary},[{call,L,
+                                                {atom,L,iolist},
+                                                [{atom,L,atomize(MsgName)},{var,L,'Record'}]}]},
+    {clause,L,[{atom,L,atomize(MsgName)},{var,L,'Record'}],Guards,[ToBin]}.
+
+expand_iolist_function(Msgs, Line, Clause) ->
+    {function,Line,iolist,2,[filter_iolist_clause(Msg, Clause) || Msg <- Msgs]}.
+
+filter_iolist_clause({MsgName, Fields}, {clause,L,_Args,Guards,_Content}) ->
+    Cons = lists:foldl(
+        fun({FNum,Tag,SType,SName,_,Default}, Acc) ->
+            {cons,L,
+                {call,L,{atom,L,pack},[
+                    {integer,L,FNum},
+                    {atom,L,Tag},
+                    {call,L,{atom,L,with_default},[{record_field,L,{var,L,'Record'},atomize(MsgName),{atom,L,atomize(SName)}},erl_parse:abstract(Default)]},
+                    {atom,L,atomize(SType)},
+                    {nil,L}]},
+                Acc}
+        end, {nil,L}, Fields),
+    {clause,L,[{atom,L,atomize(MsgName)},{var,L,'Record'}],Guards,[Cons]}.
+    
+expand_decode_function(Msgs, Line, Clause) ->
+    {function,Line,decode,2,[filter_decode_clause(Msgs, Msg, Clause) || Msg <- Msgs]}.
+    
+filter_decode_clause(Msgs, {MsgName, Fields}, {clause,L,_Args,Guards,[_,B,C]}) ->
+    Types = lists:keysort(1, [{FNum, list_to_atom(SName), list_to_atom(SType), decode_opts(Msgs, Tag, SType)} || {FNum,Tag,SType,SName,_,_} <- Fields]),
+    Cons = lists:foldl(
+        fun({FNum, FName, Type, Opts}, Acc) ->
+            {cons,L,{tuple,L,[{integer,L,FNum},{atom,L,FName},{atom,L,Type},erl_parse:abstract(Opts)]},Acc}
+        end, {nil,L}, Types),
+    A = {match,L,{var,L,'Types'},Cons},
+    C1 = replace_atom(C, pikachu, atomize(MsgName)),
+    {clause,L,[{atom,L,atomize(MsgName)},{var,L,'Bytes'}],Guards,[A,B,C1]}.
+    
+decode_opts(Msgs, Tag, Type) ->
+    Opts0 = if Tag == repeated -> [repeated]; true -> [] end,
+    case lists:keymember(Type, 1, Msgs) of
+        true ->
+            [is_record|Opts0];
+        false ->
+            Opts0
+    end.
+    
+expand_to_record_function(Msgs, Line, Clause) ->
+    {function,Line,to_record,2,[filter_to_record_clause(Msg, Clause) || Msg <- Msgs]}.
+
+filter_to_record_clause({MsgName, _}, {clause,L,[_Param1,Param2],Guards,[Fold]}) ->
+    Fold1 = replace_atom(Fold, pikachu, atomize(MsgName)),
+    {clause,L,[{atom,L,atomize(MsgName)},Param2],Guards,[Fold1]}.
+
+%% [{"Location",
+%%   [{2,required,"string","country",number,none},
+%%    {1,required,"string","region",number,none}]},
+%%  {"Person",
+%%   [{5,optional,"Location","location",number,none},
+%%    {4,required,"int32","age",number,none},
+%%    {3,required,"string","phone_number",number,none},
+%%    {2,required,"string","address",number,none},
+%%    {1,required,"string","name",number,none}]}]
+collect_full_messages(Data) -> collect_full_messages(Data, []).
+collect_full_messages([{message, Name, Fields} | Tail], Acc) ->
+    FieldsOut = lists:foldl(
+        fun (Input, TmpAcc) ->
+            case Input of
+                {_, _, _, _, _, _} ->  [Input | TmpAcc];
+                _ -> TmpAcc
+            end
+        end, [], Fields),
+    SubMessages = lists:foldl(
+        fun ({message, C, D}, TmpAcc) -> [{message, C, D} | TmpAcc];
+            (_, TmpAcc) -> TmpAcc
+        end, [], Fields),
+    collect_full_messages(Tail ++ SubMessages, [{Name, FieldsOut} | Acc]);
+collect_full_messages([{package, _Line1}, {bareword, _Line2, _PackageName}, {';', _Line3} | Tail], Acc) ->
+    collect_full_messages(Tail, Acc);
+collect_full_messages([], Acc) -> 
+    Acc.
+
+write_header_include_file(Basename, Messages) ->	
+    {ok, FileRef} = file:open(Basename ++ ".hrl", [write]),
+    [begin
+        OutFields = [string:to_lower(A) || {_, _, _, A, _, _} <- lists:keysort(1, Fields)],
+        io:format(FileRef, "-record(~s, {~s}).~n", [string:to_lower(Name), string:join(OutFields, ", ")])
+    end || {Name, Fields} <- Messages],
+    file:close(FileRef).
+    
+atomize(String) ->
+    list_to_atom(string:to_lower(String)).
+    
+replace_atom(Find, Find, Replace) -> Replace;
+    
+replace_atom(Tuple, Find, Replace) when is_tuple(Tuple) ->
+    list_to_tuple([replace_atom(Term, Find, Replace) || Term <- tuple_to_list(Tuple)]);
+
+replace_atom(List, Find, Replace) when is_list(List) ->
+    [replace_atom(Term, Find, Replace) || Term <- List];
+    
+replace_atom(Other, _Find, _Replace) ->
+    Other.
+    

apps/protobuffs/src/protobuffs_parser.erl

+%% Copyright (c) 2009 
+%% Nick Gerakines <nick@gerakines.net>
+%% Jacob Vorreuter <jacob.vorreuter@gmail.com>
+%%
+%% Permission is hereby granted, free of charge, to any person
+%% obtaining a copy of this software and associated documentation
+%% files (the "Software"), to deal in the Software without
+%% restriction, including without limitation the rights to use,
+%% copy, modify, merge, publish, distribute, sublicense, and/or sell
+%% copies of the Software, and to permit persons to whom the
+%% Software is furnished to do so, subject to the following
+%% conditions:
+%%
+%% The above copyright notice and this permission notice shall be
+%% included in all copies or substantial portions of the Software.
+%%
+%% THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+%% EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+%% OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+%% NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
+%% HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
+%% WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+%% FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
+%% OTHER DEALINGS IN THE SOFTWARE.
+-module(protobuffs_parser).
+
+-export([parse_file/1]).
+
+parse_file(Filename) ->
+    {ok, Data} = file:read_file(Filename),
+    Raw = scan(binary_to_list(Data)),
+    parse(Raw).
+
+%% @hidden
+parse(Data) -> parse(Data, []).
+
+%% @hidden
+parse([], Acc) -> lists:reverse(Acc);
+parse([{'}', _Line} | Tail], Acc) -> {Acc, Tail};
+parse([{enum, _Line}, {bareword, _Line, MessageName}, {'{', _Line} | Tail], Acc) ->
+    {Res, Tail2} = parse(Tail, []),
+    parse(Tail2, [{enum, MessageName, lists:reverse(Res)} | Acc]);
+parse([{message, _Line}, {bareword, _Line, MessageName}, {'{', _Line} | Tail], Acc) ->
+    {Res, Tail2} = parse(Tail, []),
+    parse(Tail2, [{message, MessageName, lists:reverse(Res)} | Acc]);
+parse([{bareword, _Line, FieldName}, {'=', _Line}, {number, _Line, Value}, {';', _Line} | Tail], Acc) ->
+    parse(Tail, [{enum, Value, FieldName} | Acc]);
+parse([{Type, _Line}, {bareword, _Line, Field}, {bareword, _Line, FieldName}, {'=', _Line}, {FieldType, _Line, Position}, {'[', _Line}, {bareword, _Line,"default"}, {'=', _Line}, {_DefaultType, _Line, Default}, {']', _Line}, {';', _Line} | Tail], Acc) ->
+    parse(Tail, [{Position, Type, Field, FieldName, FieldType, Default} | Acc]);
+parse([{Type, _Line}, {bareword, _Line, Field}, {bareword, _Line, FieldName}, {'=', _Line}, {FieldType, _Line, Position}, {';', _Line} | Tail], Acc) ->
+    parse(Tail, [{Position, Type, Field, FieldName, FieldType, none} | Acc]);
+parse([{'$end', _} | Tail], Acc) ->
+    parse(Tail, Acc);
+parse([Head | Tail], Acc) ->
+    parse(Tail, [Head | Acc]).
+
+scan(String) ->
+    scan(String, [], 1).
+
+%% @hidden
+scan([${|Rest], Accum, Line) ->
+    scan(Rest, [{'{', Line}|Accum], Line);
+scan([$}|Rest], Accum, Line) ->
+    scan(Rest, [{'}', Line}|Accum], Line);
+scan([$[|Rest], Accum, Line) ->
+    scan(Rest, [{'[', Line}|Accum], Line);
+scan([$]|Rest], Accum, Line) ->
+    scan(Rest, [{']', Line}|Accum], Line);
+scan([$(|Rest], Accum, Line) ->
+    scan(Rest, [{'(', Line}|Accum], Line);
+scan([$)|Rest], Accum, Line) ->
+    scan(Rest, [{')', Line}|Accum], Line);
+scan([$=|Rest], Accum, Line) ->
+    scan(Rest, [{'=', Line}|Accum], Line);
+scan([$;|Rest], Accum, Line) ->
+    scan(Rest, [{';', Line}|Accum], Line);
+scan([$,|Rest], Accum, Line) ->
+    scan(Rest, [{',', Line}|Accum], Line);
+scan([Digit|_] = String, Accum, Line)
+  when Digit >= $0, Digit =< $9 ->
+    {Number, Rest} = scan_number(String),
+    scan(Rest, [{number, Line, Number}|Accum], Line);
+scan([$-, Digit|_] = String, Accum, Line)
+  when Digit >= $0, Digit =< $9 ->
+    {Number, Rest} = scan_number(tl(String)),
+    scan(Rest, [{number, Line, -Number}|Accum], Line);
+scan([$\n|Rest], Accum, Line) ->
+    scan(Rest, Accum, Line + 1);
+scan([WS|Rest], Accum, Line)
+  when WS =:= 32; WS =:= $\t ->
+    scan(Rest, Accum, Line);
+scan([$/, $/|Rest], Accum, Line) ->
+    scan(skip_to_newline(Rest), Accum, Line);
+scan([$/, $*|Rest], Accum, Line) ->
+    {Rest1, Line1} = skip_comment(Rest, Line),
+    scan(Rest1, Accum, Line1);
+scan([$"|_] = String, Accum, Line) ->
+    {Strval, Rest, Line1} = scan_string(String, Line),
+    scan(Rest, [{string, Line, Strval}|Accum], Line1);
+scan([C|_] = String, Accum, Line)
+  when C >= $A, C =< $Z;
+       C >= $a, C =< $z;
+       C =:= $_ ->
+    {Identifier, Rest} = scan_identifier(String),
+    Token = case get_keyword(Identifier) of
+        Keyword when is_atom(Keyword) ->
+            {Keyword, Line};
+        {bareword, Bareword} ->
+            {bareword, Line, Bareword}
+    end,
+    scan(Rest, [Token|Accum], Line);
+scan([], Accum, Line) ->
+    lists:reverse([{'$end', Line}|Accum]);
+scan([C|_], _Accum, Line) ->
+    erlang:error({invalid_character, [C], Line}).
+
+%% @hidden
+scan_identifier(String) ->
+    scan_identifier(String, "").
+
+%% @hidden
+scan_identifier([C|Rest], Accum)
+  when C >= $A, C =< $Z;
+       C >= $a, C =< $z;
+       C >= $0, C =< $9;
+       C =:= $_;
+       C =:= $. ->
+    scan_identifier(Rest, [C|Accum]);
+scan_identifier(Rest, Accum) ->
+    {lists:reverse(Accum), Rest}.
+
+%% @hidden
+scan_number(String) ->
+    {A, Rest1} = scan_integer(String),
+    case Rest1 of
+        [$.|Fraction] ->
+            {B, Rest2} = scan_identifier(Fraction),
+            {A + list_to_float("0." ++ B), Rest2};
+        [$e|Exp] ->
+            {B, Rest2} = scan_integer(Exp),
+            {list_to_float(integer_to_list(A) ++ ".0e" ++ integer_to_list(B)), Rest2};
+        [$x|Rest] when A =:= 0 ->
+            {Hex, Rest2} = scan_identifier(Rest),
+            {erlang:list_to_integer(Hex, 16), Rest2};
+        _ ->
+            {A, Rest1}
+    end.
+
+%% @hidden
+scan_integer(String) ->
+    scan_integer(String, 0).
+
+%% @hidden
+scan_integer([D|Rest], Accum)
+  when D >= $0, D =< $9 ->
+    scan_integer(Rest, Accum * 10 + (D - $0));
+scan_integer(Rest, Accum) ->
+    {Accum, Rest}.
+
+%% @hidden
+scan_string([$"|String], Line) ->
+    scan_string(String, "", Line).
+
+%% @hidden
+scan_string([$"|Rest], Accum, Line) ->
+    {lists:reverse(Accum), Rest, Line};
+scan_string([$\\, $a|Rest], Accum, Line) ->
+    scan_string(Rest, [7|Accum], Line);
+scan_string([$\\, $e|Rest], Accum, Line) ->
+    scan_string(Rest, [$\e|Accum], Line);
+scan_string([$\\, $f|Rest], Accum, Line) ->
+    scan_string(Rest, [$\f|Accum], Line);
+scan_string([$\\, $n|Rest], Accum, Line) ->
+    scan_string(Rest, [$\n|Accum], Line);
+scan_string([$\\, $r|Rest], Accum, Line) ->
+    scan_string(Rest, [$\r|Accum], Line);
+scan_string([$\\, $t|Rest], Accum, Line) ->
+    scan_string(Rest, [$\t|Accum], Line);
+scan_string([$\\, $v|Rest], Accum, Line) ->
+    scan_string(Rest, [$\v|Accum], Line);
+scan_string([$\\, D1, D2, D3|Rest], Accum, Line)
+  when D1 >= $0, D1 =< $7, D2 >= $0, D2 =< $7, D3 >= $0, D3 =< $7 ->
+    scan_string(Rest, [erlang:list_to_integer([D1, D2, D3], 8)|Accum], Line);
+scan_string([$\\, $x, H1, H2|Rest], Accum, Line) ->
+    scan_string(Rest, [erlang:list_to_integer([H1, H2], 16)|Accum], Line);
+scan_string([$\\, Char|Rest], Accum, Line) ->
+    scan_string(Rest, [Char|Accum], Line);
+scan_string([$\n|Rest], Accum, Line) ->
+    scan_string(Rest, [$\n|Accum], Line + 1);
+scan_string([Char|Rest], Accum, Line) ->
+    scan_string(Rest, [Char|Accum], Line).
+
+%% @hidden
+skip_to_newline([$\n|Rest]) ->
+    Rest;
+skip_to_newline([]) ->
+    [];
+skip_to_newline([_|Rest]) ->
+    skip_to_newline(Rest).
+
+%% @hidden
+skip_comment([$*, $/|Rest], Line) ->
+    {Rest, Line};
+skip_comment([$\n|Rest], Line) ->
+    skip_comment(Rest, Line + 1);
+skip_comment([_|Rest], Line) ->
+    skip_comment(Rest, Line).
+
+%% @hidden
+get_keyword("import") ->
+    import;
+get_keyword("package") ->
+    package;
+get_keyword("option") ->
+    option;
+get_keyword("message") ->
+    message;
+get_keyword("group") ->
+    group;
+get_keyword("enum") ->
+    enum;
+get_keyword("extend") ->
+    extend;
+get_keyword("service") ->
+    service;
+get_keyword("rpc") ->
+    rpc;
+get_keyword("required") ->
+    required;
+get_keyword("optional") ->
+    optional;
+get_keyword("repeated") ->
+    repeated;
+get_keyword("returns") ->
+    returns;
+get_keyword("extensions") ->
+    extensions;
+get_keyword("max") ->
+    max;
+get_keyword("to") ->
+    to;
+get_keyword("true") ->
+    true;
+get_keyword("false") ->
+    false;
+get_keyword(Bareword) ->
+    {bareword, Bareword}.

apps/protobuffs/support/include.mk

+## -*- makefile -*-
+
+ERL := erl
+ERLC := $(ERL)c
+
+INCLUDE_DIRS := ../include $(wildcard ../deps/*/include)
+EBIN_DIRS := $(wildcard ../deps/*/ebin)
+ERLC_FLAGS := +debug_info -W $(INCLUDE_DIRS:../%=-I ../%) $(EBIN_DIRS:%=-pa %)
+
+ifdef debug
+  ERLC_FLAGS += -Ddebug
+endif
+
+EBIN_DIR := ../ebin
+DOC_DIR  := ../doc
+EMULATOR := beam
+
+ERL_TEMPLATE := $(wildcard *.et)
+ERL_SOURCES  := $(wildcard *.erl)
+ERL_HEADERS  := $(wildcard *.hrl) $(wildcard ../include/*.hrl)
+ERL_OBJECTS  := $(ERL_SOURCES:%.erl=$(EBIN_DIR)/%.beam)
+ERL_TEMPLATES := $(ERL_TEMPLATE:%.et=$(EBIN_DIR)/%.beam)
+ERL_OBJECTS_LOCAL := $(ERL_SOURCES:%.erl=./%.$(EMULATOR))
+APP_FILES := $(wildcard *.app)
+EBIN_FILES = $(ERL_OBJECTS) $(APP_FILES:%.app=../ebin/%.app) $(ERL_TEMPLATES)
+MODULES = $(ERL_SOURCES:%.erl=%)
+
+../ebin/%.app: %.app
+	cp $< $@
+
+$(EBIN_DIR)/%.$(EMULATOR): %.erl
+	$(ERLC) $(ERLC_FLAGS) -o $(EBIN_DIR) $<
+
+$(EBIN_DIR)/%.$(EMULATOR): %.et
+	$(ERL) -noshell -pa ../../elib/erltl/ebin/ -eval "erltl:compile(atom_to_list('$<'), [{outdir, \"../ebin\"}, report_errors, report_warnings, nowarn_unused_vars])." -s init stop
+
+./%.$(EMULATOR): %.erl
+	$(ERLC) $(ERLC_FLAGS) -o . $<
+
+$(DOC_DIR)/%.html: %.erl
+	$(ERL) -noshell -run edoc file $< -run init stop
+	mv *.html $(DOC_DIR)
+

apps/protobuffs/t/Makefile

+include ../support/include.mk
+
+all: $(EBIN_FILES)
+
+clean:
+	rm -rf $(EBIN_FILES) erl_crash.dump simple_pb.*
+
+test: $(MODULES)
+
+./$(MODULES):
+	@echo "Running tests for $@"
+	erl -pa ../ebin -run $@ start -run init stop -noshell

apps/protobuffs/t/empty.proto

+message Empty {
+	optional int32 id = 1;
+	optional string gamername = 2;
+	optional string name = 3;
+	optional string roles = 4;
+	optional string location = 5;
+	optional int32 logged_in_at = 6;
+	optional int32 created_at = 7;
+	optional int32 updated_at = 8;
+	optional int32 deleted = 9;
+	optional string time_zone = 10;
+	optional string security_token = 11;
+	optional int32 token_expiry = 12;
+	optional int32 verified = 13;
+	optional int32 last_seen_when = 14;
+	optional string last_seen_game = 15;
+	optional int32 fan_nofity = 16;
+	optional int32 comment_notify = 17;
+	optional int32 delta = 18;
+	optional int32 show_welcome = 19;
+	optional string tagline = 20;
+	optional int32 last_closed_site_announcement_at = 21;
+	optional string last_seen_activity = 22;
+	optional int64 nucleus_id = 23;
+	optional string nucleus_email = 24;
+	optional int32 deleted_at = 25;
+	optional int32 last_closed_rupture_challenge_at = 26;
+	optional int32 message_notify = 27;
+	optional int32 comments_count = 28;
+	optional int32 challenge_issued_notify = 29;
+	optional int32 challenge_accepted_rejected_notify = 30;
+	optional string password = 31;
+	optional string date_of_birth = 32;
+	optional string country = 33;
+	optional string language = 34;
+	optional string terms_of_service_version = 35;
+	optional string status = 36;
+	optional bool global_optin = 37;
+	optional bool third_party_optin = 38;
+}

apps/protobuffs/t/hasdefault.proto

+
+message Person {
+	required string name = 1;
+	required string address = 2;
+	required string phone_number = 3 [default = "+1 (000) 000-0000"];
+	required int32 age = 4 [default = 25];
+	optional Location location = 5;
+}
+
+message Location {
+    required string region = 1;
+    required string country = 2;
+}

apps/protobuffs/t/protobuffs_eqc.erl

+%%% erlc -o ebin t/*.erl -pa ebin
+%%% erl -name eqc_pb -pa ebin
+%%% eqc_gen:sample(protobuffs_eqc:protobuff_data()).
+%%% eqc:quickcheck(protobuffs_eqc:prop_encode_decode1()).
+%%% eqc:quickcheck(protobuffs_eqc:prop_encode_decode2()).
+%%% eqc:quickcheck(protobuffs_eqc:prop_encode_decode3()).
+%%%
+%%% File    : protobuffs_eqc.erl
+%%% Author  :  <thomas@QUVIQ-THOMAS>
+%%% Description : QuickCheck specification used in class for
+%%%               protobuffs-0.2
+%%% Created : 27 Apr 2009 by  <thomas@QUVIQ-THOMAS>
+-module(protobuffs_eqc).
+
+-include_lib("eqc/include/eqc.hrl").
+
+-compile(export_all).
+
+-define(Mach_Eps, 1.1920928955078125e-7).
+-define(NotYetImplemented(Cond,Prop), ?IMPLIES(not (Cond),Prop)).
+
+%% Properties
+
+prop_encode_decode1() ->
+    ?FORALL({FieldNum,Data,Type}, protobuff_data(),
+        collect(Type,
+            begin
+                {{N, RData}, <<>>} = protobuffs:decode(protobuffs:encode(FieldNum, Data, Type), Type),
+                FieldNum =:= N andalso 
+                (compare(Data, RData) orelse foreign_type(Type, Data, RData))  
+            end)).
+    
+prop_encode_decode2() ->
+    ?FORALL({FieldNum,Data,Type}, fault_rate(5,10,protobuff_data()),
+        case catch protobuffs:encode(FieldNum,Data,Type) of
+            {'EXIT', _} ->
+                not in_range(Data,Type);
+            Bin ->
+                {{N, RData}, <<>>} = protobuffs:decode(Bin, Type),
+                in_range(Data,Type) andalso
+                FieldNum =:= N andalso
+                (compare(Data,RData) orelse foreign_type(Type,Data,RData))
+        end).
+
+prop_encode_decode3() ->
+    ?FORALL(Many, protobuff_many(),
+        begin
+            Sorted = lists:keysort(1, Many),
+            IOList = [protobuffs:encode(FNum,Data,Type) || {FNum,Data,Type} <- Sorted],
+            Bin = iolist_to_binary(IOList),
+            {Decoded0,_} = lists:foldl(
+                fun({_,_,Type}, {Acc, Bin1}) ->
+                    {Val, Rest} = protobuffs:decode(Bin1, Type),
+                    {[Val|Acc], Rest}
+                end, {[],Bin}, Sorted),
+            Decoded = lists:reverse(Decoded0),
+            lists:foldl(
+                fun (_, false) -> false;
+                    (I, true) ->                        
+                        {FNum1, Data1, Type1} = lists:nth(I, Sorted),
+                        {FNum2, Data2} = lists:nth(I, Decoded),
+                        (FNum1 =:= FNum2 andalso
+                        (compare(Data1,Data2) orelse foreign_type(Type1,Data1,Data2)))
+                end, true, lists:seq(1, length(Sorted))) 
+        end).
+
+prop_encode_decode4() ->
+    ?FORALL({ProtoName, Msgs}, protobuff_msgs(),   
+        begin
+            code:delete(list_to_atom(ProtoName)),
+            code:purge(list_to_atom(ProtoName)),
+            protobuffs_compile:output(ProtoName, Msgs),
+            ?FORALL(Set, protobuff_set(Defs),
+                begin
+                    Record = build_record_from_defs(MsgName, Defs, Set),
+                    case catch apply(list_to_atom(ProtoName), list_to_atom("encode_" ++ MsgName), [Record]) of
+                        {'EXIT', {error, {required_field_is_undefined, FNum, _}}} ->
+                            {value, {_, required, _, _, _, _}} = lists:keysearch(FNum, 1, Defs),
+                            true;
+                        Bin when is_binary(Bin) ->
+                            Record1 = apply(list_to_atom(ProtoName), list_to_atom("decode_" ++ MsgName), [Bin]),
+                            lists:foldl(
+                                fun ({A,B}, true) ->
+                                        io:format("compare ~p and ~p~n", [A,B]),
+                                        compare(A,B);
+                                    (_, false) ->
+                                        false
+                                end, true, lists:zip(tuple_to_list(Record), tuple_to_list(Record1)))
+                    end
+                end)
+        end).
+    
+%% Data generators
+
+protobuff_msgs() ->
+    {string(), list({msg_name(), protobuff_defs()})}.
+    
+protobuff_many() ->
+    list(protobuff_data()).
+    
+protobuff_set(Defs) ->
+    [begin
+        {FNum, Name, field_value(Tag, Type)}
+     end || {FNum, Tag, Type, Name, _, _Default} <- Defs].
+    
+protobuff_data() ->
+    fault({field_num(), int(80), oneof([int32,uint32,int64,uint64,sint32,sint64])},
+        oneof([
+            {field_num(), int(32), int32},
+            {field_num(), uint(32), uint32},
+            {field_num(), int(64), int64},
+            {field_num(), uint(64), uint64},
+            {field_num(), bool(), bool},
+            {field_num(), sint(32), sint32},
+            {field_num(), sint(64), sint64},
+            {field_num(), real(), float},
+            {field_num(), real(), double},
+            {field_num(), list(char()), string},
+            {field_num(), binary(), bytes}
+        ])
+    ).
+    
+protobuff_defs() ->
+    ?SUCHTHAT(D,orderedlist(protobuff_def()),length(D) > 0).
+    
+protobuff_def() ->
+    oneof([
+        {field_num(), tag(), "int32",   field_name(), number, oneof([none, int(32)])},
+        {field_num(), tag(), "uint32",  field_name(), number, oneof([none, uint(32)])},
+        {field_num(), tag(), "int64",   field_name(), number, oneof([none, int(64)])},
+        {field_num(), tag(), "uint64",  field_name(), number, oneof([none, uint(64)])},
+        {field_num(), tag(), "bool",    field_name(), number, oneof([none, bool()])},
+        {field_num(), tag(), "sint32",  field_name(), number, oneof([none, sint(32)])},
+        {field_num(), tag(), "sint64",  field_name(), number, oneof([none, sint(64)])},
+        {field_num(), tag(), "float",   field_name(), number, oneof([none, real()])},
+        {field_num(), tag(), "double",  field_name(), number, oneof([none, real()])},
+        {field_num(), tag(), "string",  field_name(), number, oneof([none, list(char())])},
+        {field_num(), tag(), "bytes",   field_name(), number, oneof([none, binary()])}
+    ]).
+    
+field_value(repeated, "int32") -> oneof([undefined, list(int(32))]);
+field_value(repeated, "uint32") -> oneof([undefined, list(uint(32))]);
+field_value(repeated, "int64") -> oneof([undefined, list(int(64))]);
+field_value(repeated, "uint64") -> oneof([undefined, list(uint(64))]);
+field_value(repeated, "bool") -> oneof([undefined, list(bool())]);
+field_value(repeated, "sint32") -> oneof([undefined, list(sint(32))]);
+field_value(repeated, "sint64") -> oneof([undefined, list(sint(64))]);
+field_value(repeated, "float") -> oneof([undefined, list(real())]);
+field_value(repeated, "double") -> oneof([undefined, list(real())]);
+field_value(repeated, "string") -> oneof([undefined, list(list(char()))]);
+field_value(repeated, "bytes") -> oneof([undefined, list(binary())]);
+field_value(_, "int32") -> oneof([undefined, int(32)]);
+field_value(_, "uint32") -> oneof([undefined, uint(32)]);
+field_value(_, "int64") -> oneof([undefined, int(64)]);
+field_value(_, "uint64") -> oneof([undefined, uint(64)]);
+field_value(_, "bool") -> oneof([undefined, bool()]);
+field_value(_, "sint32") -> oneof([undefined, sint(32)]);
+field_value(_, "sint64") -> oneof([undefined, sint(64)]);
+field_value(_, "float") -> oneof([undefined, real()]);
+field_value(_, "double") -> oneof([undefined, real()]);
+field_value(_, "string") -> oneof([undefined, list(char())]);
+field_value(_, "bytes") -> oneof([undefined, binary()]).
+    
+field_num() ->
+    ?SUCHTHAT(N,nat(),N>0).
+    
+tag() ->
+    oneof([optional, required, repeated]).
+    
+field_name() ->
+   ?SUCHTHAT(N,string(),length(N)>0).
+    
+msg_name() ->
+    ?SUCHTHAT(N,string(),length(N)>0).
+    
+string() ->
+    list(oneof([choose(97,122), choose(65,90)])).
+    
+%% Internal Functions
+
+foreign_type(bool,false,0) ->
+    true;
+foreign_type(bool,true,1) ->
+    true;
+foreign_type(_,_,_) ->
+    false.
+
+prop_varint() ->
+    ?FORALL(Base,oneof([32,64]),
+        ?FORALL(I,int(Base),
+            begin
+                {Bits,Data} = decompose(protobuffs:encode_varint(I)),
+                right_bits(Bits) andalso 
+                concatenate(Data) == I
+            end
+        )
+    ).
+
+build_record_from_defs(MsgName, Defs, Set) ->
+    lists:foldl(
+        fun({FNum, _Tag, _Type, _Name, _, _Default}, Acc) ->
+            {value, {_,_,Value}} = lists:keysearch(FNum, 1, Set),
+            erlang:append_element(Acc, Value)
+        end, {list_to_atom(MsgName)}, Defs).
+
+%% Bits are in reverse order: First bit should be zero, rest should be 1 
+right_bits([0|Rest]) ->
+    lists:all(fun(B) -> B==1 end,Rest).
+
+int(Base) ->
+    ?LET(I,uint(Base),
+        begin 
+            << N:Base/signed >> = <<I:Base>>, N 
+        end
+    ).
+
+uint(Base) ->
+    oneof([ choose(0,exp(B)) || B<-lists:seq(1,Base)]).
+
+sint(Base) ->
+    int(Base).
+
+exp(1) ->
+    2;
+exp(N) ->
+    2*exp(N-1).
+
+decompose(<<Bit:1,Data:7>>) ->
+    {[Bit],<<Data:7>>};
+decompose(<<Bit:1,Data:7,Rest/binary>>) ->
+    {Bs,Ds} = decompose(Rest),
+    {Bs++[Bit],<<Ds/bitstring,Data:7>>}.
+
+concatenate(Bin) ->
+    S = bit_size(Bin),
+    << N:S >> = Bin,
+    N.
+
+in_range(Int,int32) ->
+    fitbits(Int,32);
+in_range(Int,sint32) ->
+    fitbits(abs(Int),31);
+in_range(Int,uint32) ->
+    fitbits(Int,32);
+in_range(Int,int64) ->
+    fitbits(Int,64);
+in_range(Int,sint64) ->
+    fitbits(abs(Int),63);
+in_range(Int,uint64) ->
+    fitbits(Int,64);
+in_range(Float,float) ->
+    fitbits(Float,32);
+in_range(Float,double) ->
+    fitbits(Float,64);
+in_range(_,string) ->
+    true;
+in_range(_,bytes) ->
+    true;
+in_range(false,bool) ->
+    true;
+in_range(true,bool) ->
+    true.
+
+compare(Float1, Float2) when is_float(Float1), is_float(Float2) ->
+    (abs(Float1 - Float2) =< ?Mach_Eps);
+compare(String, Binary) when is_list(String), is_binary(Binary) ->
+    String =:= binary_to_list(Binary);
+compare(A,A) -> true;
+compare(_,_) -> false.
+
+fitbits(Float,32) when is_float(Float) -> true;
+fitbits(Float,64) when is_float(Float) -> true;
+fitbits(Int,Bits) ->
+    RestBits = 80-Bits,
+    << NoFit:RestBits, _:Bits >> = <<Int:80>>,
+    NoFit == 0.

apps/protobuffs/t/protobuffs_t_001.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+main(_) ->
+    etap:plan(8),
+    etap_can:loaded_ok(protobuffs, "module 'protobuffs' loaded"),
+    etap_can:can_ok(protobuffs, encode),
+    etap_can:can_ok(protobuffs, encode, 3),
+    etap_can:can_ok(protobuffs, decode),
+    etap_can:can_ok(protobuffs, decode, 2),
+    etap_can:loaded_ok(protobuffs_compile, "module 'protobuffs_compile' loaded"),
+    etap_can:can_ok(protobuffs_compile, scan_file),
+    etap_can:can_ok(protobuffs_compile, scan_file, 1),
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_002.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+main(_) ->
+    etap:plan(3),
+    etap:is(protobuffs:encode(1, 1, uint32), <<8,1>>, "1, 1, unit32"),
+    etap:is(protobuffs:encode(2, 1, uint32), <<16,1>>, "2, 1, unit32"),
+    etap:is(protobuffs:encode(3, 1, uint32), <<24,1>>, "3, 1, unit32"),
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_003.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -noshell
+
+main(_) ->
+    etap:plan(18),
+    Tests = [
+        {8, uint32},
+        {16, uint32},
+        {24, uint32},
+        {1, fixed32},
+        {1, enum},
+        {200, enum},
+        {9933, int64},
+        {-391, sint64},
+        {5, fixed32},
+        {-5, sfixed32},
+        {30, fixed64},
+        {500, sfixed64},
+        {"Whirlwind tickles.", string, <<"Whirlwind tickles.">>},
+        {"", string, <<>>},
+        {<<"It's a secret to everyone.">>, string},
+        {<<4,8,15,16,23,42>>, bytes},
+        {3.141592025756836, float},
+        {1.00000000000000022204460492503130808472633361816406, double}
+    ],
+    lists:foreach(
+        fun(Test) ->
+            {Value, Type, Expected} = case Test of
+                {A, B} -> {A, B, A};
+                _ -> Test
+            end,
+            Decode = protobuffs:decode(iolist_to_binary(protobuffs:encode(1, Value, Type)), Type),
+            etap:is(
+                protobuffs:decode(iolist_to_binary(protobuffs:encode(1, Value, Type)), Type),
+                {{1, Expected}, <<>>},
+                "encode/decode test -- random " ++ atom_to_list(Type)
+            )
+        end,
+        Tests
+    ),
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_005.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+-record(location, {region, country}).
+-record(person, {name, address, phone_number, age, location}).
+
+main(_) ->
+    etap:plan(2),
+    etap:is(protobuffs_compile:scan_file("t/simple.proto"), ok, "simple.proto compiled"),
+	
+    Person = #person{
+        name = "Nick",
+        address = "Mountain View",
+        phone_number = "+1 (000) 555-1234",
+        age = 25,
+		location = #location{region="CA", country="US"}
+    },
+
+	Bin = simple_pb:encode_person(Person),
+
+	etap:is(simple_pb:decode_person(Bin), Person, "Encoded and decoded person"),
+	
+	ok = file:delete("simple_pb.hrl"),
+	ok = file:delete("simple_pb.beam"),
+	
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_006.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+-record(location, {region, country}).
+-record(person, {name, address, phone_number, age, location}).
+
+main(_) ->
+    etap:plan(3),
+    etap:is(protobuffs_compile:scan_file("t/simple.proto"), ok, "simple.proto compiled"),
+
+	Fields1 = [
+		{1, "California", string},
+		{2, "USA", string}
+	],
+
+    LocationBinData = erlang:iolist_to_binary([protobuffs:encode(Pos, Value, Type) || {Pos, Value, Type} <- Fields1]),
+
+    Fields2 = [
+        {1, "Nick", string},
+        {2, "Mountain View", string},
+        {3, "+1 (000) 555-1234", string},
+        {4, 25, int32},
+        {5, LocationBinData, bytes}
+    ],
+
+	PersonBinData = erlang:iolist_to_binary([protobuffs:encode(Pos, Value, Type) || {Pos, Value, Type} <- Fields2]),
+
+	Location = #location{region="California", country="USA"},
+	
+    Person = #person{
+        name = "Nick",
+        address = "Mountain View",
+        phone_number = "+1 (000) 555-1234",
+        age = 25,
+        location = Location
+    },
+
+	etap:is(simple_pb:encode_location(Location), LocationBinData, "Encoded locations match"),
+	etap:is(simple_pb:encode_person(Person), PersonBinData, "Encoded persons match"),
+
+	ok = file:delete("simple_pb.hrl"),
+	ok = file:delete("simple_pb.beam"),
+	
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_007.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+-record(location, {region, country}).
+-record(person, {name, address, phone_number, age, hobbies, locations}).
+
+main(_) ->
+    etap:plan(5),
+    etap:is(protobuffs_compile:scan_file("t/repeater.proto"), ok, "repeater.proto compiled"),
+
+	Fields1 = [
+		{1, "Lyon", string},
+		{2, "France", string}
+	],
+	
+	Fields2 = [
+		{1, "Reykjavik", string},
+		{2, "Iceland", string}
+	],
+	
+	LocationBinData1 = erlang:iolist_to_binary([protobuffs:encode(Pos, Value, Type) || {Pos, Value, Type} <- Fields1]),
+	LocationBinData2 = erlang:iolist_to_binary([protobuffs:encode(Pos, Value, Type) || {Pos, Value, Type} <- Fields2]),
+	
+	Fields3 = [
+        {1, "Nick", string},
+        {2, "Mountain View", string},
+        {3, "+1 (000) 555-1234", string},
+        {4, 25, int32},
+		{5, "paddling", string},
+		{5, "floating", string},
+        {6, LocationBinData1, bytes},
+		{6, LocationBinData2, bytes}
+    ],
+
+	PersonBinData1 = erlang:iolist_to_binary([protobuffs:encode(Pos, Value, Type) || {Pos, Value, Type} <- Fields3]),
+
+    Person = #person{
+        name = "Nick",
+        address = "Mountain View",
+        phone_number = "+1 (000) 555-1234",
+        age = 25,
+		hobbies = ["paddling", "floating"],
+        locations = 
+			[#location{region = "Lyon", country = "France"},
+			 #location{region = "Reykjavik", country = "Iceland"}]
+    },
+	
+	PersonBinData2 = repeater_pb:encode_person(Person),
+
+	Person1 = repeater_pb:decode_person(PersonBinData1),
+	Person2 = repeater_pb:decode_person(PersonBinData2),
+	
+	etap:is(PersonBinData1, PersonBinData2, "Person binaries match"),
+
+	etap:is(Person, Person1, "Encoded and decoded persons match"),
+	
+	etap:is(Person1, Person2, "Encoded and decoded persons match"),
+	
+	Person3 = #person {
+		name = "Nick",
+        address = "Mountain View",
+        phone_number = "+1 (000) 555-1234",
+        age = 25,
+		hobbies = ["paddling", "floating"]
+	},
+	
+	etap:is(repeater_pb:decode_person(repeater_pb:encode_person(Person3)), Person3, "Encoded and decoded person without locations"),
+
+	ok = file:delete("repeater_pb.hrl"),
+	ok = file:delete("repeater_pb.beam"),
+	
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_008.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+-record(location, {region, country}).
+-record(person, {name, address, phone_number, age, location}).
+
+main(_) ->
+    etap:plan(2),
+    etap:is(protobuffs_compile:scan_file("t/hasdefault.proto"), ok, "hasdefault.proto created"),
+
+	Person = #person {
+		name = "Nick",
+		address = "Mountain View",
+		location = #location{region = "Lyon", country = "France"}	
+	},
+	
+    DefaultPerson = #person{
+    	name = "Nick",
+    	address = "Mountain View",
+        phone_number = "+1 (000) 000-0000",
+		age = 25,
+		location = #location{region = "Lyon", country = "France"}
+    },
+
+	Bin = hasdefault_pb:encode_person(Person),
+	
+	etap:is(hasdefault_pb:decode_person(Bin), DefaultPerson, "Default person encoded and decoded"),
+
+	ok = file:delete("hasdefault_pb.hrl"),
+	ok = file:delete("hasdefault_pb.beam"),
+
+    etap:end_tests().

apps/protobuffs/t/protobuffs_t_009.t

+#!/usr/bin/env escript
+%% -*- erlang -*-
+%%! -pa ./ebin -sasl errlog_type error -boot start_sasl -noshell
+
+-record(location, {region, country}).
+-record(person, {name, address, phone_number, age, location}).
+
+main(_) ->
+    etap:plan(2),
+    etap:is(protobuffs_compile:scan_file("t/hasdefault.proto"), ok, "hasdefault.proto created"),
+
+	case catch hasdefault_pb:encode_person(#person{}) of
+		{'EXIT', {error, {required_field_is_undefined,1,string}}} ->
+			etap:ok(true, "Required field is undefined");
+		_ ->
+			etap:ok(false, "Required field is undefined")
+	end,
+	
+	ok = file:delete("hasdefault_pb.hrl"),
+	ok = file:delete("hasdefault_pb.beam"),
+
+    etap:end_tests().

apps/protobuffs/t/repeater.proto

+package repeater;
+
+message Person {
+	required string name = 1;
+	required string address = 2;
+	required string phone_number = 3;
+	required int32 age = 4;
+	repeated string hobbies = 5;
+	repeated Location locations = 6;
+}
+
+message Location {
+    required string region = 1;
+    required string country = 2;
+}

apps/protobuffs/t/simple.proto

+package simple;
+
+message Person {
+	required string name = 1;
+	required string address = 2;
+	required string phone_number = 3;
+	required int32 age = 4;
+	optional Location location = 5;
+}
+
+message Location {
+    required string region = 1;
+    required string country = 2;
+}

apps/riak/ebin/riak.app

              riak_map_phase_fsm,
              riak_mapreduce,
              riak_mapreduce_fsm,
-			 riak_multi_backend,
+             riak_multi_backend,
              riak_object,
              riak_osmos_backend,
              riak_put_fsm,
              riak_util,
              riak_vnode,
              riak_vnode_master,
+             riak_vnode_sup,
              riak_web,
+             riakserver_pb,
              slide,
              spiraltime,
              vclock

apps/riak/include/.empty_for_hg

Empty file added.

apps/riak/src/riak_sup.erl

     Eventer = {riak_eventer,
                {riak_eventer, start_link, []},
                permanent, 5000, worker, [riak_eventer]},
+    VSup = {riak_vnode_sup,
+            {riak_vnode_sup, start_link, []},
+            permanent, infinity, supervisor, [riak_vnode_sup]},
     VMaster = {riak_vnode_master,
                {riak_vnode_master, start_link, []},
                permanent, 5000, worker, [riak_vnode_master]},
     
     % Build the process list...
     Processes = lists:flatten([
-        Eventer,
+        Eventer, 
+        VSup,                               
         ?IF(HasStorageBackend, VMaster, []),
         RingMgr, 
         Connect, 

apps/riak/src/riak_vnode.erl

 -module(riak_vnode).
 -behaviour(gen_fsm).