Commits

Jacob Perkins committed 4e7cc7b Merge

merge

  • Participants
  • Parent commits 4317d86, d07fbf1

Comments (0)

Files changed (5)

 Erlaws provides Erlang interfaces to various Amazon WebService offerings.
 
-This code is no longer maintained, so feel free to fork it.
-
 -- original documentation from Google Code wiki --
 
 = Description = 
 
 = Build =
 
-Check out the latest code from svn and issue {{{erl -make}}} to build the sources.
+Issue `erl -make` to build the sources.
 
 = Usage =
 
   * list_contents/1
   * list_contents/2
   * put_object/5
+  * put_file/5
   * get_object/2
   * info_object/2
   * delete_object/2
+-module(erlaws). 
+
+-behaviour(application). 
+
+-export([start/0, start/2, stop/1]). 
+
+start() ->
+    application:start(sasl),
+    crypto:start(),
+    inets:start().
+
+start(_Type, _Args) -> 
+	erlaws:start().
+	
+stop(_State) -> 
+    ok. 

src/erlaws_s3.erl

 
 %% API
 -export([list_buckets/0, create_bucket/1, create_bucket/2, delete_bucket/1]).
--export([list_contents/1, list_contents/2, put_object/5, get_object/2]).
+-export([list_contents/1, list_contents/2, put_object/5, put_file/5, get_object/2]).
 -export([info_object/2, delete_object/2]).
 
 %% include record definitions
 -include_lib("xmerl/include/xmerl.hrl").
+-include_lib("kernel/include/file.hrl").
 -include("../include/erlaws.hrl").
 
 %% macro definitions
--define( AWS_S3_HOST, "s3.amazonaws.com").
--define( NR_OF_RETRIES, 3).
--define( CALL_TIMEOUT, indefinite).
--define( S3_REQ_ID_HEADER, "x-amz-request-id").
--define( PREFIX_XPATH, "//CommonPrefixes/Prefix/text()").
+-define(AWS_S3_HOST, "s3.amazonaws.com").
+-define(NR_OF_RETRIES, 3).
+-define(CALL_TIMEOUT, indefinite).
+-define(S3_REQ_ID_HEADER, "x-amz-request-id").
+-define(PREFIX_XPATH, "//CommonPrefixes/Prefix/text()").
+-define(CHUNK_SIZE, 8 * 1024).
 
 %% Returns a list of all of the buckets owned by the authenticated sender 
 %% of the request.
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
 list_buckets() ->
-    try genericRequest(get, "", "", "", [], "", <<>>) of
+    try genericRequest(get, "", "", [], [], [], <<>>) of
 	{ok, Headers, Body} -> 
 	    {XmlDoc, _Rest} = xmerl_scan:string(binary_to_list(Body)),
 	    TextNodes       = xmerl_xpath:string("//Bucket/Name/text()", XmlDoc),
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
 create_bucket(Bucket) ->
-    try genericRequest(put, Bucket, "", "", [], "", <<>>) of
+    try genericRequest(put, Bucket, "", [], [], [], <<>>) of
 	{ok, Headers, _Body} -> 
 	    RequestId = case lists:keytake("x-amz-request-id", 1, Headers) of
 			{value, {_, ReqId}, _} -> ReqId;
     LCfg = <<"<CreateBucketConfiguration>
                   <LocationConstraint>EU</LocationConstraint>
              </CreateBucketConfiguration>">>,
-    try genericRequest(put, Bucket, "", "", [], "", LCfg) of
+    try genericRequest(put, Bucket, "", [], [], [], LCfg) of
 	{ok, Headers, _Body} ->
 		RequestId = case lists:keytake("x-amz-request-id", 1, Headers) of
 			{value, {_, ReqId}, _} -> ReqId;
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
 delete_bucket(Bucket) ->
-    try genericRequest(delete, Bucket, "", "", [], "", <<>>) of
+    try genericRequest(delete, Bucket, "", [], [], [], <<>>) of
 	{ok, Headers, _Body} ->
 	    RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
 			{value, {_, ReqId}, _} -> ReqId;
 %%
 list_contents(Bucket, Options) when is_list(Options) ->
     QueryParameters = [makeParam(X) || X <- Options],
-    try genericRequest(get, Bucket, "", QueryParameters, [], "", <<>>) of
+    try genericRequest(get, Bucket, "", QueryParameters, [], [], <<>>) of
 	{ok, Headers, Body} -> 
 	    {XmlDoc, _Rest} = xmerl_scan:string(binary_to_list(Body)),
 	    [Truncated| _Tail] = xmerl_xpath:string("//IsTruncated/text()", 
 	    {error, Descr}
     end.
 
-%% Uploads data for key.
+%% Uploads data for key. Backwards-compatible version.
 %%
 %% Spec: put_object(Bucket::string(), Key::string(), Data::binary(),
 %%                  ContentType::string(), 
 %%       {ok, #s3_object_info(key=Key::string(), size=Size::integer())} |
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
-put_object(Bucket, Key, Data, ContentType, Metadata) ->
-    try genericRequest(put, Bucket, Key, [], Metadata, ContentType, Data) of
+put_object(Bucket, Key, Data, ContentType, Metadata) when is_integer(hd(ContentType)) ->
+    put_object(Bucket, Key, Data, [{"Content-Type", ContentType}], Metadata);
+
+%% Uploads data for key. More general version.
+%%
+%% Spec: put_object(Bucket::string(), Key::string(), Data::binary(),
+%%                  HTTPHeaders::[{Key::string(), Value::string()}]
+%%                  Metadata::[{Key::string(), Value::string()}]) ->
+%%       {ok, #s3_object_info(key=Key::string(), size=Size::integer())} |
+%%       {error, {Code::string(), Msg::string(), ReqId::string()}}
+%%
+%% EXAMPLE:
+%% S3 = ?MODULE:new(...),	% Fill it according to your preferences.
+%% S3:put_object("someBucket", "filename.js", <<"...">>, [{"Content-Type", "application/x-javascript; charset=\"utf-8\""},{"Cache-Control", "max-age=86400"},{"x-amz-acl", "public-read"}], [{"name", "metavalue"}]).
+%%
+%%
+put_object(Bucket, Key, Data, HTTPHeaders, Metadata) ->
+    try genericRequest(put, Bucket, Key, [], Metadata, HTTPHeaders, Data) of
 	{ok, Headers, _Body} -> 
 	    RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
 			{value, {_, ReqId}, _} -> ReqId;
 	throw:{error, Descr} ->
 	    {error, Descr}
     end.
+
+put_file(Bucket, Key, FileName, ContentType, Metadata) ->
+    Date = httpd_util:rfc1123_date(erlang:localtime()),
+    {FileSize, File} = openAndGetFileSize(FileName),
+    Headers = 
+        buildContentHeaders(FileSize) ++
+	buildMetadataHeaders(Metadata),
+    Signature = sign(AWS_SEC_KEY,
+                     stringToSign("PUT", "", ContentType, Date,
+                                  Bucket, Key, Headers)),
+    FinalHeaders = [ {"Authorization", "AWS " ++ AWS_KEY ++ ":" ++ Signature },
+		     {"Host", buildHost(Bucket) },
+		     {"Date", Date },
+		     {"Content-Type", ContentType}
+		     | Headers ],
+    Payload = 
+        lists:append(
+          ["PUT /", Key, " HTTP/1.1\n",
+           lists:flatten([lists:append([K, ": ", V, "\n"]) || 
+                             {K, V} <- lists:reverse(FinalHeaders)]),
+           "\n"]),
+    {ok, Socket} = gen_tcp:connect(?AWS_S3_HOST, 80, 
+                                   [binary, {active, false}, {packet, 0}]),
+    gen_tcp:send(Socket, list_to_binary(Payload)),
+    sendData(Socket, File),
+    gen_tcp:close(Socket),
+    file:close(File).
        
 %% Retrieves the data associated with the given key.
 %% 
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
 get_object(Bucket, Key) ->
-    try genericRequest(get, Bucket, Key, [], [], "", <<>>) of
+    try genericRequest(get, Bucket, Key, [], [], [], <<>>) of
 	{ok, Headers, Body} -> 
 		RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
 			{value, {_, ReqId}, _} -> ReqId;
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
 info_object(Bucket, Key) ->
-    try genericRequest(head, Bucket, Key, [], [], "", <<>>) of
+    try genericRequest(head, Bucket, Key, [], [], [], <<>>) of
 	{ok, Headers, _Body} ->
 	    io:format("Headers: ~p~n", [Headers]),
 		MetadataList = [{string:substr(MKey, 12), Value} || {MKey, Value} <- Headers, string:str(MKey, "x-amz-meta") == 1],
 %%       {error, {Code::string(), Msg::string(), ReqId::string()}}
 %%
 delete_object(Bucket, Key) ->
-    try genericRequest(delete, Bucket, Key, [], [], "", <<>>) of
+    try genericRequest(delete, Bucket, Key, [], [], [], <<>>) of
 	{ok, Headers, _Body} ->
 		RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
 			{value, {_, ReqId}, _} -> ReqId;
     Strings = lists:map( 
 		fun mkHdr/1, 
 		collapse(XAmzHeaders)),
-    erlaws_util:mkEnumeration( lists:map( fun (String) -> String ++ "\n" end, 
-					  Strings), "").
+    erlaws_util:mkEnumeration( [[String, "\n"] || String <- Strings], "").
 
 canonicalizeResource ( "", "" ) -> "/";
 canonicalizeResource ( Bucket, "" ) -> "/" ++ Bucket ++ "/";
     buildProtocol() ++ Bucket ++ "." ++ ?AWS_S3_HOST ++ "/" ++ Path ++ 
 	erlaws_util:queryParams(QueryParams).
 
-buildContentHeaders( <<>>, _ ) -> [];
-buildContentHeaders( Contents, ContentType ) -> 
+buildContentHeaders(Contents) when is_integer(Contents) -> 
+    [{"Content-Length", integer_to_list(Contents)}];
+% Detect gzip header and put appropriate Content-Encoding. Questionable?..
+buildContentHeaders(<<16#1f, 16#8b, _/binary>> = Contents) -> 
     [{"Content-Length", integer_to_list(size(Contents))},
-     {"Content-Type", ContentType}].
+     {"Content-Encoding", "gzip"}];
+buildContentHeaders(Contents) -> 
+    [{"Content-Length", integer_to_list(size(Contents))}].
 
 buildMetadataHeaders(Metadata) ->
-    buildMetadataHeaders(Metadata, []).
-
-buildMetadataHeaders([], Acc) ->
-    Acc;
-buildMetadataHeaders([{Key, Value}|Tail], Acc) ->
-    buildMetadataHeaders(Tail, [{string:to_lower("x-amz-meta-"++Key), Value} 
-				| Acc]).
+    lists:foldl(fun({Key, Value}, Acc) ->
+		[{string:to_lower("x-amz-meta-"++Key), Value} | Acc]
+	end, [], Metadata).
 
 buildContentMD5Header(ContentMD5) ->
     case ContentMD5 of
 	canonicalizeResource(Bucket, Path).
 
 sign (Key,Data) ->
-    %io:format("StringToSign:~n ~p~n", [Data]),
     binary_to_list( base64:encode( crypto:sha_mac(Key,Data) ) ).
 
 genericRequest( Method, Bucket, Path, QueryParams, Metadata,
-		ContentType, Body ) ->
+		HTTPHeaders, Body ) ->
     genericRequest( Method, Bucket, Path, QueryParams, Metadata,
-		    ContentType, Body, ?NR_OF_RETRIES).
+		    HTTPHeaders, Body, ?NR_OF_RETRIES).
 
 genericRequest( Method, Bucket, Path, QueryParams, Metadata, 
-		ContentType, Body, NrOfRetries) ->
+		HTTPHeaders, Body, NrOfRetries) ->
     Date = httpd_util:rfc1123_date(erlang:localtime()),
     MethodString = string:to_upper( atom_to_list(Method) ),
     Url = buildUrl(Bucket,Path,QueryParams),
 		     _ -> binary_to_list(base64:encode(erlang:md5(Body)))
 		 end,
     
-    Headers = buildContentHeaders( Body, ContentType ) ++
+    Headers =
+        buildContentHeaders(Body) ++
 	buildMetadataHeaders(Metadata) ++ 
-	buildContentMD5Header(ContentMD5),
+	buildContentMD5Header(ContentMD5) ++
+	HTTPHeaders,
+
+    ContentType = case [Value || {"Content-Type", Value} <- HTTPHeaders] of
+		[CT|_] -> CT;
+		[] -> ""
+	end,
     
     {AccessKey, SecretAccessKey } = {AWS_KEY, AWS_SEC_KEY},
 
     Signature = sign(SecretAccessKey,
-		     stringToSign( MethodString, ContentMD5, ContentType, Date,
+		     stringToSign(MethodString, ContentMD5, ContentType, Date,
 				   Bucket, Path, Headers )),
     
     FinalHeaders = [ {"Authorization","AWS " ++ AccessKey ++ ":" ++ Signature },
     HttpOptions = [{autoredirect, true}],
     Options = [ {sync,true}, {headers_as_is,true}, {body_format, binary} ],
 
-    %%io:format("Request:~n ~p~n", [Request]),
-
     Reply = http:request( Method, Request, HttpOptions, Options ),
     
     %%     {ok, {Status, ReplyHeaders, RBody}} = Reply,
  	    {ok, ResponseHeaders, ResponseBody};
 	
 	{ok, {{_HttpVersion, Code, ReasonPhrase}, ResponseHeaders, 
-	      _ResponseBody }} when Code=:=500, NrOfRetries > 0 ->
+	      _ResponseBody }} when Code=:=500, NrOfRetries == 0 ->
 	    throw ({error, "500", ReasonPhrase, 
 		    proplists:get_value(?S3_REQ_ID_HEADER, ResponseHeaders)});
 	
 	      _ResponseBody }} when Code=:=500 ->
 	    timer:sleep((?NR_OF_RETRIES-NrOfRetries)*500),
 	    genericRequest(Method, Bucket, Path, QueryParams, 
-			   Metadata, ContentType, Body, NrOfRetries-1);
+			   Metadata, HTTPHeaders, Body, NrOfRetries-1);
 	
  	{ok, {{_HttpVersion, _HttpCode, _ReasonPhrase}, ResponseHeaders, 
 	      ResponseBody }} ->
     #s3_object_info{key=Key#xmlText.value, lastmodified=LastModified#xmlText.value,
 		 etag=ETag#xmlText.value, size=Size#xmlText.value}.
 
+openAndGetFileSize(FileName) ->
+    case file:open(FileName, [read, binary]) of
+        {ok, File} ->
+            {ok, #file_info{size=Size}} = file:read_file_info(FileName),
+            {Size, File};
+        _ ->
+            {error, no_file}
+    end.
 
-
+sendData(Socket, File) ->
+    case file:read(File, ?CHUNK_SIZE) of
+        {ok, Data} ->
+            gen_tcp:send(Socket, Data),
+            sendData(Socket, File);
+        eof ->
+            ok
+    end.

src/erlaws_sdb.erl

     try genericRequest("CreateDomain", 
 		       Domain, "", [], []) of
 	{ok, Body} ->
-		{XmlDoc, _Rest} = xmerl:scan_string(Body),
+		{XmlDoc, _Rest} = xmerl_scan:string(Body),
 		[#xmlText{value=RequestId}|_] =
 			xmerl_xpath:string("//ResponseMetadata/RequestId/text()", XmlDoc), 
 	    {ok, {requestId, RequestId}}

src/erlaws_util.erl

 		  [Year, Month, Day, Hour, Min, Sec, Zone]).
 
 mkEnumeration(Values, Separator) ->
-    lists:flatten(lists:reverse(mkEnumeration(Values, Separator, []))).
-
-mkEnumeration([], _Separator, Acc) ->
-    Acc;
-mkEnumeration([Head|[]], _Separator, Acc) ->
-    [Head | Acc];
-mkEnumeration([Head|Tail], Separator, Acc) ->
-    mkEnumeration(Tail, Separator, [Separator, Head | Acc]).
+    lists:flatten(string:join(Values, Separator)).
 
 queryParams( [] ) -> "";
 queryParams( ParamList ) -> 
-    "?" ++ mkEnumeration([url_encode(Param) ++ "=" ++ url_encode(Value) 
+    "?" ++ mkEnumeration([[url_encode(Param), "=", url_encode(Value)]
 			  || {Param, Value} <- ParamList], "&" ).
 
 %% The following code is taken from the ibrowse Http client