From 19905606cdd49f2ab3f1a2be035301c5f1f0d9ef Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Fri, 28 Feb 2014 17:04:27 -0800 Subject: [PATCH 01/12] Large trace support --- mondemand_client-dev | 12 +++++ mondemand_client_dev.config | 8 +++ src/mondemand.erl | 103 +++++++++++++++++++++++++++++++++--- 3 files changed, 117 insertions(+), 6 deletions(-) create mode 100755 mondemand_client-dev create mode 100644 mondemand_client_dev.config diff --git a/mondemand_client-dev b/mondemand_client-dev new file mode 100755 index 0000000..2af4807 --- /dev/null +++ b/mondemand_client-dev @@ -0,0 +1,12 @@ +#!/bin/sh + +deps= +for d in deps/* ; do + deps="$deps -pa $d/ebin" +done +exec erl \ + $deps \ + -pa ebin \ + -config mondemand_client_dev.config \ + -s reloader \ + -s mondemand_app diff --git a/mondemand_client_dev.config b/mondemand_client_dev.config new file mode 100644 index 0000000..d82e39a --- /dev/null +++ b/mondemand_client_dev.config @@ -0,0 +1,8 @@ +[ + { mondemand, [ { lwes_channel, { "127.0.0.1", 20502 } }, + { http_endpoint, + [ {trace, "http://127.0.0.1:20602/tcp/trace"} ] + } + ] + } +]. diff --git a/src/mondemand.erl b/src/mondemand.erl index 5fa1c5d..40b35fc 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -35,7 +35,7 @@ code_change/3 ]). --record (state, {config, table, timer, channel}). +-record (state, {config, table, timer, channel, http_config}). -define (TABLE, mondemand_stats). @@ -113,6 +113,28 @@ get_lwes_config () -> end end. +get_http_config () -> + case application:get_env (mondemand, http_endpoint) of + {ok, HttpConfig} -> + HttpConfig; + undefined -> + case application:get_env (mondemand, config_file) of + {ok, File} -> + case file:read_file (File) of + {ok, Bin} -> + {match, [TraceEndPoint]} = + re:run (Bin, "MONDEMAND_HTTP_ENDPOINT_TRACE=\"([^\"]+)\"", + [{capture, all_but_first, list}]), + [{trace, TraceEndPoint}]; + E -> + E + end; + undefined -> + {error, no_http_configured} + end + end. + + all () -> ets:tab2list (?TABLE). @@ -137,7 +159,8 @@ stringify (L) -> L. send_trace (ProgId, Message, Context) -> - case Context of + Context2 = split_heavy_contexts (Context), + case Context2 of Dict when is_tuple (Context) andalso element (1, Context) =:= dict -> case key_in_dict (?TRACE_ID_KEY, Dict) andalso key_in_dict (?OWNER_ID_KEY, Dict) of @@ -173,7 +196,8 @@ send_trace (ProgId, Message, Context) -> end. send_trace (ProgId, Owner, TraceId, Message, Context) -> - case Context of + Context2 = split_heavy_contexts (Context), + case Context2 of Dict when is_tuple (Context) andalso element (1, Context) =:= dict -> send_event ( #lwes_event { @@ -279,6 +303,12 @@ current_config () -> %% gen_server callbacks %%==================================================================== init([]) -> + HttpConfig = + case get_http_config () of + {error, _} -> undefined; + HC -> HC + end, + case get_lwes_config () of {error, Error} -> {stop, {error, Error}}; @@ -296,7 +326,8 @@ init([]) -> config = Config, table = TabId, timer = TRef, - channel = Channel + channel = Channel, + http_config = HttpConfig } }; {error, Error} -> @@ -330,9 +361,23 @@ handle_call (_Request, _From, State) -> {reply, ok, State}. % send an event -handle_cast ({send, Event}, State = #state { channel = Channel }) -> - lwes:emit (Channel, Event), +handle_cast ({send, Event}, + State = #state { channel = Channel, + http_config = HttpConfig}) -> + {lwes_event, EventName, Attrs} = Event, + error_logger:info_msg("Event is ~p~n", [Event]), + case EventName of + "MonDemand::TraceMsg" + -> Bin = lwes_event:to_binary(Event), + error_logger:info_msg("The size of the event ~p~n", [Bin]), + case size(Bin) of + X when X > 65535 -> post_via_http (Bin, HttpConfig); + _ -> lwes:emit (Channel, Event) + end; + _ -> lwes:emit (Channel, Event) + end, {noreply, State}; + % fallback handle_cast (_Request, State) -> {noreply, State}. @@ -352,6 +397,52 @@ code_change (_OldVsn, State, _Extra) -> send_event (Event) -> gen_server:cast (?MODULE, {send, Event}). +post_via_http(Bin, HttpConfig) -> + case HttpConfig of + undefined -> ok; + _ -> Endpoint = proplists:get_value("trace", HttpConfig), + error_logger:info_msg("Calling ~p~n", [Endpoint]), + timer:sleep(5000), + httpc:request + (post, + {Endpoint, [], "", Bin}, + [], []) + end. + +split_heavy_contexts (Context) when is_tuple (Context) -> + dict:from_list ( + split_heavy_contexts ( + dict:to_list (Context))); +split_heavy_contexts (Context) -> + lists:foldl ( + fun ({K, V}, A) + -> case V of + B when is_binary(B) -> A ++ split_chunk (K, V, 50000, 0); + O -> A ++ [{K, O}] + end + end, + [], + Context). + +split_chunk (_, <<>>, _, _) -> []; +split_chunk (FieldName, Bin, ChunkSize, PartIndex) + when size(Bin) =< ChunkSize -> + case PartIndex of + 0 -> [{FieldName, Bin}]; + _ -> [{part_name(FieldName, PartIndex), Bin}] + end; + +split_chunk (FieldName, Bin, ChunkSize, PartIndex) -> + { Chunk, Rest} = split_binary (Bin, ChunkSize), + [ {part_name(FieldName, PartIndex), Chunk} | + split_chunk (FieldName, Rest, ChunkSize, PartIndex + 1) ]. + +part_name (Name, Part) when is_binary (Name) -> + list_to_binary ( + part_name (binary_to_list (Name), Part)); +part_name (Name, Part) -> + Name ++ "-" ++ integer_to_list (Part). + to_string (In) when is_list (In) -> In; to_string (In) when is_atom (In) -> From 5332f0cd787f8afb2e99c64e8eaa84f3fffba422 Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Sun, 2 Mar 2014 10:16:57 -0800 Subject: [PATCH 02/12] removing debug statements --- src/mondemand.erl | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/mondemand.erl b/src/mondemand.erl index 40b35fc..d9543f5 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -365,11 +365,9 @@ handle_cast ({send, Event}, State = #state { channel = Channel, http_config = HttpConfig}) -> {lwes_event, EventName, Attrs} = Event, - error_logger:info_msg("Event is ~p~n", [Event]), case EventName of "MonDemand::TraceMsg" -> Bin = lwes_event:to_binary(Event), - error_logger:info_msg("The size of the event ~p~n", [Bin]), case size(Bin) of X when X > 65535 -> post_via_http (Bin, HttpConfig); _ -> lwes:emit (Channel, Event) @@ -401,8 +399,6 @@ post_via_http(Bin, HttpConfig) -> case HttpConfig of undefined -> ok; _ -> Endpoint = proplists:get_value("trace", HttpConfig), - error_logger:info_msg("Calling ~p~n", [Endpoint]), - timer:sleep(5000), httpc:request (post, {Endpoint, [], "", Bin}, From 751cb72b5926212963e1d41aa49b0aec19b42690 Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Mon, 3 Mar 2014 18:50:33 -0800 Subject: [PATCH 03/12] depending on the lwes library to provide support for large binaries --- rebar.config | 2 +- src/mondemand.app.src | 2 +- src/mondemand.erl | 65 +++++++++++++------------------------------ src/mondemand_app.erl | 2 +- 4 files changed, 22 insertions(+), 49 deletions(-) diff --git a/rebar.config b/rebar.config index 7a67664..9f9b859 100644 --- a/rebar.config +++ b/rebar.config @@ -6,7 +6,7 @@ {deps, [ {lwes, "2.3.1", - {git, "git://github.com/lwes/lwes-erlang.git", {tag, "2.3.1"}} + {git, "git://github.com/lwes/lwes-erlang.git", {branch, "feature/large-data-support"}} } ] }. diff --git a/src/mondemand.app.src b/src/mondemand.app.src index 5cb3893..30dea51 100644 --- a/src/mondemand.app.src +++ b/src/mondemand.app.src @@ -4,7 +4,7 @@ { vsn, "4.1.1" }, { modules, [] }, { registered, [mondemand,mondemand_sup]}, - { applications, [kernel,stdlib,lwes]}, + { applications, [kernel,stdlib,lwes,inets]}, { env, [ { config_file,"/etc/mondemand/mondemand.conf"}]}, { mod, { mondemand_app, []} diff --git a/src/mondemand.erl b/src/mondemand.erl index d9543f5..18a4172 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -158,8 +158,11 @@ stringify (A) when is_atom (A) -> stringify (L) -> L. +send_trace (ProgId, Message, Context) + when is_tuple (Context) andalso element (1, Context) =:= dict -> + send_trace (ProgId, Message, dict:to_list (Context)); send_trace (ProgId, Message, Context) -> - Context2 = split_heavy_contexts (Context), + Context2 = mark_large_binaries (Context), case Context2 of Dict when is_tuple (Context) andalso element (1, Context) =:= dict -> case key_in_dict (?TRACE_ID_KEY, Dict) @@ -195,20 +198,12 @@ send_trace (ProgId, Message, Context) -> {error, context_format_not_recognized} end. +send_trace (ProgId, Owner, TraceId, Message, Context) + when is_tuple (Context) andalso element (1, Context) =:= dict -> + send_trace (ProgId, Owner, TraceId, Message, dict:to_list (Context)); send_trace (ProgId, Owner, TraceId, Message, Context) -> - Context2 = split_heavy_contexts (Context), + Context2 = mark_large_binaries (Context), case Context2 of - Dict when is_tuple (Context) andalso element (1, Context) =:= dict -> - send_event ( - #lwes_event { - name = ?TRACE_EVENT, - attrs = dict:store (?PROG_ID_KEY, ProgId, - dict:store (?OWNER_ID_KEY, Owner, - dict:store (?TRACE_ID_KEY, TraceId, - dict:store (?SRC_HOST_KEY, net_adm:localhost(), - dict:store (?MESSAGE_KEY, Message, - Dict))))) - }); List when is_list (Context) -> send_event ( #lwes_event { @@ -361,10 +356,10 @@ handle_call (_Request, _From, State) -> {reply, ok, State}. % send an event -handle_cast ({send, Event}, +handle_cast ({send, + Event = #lwes_event { name = EventName }}, State = #state { channel = Channel, http_config = HttpConfig}) -> - {lwes_event, EventName, Attrs} = Event, case EventName of "MonDemand::TraceMsg" -> Bin = lwes_event:to_binary(Event), @@ -398,46 +393,24 @@ send_event (Event) -> post_via_http(Bin, HttpConfig) -> case HttpConfig of undefined -> ok; - _ -> Endpoint = proplists:get_value("trace", HttpConfig), + _ -> Endpoint = proplists:get_value (trace, HttpConfig), httpc:request (post, {Endpoint, [], "", Bin}, [], []) end. -split_heavy_contexts (Context) when is_tuple (Context) -> - dict:from_list ( - split_heavy_contexts ( - dict:to_list (Context))); -split_heavy_contexts (Context) -> - lists:foldl ( +mark_large_binaries (Context) -> + lists:foldl( fun ({K, V}, A) - -> case V of - B when is_binary(B) -> A ++ split_chunk (K, V, 50000, 0); - O -> A ++ [{K, O}] + -> case iolist_size (V) of + N when N > 61440 + -> [ {?LWES_LARGE_BINARY, K, V} | A ]; + _ -> [ {K, V} | A ] end end, - [], - Context). - -split_chunk (_, <<>>, _, _) -> []; -split_chunk (FieldName, Bin, ChunkSize, PartIndex) - when size(Bin) =< ChunkSize -> - case PartIndex of - 0 -> [{FieldName, Bin}]; - _ -> [{part_name(FieldName, PartIndex), Bin}] - end; - -split_chunk (FieldName, Bin, ChunkSize, PartIndex) -> - { Chunk, Rest} = split_binary (Bin, ChunkSize), - [ {part_name(FieldName, PartIndex), Chunk} | - split_chunk (FieldName, Rest, ChunkSize, PartIndex + 1) ]. - -part_name (Name, Part) when is_binary (Name) -> - list_to_binary ( - part_name (binary_to_list (Name), Part)); -part_name (Name, Part) -> - Name ++ "-" ++ integer_to_list (Part). + [], + Context). to_string (In) when is_list (In) -> In; diff --git a/src/mondemand_app.erl b/src/mondemand_app.erl index 869291a..0cf3042 100644 --- a/src/mondemand_app.erl +++ b/src/mondemand_app.erl @@ -47,7 +47,7 @@ stop(_State) -> %% Description: Starts apps this depends on, then starts this %%-------------------------------------------------------------------- start() -> - [application:start(App) || App <- [sasl, lwes, mondemand]]. + [application:start(App) || App <- [sasl, lwes, inets, mondemand]]. %%-------------------------------------------------------------------- %%% Test functions From 4e05aef0b3fb7936f38f259efa9e3798ef36eba5 Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Mon, 3 Mar 2014 19:40:59 -0800 Subject: [PATCH 04/12] unused code --- src/mondemand.erl | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/src/mondemand.erl b/src/mondemand.erl index 18a4172..8d9805c 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -164,21 +164,6 @@ send_trace (ProgId, Message, Context) send_trace (ProgId, Message, Context) -> Context2 = mark_large_binaries (Context), case Context2 of - Dict when is_tuple (Context) andalso element (1, Context) =:= dict -> - case key_in_dict (?TRACE_ID_KEY, Dict) - andalso key_in_dict (?OWNER_ID_KEY, Dict) of - true -> - send_event ( - #lwes_event { - name = ?TRACE_EVENT, - attrs = dict:store (?PROG_ID_KEY, ProgId, - dict:store (?SRC_HOST_KEY, net_adm:localhost(), - dict:store (?MESSAGE_KEY, Message, - Dict))) - }); - false -> - {error, required_fields_not_set} - end; List when is_list (Context) -> case key_in_list (?TRACE_ID_KEY, List) andalso key_in_list (?OWNER_ID_KEY, List) of @@ -419,11 +404,6 @@ to_string (In) when is_atom (In) -> to_string (In) when is_integer (In) -> integer_to_list (In). -key_in_dict (Key, Dict) when is_list (Key) -> - dict:is_key (Key, Dict) - orelse dict:is_key (list_to_binary(Key), Dict) - orelse dict:is_key (list_to_atom(Key), Dict). - key_in_list (Key, List) when is_list (Key), is_list (List) -> proplists:is_defined (Key, List) orelse proplists:is_defined (list_to_binary(Key), List) From 2a912850a3a314a3791eff1c07e55f563aa9d7aa Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Tue, 4 Mar 2014 15:50:41 -0800 Subject: [PATCH 05/12] updating lwes version --- rebar.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rebar.config b/rebar.config index 9f9b859..379f920 100644 --- a/rebar.config +++ b/rebar.config @@ -5,7 +5,7 @@ {cover_print_enabled, true}. {deps, [ - {lwes, "2.3.1", + {lwes, "2.3.2", {git, "git://github.com/lwes/lwes-erlang.git", {branch, "feature/large-data-support"}} } ] From 4ff5ced7f3539142f52d2c5d8d41ee462e76b0a8 Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Tue, 4 Mar 2014 16:33:48 -0800 Subject: [PATCH 06/12] bumping the version --- src/mondemand.app.src | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mondemand.app.src b/src/mondemand.app.src index 30dea51..090afab 100644 --- a/src/mondemand.app.src +++ b/src/mondemand.app.src @@ -1,7 +1,7 @@ { application, mondemand, [ { description, "Erlang Mondemand Bindings." }, - { vsn, "4.1.1" }, + { vsn, "4.1.2" }, { modules, [] }, { registered, [mondemand,mondemand_sup]}, { applications, [kernel,stdlib,lwes,inets]}, From 248713f8725c7360d3c071a271a2c6aeda61192b Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Wed, 5 Mar 2014 16:17:30 -0800 Subject: [PATCH 07/12] changes corresponding to changed lwes --- src/mondemand.erl | 18 ++---------------- 1 file changed, 2 insertions(+), 16 deletions(-) diff --git a/src/mondemand.erl b/src/mondemand.erl index 8d9805c..28ea699 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -162,8 +162,7 @@ send_trace (ProgId, Message, Context) when is_tuple (Context) andalso element (1, Context) =:= dict -> send_trace (ProgId, Message, dict:to_list (Context)); send_trace (ProgId, Message, Context) -> - Context2 = mark_large_binaries (Context), - case Context2 of + case Context of List when is_list (Context) -> case key_in_list (?TRACE_ID_KEY, List) andalso key_in_list (?OWNER_ID_KEY, List) of @@ -187,8 +186,7 @@ send_trace (ProgId, Owner, TraceId, Message, Context) when is_tuple (Context) andalso element (1, Context) =:= dict -> send_trace (ProgId, Owner, TraceId, Message, dict:to_list (Context)); send_trace (ProgId, Owner, TraceId, Message, Context) -> - Context2 = mark_large_binaries (Context), - case Context2 of + case Context of List when is_list (Context) -> send_event ( #lwes_event { @@ -385,18 +383,6 @@ post_via_http(Bin, HttpConfig) -> [], []) end. -mark_large_binaries (Context) -> - lists:foldl( - fun ({K, V}, A) - -> case iolist_size (V) of - N when N > 61440 - -> [ {?LWES_LARGE_BINARY, K, V} | A ]; - _ -> [ {K, V} | A ] - end - end, - [], - Context). - to_string (In) when is_list (In) -> In; to_string (In) when is_atom (In) -> From a1b2215bc2e2bf100f66d1abf2cdbf135a0ff703 Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Wed, 5 Mar 2014 16:48:58 -0800 Subject: [PATCH 08/12] code review changes --- mondemand_client_dev.config | 2 +- src/mondemand.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/mondemand_client_dev.config b/mondemand_client_dev.config index d82e39a..7d7517a 100644 --- a/mondemand_client_dev.config +++ b/mondemand_client_dev.config @@ -1,7 +1,7 @@ [ { mondemand, [ { lwes_channel, { "127.0.0.1", 20502 } }, { http_endpoint, - [ {trace, "http://127.0.0.1:20602/tcp/trace"} ] + [ {trace, "http://127.0.0.1:20602/tcp"} ] } ] } diff --git a/src/mondemand.erl b/src/mondemand.erl index 28ea699..684290d 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -377,7 +377,7 @@ post_via_http(Bin, HttpConfig) -> case HttpConfig of undefined -> ok; _ -> Endpoint = proplists:get_value (trace, HttpConfig), - httpc:request + {ok, _ } = httpc:request (post, {Endpoint, [], "", Bin}, [], []) From 5def254e3f3c6b2cc7e455aa96c5311c34c07cfa Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Wed, 5 Mar 2014 18:13:10 -0800 Subject: [PATCH 09/12] updated change log and dependency versions --- ChangeLog | 4 ++++ rebar.config | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 22fdffb..22faa9f 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,7 @@ +Version 4.1.2 (vikramkadi) + * now able to post traces over 65kb + * updated the version of lwes dependency to support long strings + Version 4.1.1 (molinaro) * updated dependency on version of lwes diff --git a/rebar.config b/rebar.config index 379f920..8efdfa9 100644 --- a/rebar.config +++ b/rebar.config @@ -5,7 +5,7 @@ {cover_print_enabled, true}. {deps, [ - {lwes, "2.3.2", + {lwes, "2.4.0", {git, "git://github.com/lwes/lwes-erlang.git", {branch, "feature/large-data-support"}} } ] From b898688be3c83ff1c83e246483d9d45aa8b4eb6c Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Fri, 7 Mar 2014 11:02:38 -0800 Subject: [PATCH 10/12] upgrading the versions --- rebar.config | 2 +- src/mondemand.app.src | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/rebar.config b/rebar.config index 8efdfa9..38045e1 100644 --- a/rebar.config +++ b/rebar.config @@ -6,7 +6,7 @@ {deps, [ {lwes, "2.4.0", - {git, "git://github.com/lwes/lwes-erlang.git", {branch, "feature/large-data-support"}} + {git, "git://github.com/lwes/lwes-erlang.git", {tag, "2.4.0"}} } ] }. diff --git a/src/mondemand.app.src b/src/mondemand.app.src index 090afab..919ff43 100644 --- a/src/mondemand.app.src +++ b/src/mondemand.app.src @@ -1,7 +1,7 @@ { application, mondemand, [ { description, "Erlang Mondemand Bindings." }, - { vsn, "4.1.2" }, + { vsn, "4.2.0" }, { modules, [] }, { registered, [mondemand,mondemand_sup]}, { applications, [kernel,stdlib,lwes,inets]}, From 2d5bebf92f1ac97b985dd8770f8c98cd0b7d5b06 Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Sat, 8 Mar 2014 16:37:21 -0800 Subject: [PATCH 11/12] Change log --- ChangeLog | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ChangeLog b/ChangeLog index 22faa9f..e6cf043 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,4 +1,4 @@ -Version 4.1.2 (vikramkadi) +Version 4.2.0 (vikramkadi) * now able to post traces over 65kb * updated the version of lwes dependency to support long strings From 1a8c1ffa409c8fcf90fa61fa00176fefd79f477b Mon Sep 17 00:00:00 2001 From: Vikram Kadi Date: Tue, 11 Mar 2014 15:08:44 -0700 Subject: [PATCH 12/12] Using the macro instead of event name --- src/mondemand.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/mondemand.erl b/src/mondemand.erl index 684290d..87aabc6 100644 --- a/src/mondemand.erl +++ b/src/mondemand.erl @@ -344,7 +344,7 @@ handle_cast ({send, State = #state { channel = Channel, http_config = HttpConfig}) -> case EventName of - "MonDemand::TraceMsg" + ?TRACE_EVENT -> Bin = lwes_event:to_binary(Event), case size(Bin) of X when X > 65535 -> post_via_http (Bin, HttpConfig);