Merge pull request #4 from carlwitt/dev

First draft of kickstart profiling.

.gitignore

@@ -12,3 +12,5 @@ rel/example_project
 doc
 _build
 rebar.lock
+
+*.DS_Store

rebar.config

@@ -5,3 +5,6 @@
                         {branch, "master"}}}
        ]}.
 {escript_incl_apps, [getopt]}.
+{edoc_opts, [
+		{private, true}
+	]}.

src/effi.erl

@@ -16,6 +16,12 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 
+%% @doc The standalone application entry point is {@link main/1}. 
+%% The create_port callback defined here is an abstract way to execute child 
+%% processes in foreign languages. 
+%% There are two foreign language interfaces, both implementing this callback,
+%% {@link effi_script} (e.g., Perl, Python) and {@link effi_interact} (e.g., Bash, R).
+
 %% @author Jörgen Brandt <[email protected]>
 
 
@@ -36,10 +42,11 @@
 %% Callback definitions
 %% ------------------------------------------------------------
 
--callback create_port( Lang, Script, Dir ) -> {port(), string()}
+-callback create_port( Lang, Script, Dir, Prof ) -> {port(), string()}
 when Lang   :: atom(),
      Script :: string(),
-     Dir    :: string().
+     Dir    :: string(),
+     Prof :: effi_profiling:profilingsettings().
 
 %% ------------------------------------------------------------
 %% Type definitions
@@ -61,14 +68,14 @@ when Lang   :: atom(),
 %% API export
 %% ------------------------------------------------------------
 
--export( [check_run/5, main/1] ).
+-export( [check_run/6, main/1, get_optspec_lst/0] ).
 
 %% ------------------------------------------------------------
 %% API functions
 %% ------------------------------------------------------------
 
 %% main/1
-%
+%% @doc Parses command line arguments and processes the request file using {@link runscript/4}.
 -spec main( ArgList::[string()] ) -> ok.
 
 main( [] ) ->
@@ -91,29 +98,33 @@ main( CmdLine ) ->
                 false ->
                   {dir, Dir} = lists:keyfind( dir, 1, OptList ),
                   {refactor, Refactor} = lists:keyfind( refactor, 1, OptList ),
-                  runscript( Dir, Refactor, NonOptList )
+                  Prof = effi_profiling:get_profiling_settings_from_commandline_args(OptList, NonOptList),
+                  runscript( Dir, Refactor, Prof, NonOptList )
               end
           end
       end;
     {error, {Reason, Data}} ->
       error( {Reason, Data} )
   end.
 
-%% check_run/5
-%
--spec check_run( Lam, Fa, R, Dir, LibMap ) -> result()
+%% check_run/6
+%% @doc Tries to process a parsed request file (Lam, Fa, R, LibMap) using {@link run/5} and handles possible failures.
+%% Also measures the execution time. If successful, returns a summary, as generated by {@link get_summary/10}.
+-spec check_run( Lam, Fa, R, Dir, LibMap, Prof ) -> result()
 when Lam    :: lam(),
      Fa     :: #{string() => [str()]},
      R      :: pos_integer(),
      Dir    :: string(),
-     LibMap :: #{atom() => [string()]}.
+     LibMap :: #{atom() => [string()]},
+     Prof :: effi_profiling:profilingsettings().
 
-check_run( Lam, Fa, R, Dir, LibMap )
+check_run( Lam, Fa, R, Dir, LibMap, Prof )
 when is_tuple( Lam ),
      is_map( Fa ),
      is_integer( R ), R > 0,
      is_list( Dir ),
-     is_map( LibMap ) ->
+     is_map( LibMap ),
+     is_tuple(Prof) ->
 
   % take start time
   Tstart = trunc( os:system_time()/1000000 ),
@@ -129,7 +140,7 @@ when is_tuple( Lam ),
     []    ->
 
       % run
-      case run( Lam, Fa, Dir, LibMap ) of
+      case run( Lam, Fa, Dir, LibMap, Prof ) of
         {failed, script_error, Data} -> {failed, script_error, R, Data};
         {finished, RMap, Out}        ->
 
@@ -146,33 +157,42 @@ when is_tuple( Lam ),
               InputSizeMap = gather_file_size( Li, Fa, Dir ),
               OutputSizeMap = gather_file_size( Lo, RMap, Dir ),
 
+              % read the contents of the generated profiling file
+			        ProfilingResult = case file:read_file( effi_profiling:out_file( Prof ) ) of
+		            {error, _R1} -> <<>>;
+			          {ok, X}     -> X
+			        end,
+              % delete the profiling file
+              file:delete( effi_profiling:out_file( Prof ) ),
+
               % generate summary
               {finished, get_summary( Lam, Fa, R, RMap, Out, Tstart, Tdur,
-                                      InputSizeMap, OutputSizeMap )}
+                                      InputSizeMap, OutputSizeMap, ProfilingResult )}
           end
       end
   end.
 
-
-%% ------------------------------------------------------------
-%% Internal functions
-%% ------------------------------------------------------------
-
-%% print_usage/0
-%
-print_usage() -> getopt:usage( get_optspec_lst(), "effi", "<requestfile> <summaryfile>" ).
-
 %% opt_spec_list/0
-%
+%% @doc Returns the command line parameters that effi can parse, in a format that the getopt module understands. 
 get_optspec_lst() ->
   [
    {version,  $v, "version",  undefined,         "Show Effi version"},
    {help,     $h, "help",     undefined,         "Show command line options"},
    {cite,     $c, "cite",     undefined,         "Show Bibtex entry for citation"},
    {dir,      $d, "dir",      {string, "."},     "Working directory"},
-   {refactor, $r, "refactor", {boolean, false},  "Refactor output files"}
+   {refactor, $r, "refactor", {boolean, false},  "Refactor output files"},
+   {profiling, $p, "profiling", {boolean, false}, "Profile the process using the Pegasus Kickstart tool"},
+   {profile_file, $x, "profile-out", {string, "<requestfile>_profile.xml"}, "Output file for profiling results"}
   ].
 
+%% ------------------------------------------------------------
+%% Internal functions
+%% ------------------------------------------------------------
+
+%% print_usage/0
+%
+print_usage() -> getopt:usage( get_optspec_lst(), "effi", "<requestfile> <summaryfile>" ).
+
 %% print_bibtex_entry/0
 %
 print_bibtex() -> io:format( "~n~s~n~n", [get_bibtex()] ).
@@ -214,9 +234,9 @@ get_banner() ->
 print_vsn() -> io:format( "~s~n", [?VSN] ).
 
 
-%% runscript/3
-%
-runscript( Dir, Refactor, [RequestFile, SumFile] ) ->
+%% runscript/4
+%% @doc Parses a request file, processes it using {@link check_run/6} and writes a summary to a file.
+runscript( Dir, Refactor, ProfilingSetting, [RequestFile, SumFile] ) ->
 
   % read script from file
   B = case file:read_file( RequestFile ) of
@@ -232,7 +252,7 @@ runscript( Dir, Refactor, [RequestFile, SumFile] ) ->
   end,
 
   % run script
-  Summary = case check_run( Lam, Fa, R, Dir, LibMap ) of
+  Summary = case check_run( Lam, Fa, R, Dir, LibMap, ProfilingSetting ) of
 
               {failed, script_error, R, {ActScript, Out}} ->
 
@@ -286,12 +306,13 @@ runscript( Dir, Refactor, [RequestFile, SumFile] ) ->
 
 
 
-runscript( _Dir, _Refactor, NonOptList ) ->
+runscript( _Dir, _Refactor, _DoProfiling, NonOptList ) ->
   error( {request_and_summary_expected, NonOptList} ).
 
-%% get_summary/5
-%
--spec get_summary( Lam, Fa, R, Ret, Out, Tstart, Tdur, InSizeMap, OutSizeMap ) -> #{atom() => term()}
+%% get_summary/10
+%% @doc takes information about an invocation and returns it as a map.
+%% If the atom none is given for ProfilingResults, the results map will not contain 
+-spec get_summary( Lam, Fa, R, Ret, Out, Tstart, Tdur, InSizeMap, OutSizeMap, ProfilingResults ) -> #{atom() => term()}
 when Lam    :: lam(),
      Fa     :: #{string => [str()]},
      R      :: pos_integer(),
@@ -300,9 +321,10 @@ when Lam    :: lam(),
      Tstart :: integer(),
      Tdur   :: integer(),
      InSizeMap :: #{string() => pos_integer()},
-     OutSizeMap :: #{string() => pos_integer()}.
+     OutSizeMap :: #{string() => pos_integer()},
+     ProfilingResults :: binary().
 
-get_summary( Lam, Fa, R, Ret, Out, Tstart, Tdur, InSizeMap, OutSizeMap )
+get_summary( Lam, Fa, R, Ret, Out, Tstart, Tdur, InSizeMap, OutSizeMap, ProfilingResults )
 when is_tuple( Lam ), is_map( Fa ), is_map( Ret ), is_list( Out ),
      is_integer( Tstart ), Tstart >= 0, is_integer( Tdur ), Tdur >= 0,
      is_integer( R ), R > 0,
@@ -318,8 +340,8 @@ when is_tuple( Lam ), is_map( Fa ), is_map( Ret ), is_list( Out ),
     out      => Out,
     state    => ok,
     out_size_map => OutSizeMap,
-    in_size_map  => InSizeMap}.
-
+    in_size_map  => InSizeMap,
+    profiling_results => ProfilingResults}.
 
 gather_file_size( ParamLst, Fa, Dir ) ->
 
@@ -356,47 +378,56 @@ acc_file( {str, File}, Acc, Dir ) ->
     true  -> Acc
   end.
 
--spec run( Lam, Fa, Dir, LibMap ) -> Result
+-spec run( Lam, Fa, Dir, LibMap, Prof ) -> Result
 when Lam    :: lam(),
      Fa     :: #{string() => [str()]},
      Dir    :: string(),
      LibMap :: #{atom() => [string()]},
+     Prof :: effi_profiling:profilingsettings(),
      Result :: {finished, #{string() => [str()]}, [binary()]}
              | {failed, script_error, {iolist(), [binary()]}}.
 
 
-%% run/4
-%
-run( Lam, Fa, Dir, LibMap )
+%% run/5
+%% @doc Creates a port using {@link create_port/5} and listens to it, i.e., collects and parses the output from the stdout.
+%% Returns a tuple containing a status message (e.g., finished, failed) a result map and the program output.
+run( Lam, Fa, Dir, LibMap, Prof )
 when is_tuple( Lam ),
      is_map( Fa ),
      is_list( Dir ),
-     is_map( LibMap ) ->
+     is_map( LibMap ),
+     is_tuple(Prof) ->
 
   % create port
-  {Port, ActScript} = create_port( Lam, Fa, Dir, LibMap ),
+  {Port, ActScript} = create_port( Lam, Fa, Dir, LibMap, Prof ),
 
   % receive result
   listen_port( Port, ActScript ).
 
 
-%% create_port/3
-%
--spec create_port( Lam, Fa, Dir, LibMap ) -> {port(), string()}
+%% create_port/5
+%% @doc This is a wrapper for the language specific port creation function.
+%% Hands over the name of the language module (e.g., effi_python), the script to execute,
+%% the working directory and the profiling options to the language specific create_port/4 function.
+-spec create_port( Lam, Fa, Dir, LibMap, Prof ) -> {port(), string()}
 when Lam    :: lam(),
      Fa     :: #{string() => [str()]},
      Dir    :: string(),
-     LibMap :: #{atom() => [string()]}.
+     LibMap :: #{atom() => [string()]},
+     Prof :: effi_profiling:profilingsettings().
 
-create_port( Lam, Fa, Dir, LibMap )
+create_port( Lam, Fa, Dir, LibMap, Prof )
 when is_tuple( Lam ),
      is_map( Fa ),
      is_list( Dir ),
-     is_map( LibMap ) ->
+     is_map( LibMap ),
+     is_tuple(Prof) ->
 
   {lam, _Line, _LamName, Sign, Body} = Lam,
   {forbody, Lang, Script} = Body,
   {sign, Lo, Li} = Sign,
+
+  % the module to call, depends on the language, e.g., effi_python
   Mod = list_to_atom("effi_" ++ atom_to_list(Lang)),
 
   % get Foreign Function Interface type
@@ -426,12 +457,12 @@ when is_tuple( Lam ),
   Script2 = io_lib:format( "~s~n~s~n~s~n~s~n", [LibPath, Assign, Script1, Suffix] ),
 
   % run script
-  {_Port, _ActScript} = apply( FfiType, create_port, [Mod, Script2, Dir] ).
+  {_Port, _ActScript} = apply( FfiType, create_port, [Mod, Script2, Dir, Prof] ).
 
 
 
 %% listen_port/2
-%
+%% @doc Processes the output of the child process (in a foreign language) and builds a result map from it.
 listen_port( Port, ActScript ) ->
   listen_port( Port, ActScript, <<>>, #{}, [] ).
 
@@ -497,7 +528,6 @@ listen_port( Port, ActScript, LineAcc, ResultAcc, OutAcc ) ->
 
   end.
 
-
 %% =============================================================================
 %% Unit Tests
 %% =============================================================================
@@ -514,11 +544,10 @@ greet_bash_test_() ->
   Body     = {forbody, bash, Script},
   Lam      = {lam, 12, "greet", Sign, Body},
   Fa       = #{"person" => [{str, "Jorgen"}]},
-
-  {finished, ResultMap, _} = run( Lam, Fa, Dir, #{} ),
-
+  Prof     = effi_profiling:get_profiling_settings(false, "profile.xml"),
+  {finished, ResultMap, _} = run( Lam, Fa, Dir, #{}, Prof ),
+    
   Result = maps:get( "out", ResultMap ),
-
   ?_assertEqual( [{str, "Hello Jorgen"}], Result ).
 
 -endif.

src/effi_interact.erl

@@ -16,6 +16,9 @@
 %% See the License for the specific language governing permissions and
 %% limitations under the License.
 
+%% @doc Prototype for interactively interpreted languages, (e.g., Python).
+%% As opposed to executing scripts directly ({@link effi_script}).
+
 %% @author Jörgen Brandt <[email protected]>
 
 
@@ -49,24 +52,28 @@
 %% Callback function exports
 %% ------------------------------------------------------------
 
--export( [create_port/3] ).
+-export( [create_port/4] ).
 
 
 %% ------------------------------------------------------------
 %% Callback functions
 %% ------------------------------------------------------------
 
-%% create_port/3
+%% create_port/4
 %
-create_port( Mod, Script, Dir )
+create_port( Mod, Script, Dir, Prof )
 
 when is_atom( Mod ),
      is_list( Script ),
-     is_list( Dir ) ->
+     is_list( Dir ),
+     is_tuple( Prof ) ->
 
   % get interpreter
   Interpreter = apply( Mod, interpreter, [] ),
 
+  % get dynamic instrumentation wrapper
+  ProfilingWrapper = effi_profiling:wrapper_call( Prof ),
+
   % get prefix
   Prefix = apply( Mod, prefix, [] ),
 
@@ -77,7 +84,11 @@ when is_atom( Mod ),
   ActScript = string:join( [Prefix, Script, Suffix, ""], "\n" ),
 
   % run ticket
-  Port = open_port( {spawn, Interpreter},
+  Command = case effi_profiling:is_on( Prof ) of 
+    true -> string:join( [ProfilingWrapper, Interpreter], " " ); 
+    false -> Interpreter 
+  end,
+  Port = open_port( {spawn, Command},
                     [exit_status,
                      stderr_to_stdout,
                      binary,