Skip to content

Commit

Permalink
Use a local tempfile for preprocessed output.
Browse files Browse the repository at this point in the history
Preprocessed output tends to be quite large, often multiple megabytes.
Rather than transferring it over the RPC interface, use a tempfile so
that the server doesn't use a lot of memory uneccessarily.

This updates nelhage#45.
  • Loading branch information
jpeach committed Jun 19, 2021
1 parent 28e5289 commit 5730e57
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 2 deletions.
11 changes: 11 additions & 0 deletions cmd/llamacc/args.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,17 @@ func (c *Compilation) RemoteCompiler(cfg *Config) string {
return "cc"
}

// LanguageExt returns the file extension for the current language.
func (c *Compilation) LanguageExt() string {
for k, v := range extLangs {
if v == c.Language {
return k
}
}

panic("unknown language extension")
}

type Flags struct {
MD bool
MMD bool
Expand Down
18 changes: 16 additions & 2 deletions cmd/llamacc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,16 @@ func buildLocalPreprocess(ctx context.Context, client *daemon.Client, cfg *Confi
return fmt.Errorf("find %s: %w", comp.LocalCompiler(cfg), err)
}

tmp, err := ioutil.TempFile("", fmt.Sprintf("llamacc-*%s", comp.LanguageExt()))
if err != nil {
return err
}

defer func() {
os.Remove(tmp.Name())
tmp.Close()
}()

var preprocessed bytes.Buffer
{
var preprocessor exec.Cmd
Expand All @@ -197,8 +207,9 @@ func buildLocalPreprocess(ctx context.Context, client *daemon.Client, cfg *Confi
preprocessor.Args = append(preprocessor.Args, comp.LocalArgs...)
if !cfg.FullPreprocess {
preprocessor.Args = append(preprocessor.Args, "-fdirectives-only")

}
preprocessor.Args = append(preprocessor.Args, "-E", "-o", "-", comp.Input)
preprocessor.Args = append(preprocessor.Args, "-E", "-o", tmp.Name(), comp.Input)
preprocessor.Stdout = &preprocessed
preprocessor.Stderr = os.Stderr
if cfg.Verbose {
Expand All @@ -212,6 +223,9 @@ func buildLocalPreprocess(ctx context.Context, client *daemon.Client, cfg *Confi

args := daemon.InvokeWithFilesArgs{
Function: cfg.Function,
Files: []files.Mapped{
remap(tmp.Name(), wd),
},
Outputs: []files.Mapped{
{
Local: files.LocalFile{Path: path.Join(wd, comp.Output)},
Expand All @@ -226,7 +240,7 @@ func buildLocalPreprocess(ctx context.Context, client *daemon.Client, cfg *Confi
if !cfg.FullPreprocess {
args.Args = append(args.Args, "-fdirectives-only", "-fpreprocessed")
}
args.Args = append(args.Args, "-x", comp.PreprocessedLanguage, "-o", comp.Output, "-")
args.Args = append(args.Args, "-x", comp.PreprocessedLanguage, "-o", comp.Output, toRemote(tmp.Name(), wd))

out, err := client.InvokeWithFiles(&args)
if err != nil {
Expand Down

0 comments on commit 5730e57

Please sign in to comment.