add [untyped] config for files that exist but should be untyped

Summary:
the `[ignore]` config section causes matching files to be ignored by the module resolver, which inherently makes them un-typechecked, and also unresolvable by `import` or `require`.

this diff introduces `[untyped]`, which causes a file to be ignored by the typechecker as if it had `noflow` in it, but NOT ignored by the module resolver. any matching file is skipped by flow (not even parsed, like other noflow files!), but can still be `require()`'d.

Fixes #3208
Fixes #869

Thanks to LegNeato for a similar PR in #4916. That PR proposes a `[silence]` section which still typechecks matching files but suppresses all errors. Whereas `[untyped]` treats the whole file as `any`, `[silence]` would treat only broken stuff as `any`, which is also useful.

Reviewed By: avikchaudhuri

Differential Revision: D6378529

fbshipit-source-id: a0fd1444824affb40f50f47d9863640205973779

src/commands/commandUtils.ml

@@ -309,6 +309,7 @@ let assert_version flowconfig =
 
 type flowconfig_params = {
   ignores: string list;
+  untyped: string list;
   includes: string list;
   libs: string list;
   (* We store raw_lint_severities as a string list instead of as a LintSettings.t so we
@@ -321,12 +322,13 @@ let list_of_string_arg = function
 | None -> []
 | Some arg_str -> Str.split (Str.regexp ",") arg_str
 
-let collect_flowconfig_flags main ignores_str includes_str lib_str lints_str =
+let collect_flowconfig_flags main ignores_str untyped_str includes_str lib_str lints_str =
   let ignores = list_of_string_arg ignores_str in
+  let untyped = list_of_string_arg untyped_str in
   let includes = list_of_string_arg includes_str in
   let libs = list_of_string_arg lib_str in
   let raw_lint_severities = list_of_string_arg lints_str in
-  main { ignores; includes; libs; raw_lint_severities; }
+  main { ignores; includes; libs; raw_lint_severities; untyped; }
 
 let file_options =
   let default_lib_dir ~no_flowlib tmp_dir =
@@ -380,7 +382,7 @@ let file_options =
     | [] -> config_libs
     | _ -> config_libs @ (List.map Path.make extras)
   in
-  fun ~root ~no_flowlib ~temp_dir ~includes ~ignores ~libs flowconfig ->
+  fun ~root ~no_flowlib ~temp_dir ~includes ~ignores ~libs ~untyped flowconfig ->
     let default_lib_dir =
       let no_flowlib = no_flowlib || FlowConfig.no_flowlib flowconfig in
       Some (default_lib_dir ~no_flowlib temp_dir)
@@ -389,13 +391,18 @@ let file_options =
       root
       (FlowConfig.ignores flowconfig)
       ignores in
+    let untyped = ignores_of_arg
+      root
+      (FlowConfig.untyped flowconfig)
+      untyped in
     let includes =
       includes
       |> List.rev_append (FlowConfig.includes flowconfig)
       |> includes_of_arg root in
     { Files.
       default_lib_dir;
       ignores;
+      untyped;
       includes;
       lib_paths = lib_paths ~root flowconfig libs;
       module_file_exts = FlowConfig.module_file_exts flowconfig;
@@ -409,6 +416,12 @@ let ignore_flag prev = CommandSpec.ArgSpec.(
     ~doc:"Specify one or more ignore patterns, comma separated"
 )
 
+let untyped_flag prev = CommandSpec.ArgSpec.(
+  prev
+  |> flag "--untyped" (optional string)
+    ~doc:"Specify one or more patterns, comma separated, for files to treat as untyped"
+)
+
 let include_flag prev = CommandSpec.ArgSpec.(
   prev
   |> flag "--include" (optional string)
@@ -437,6 +450,7 @@ let flowconfig_flags prev = CommandSpec.ArgSpec.(
   prev
   |> collect collect_flowconfig_flags
   |> ignore_flag
+  |> untyped_flag
   |> include_flag
   |> lib_flag
   |> lints_flag
@@ -626,8 +640,14 @@ let make_options ~flowconfig ~lazy_mode ~root (options_flags: Options_flags.t) =
   let open Options_flags in
   let file_options =
     let no_flowlib = options_flags.no_flowlib in
-    let { includes; ignores; libs; raw_lint_severities=_; } = options_flags.flowconfig_flags in
-    file_options ~root ~no_flowlib ~temp_dir ~includes ~ignores ~libs flowconfig
+    let {
+      includes;
+      ignores;
+      libs;
+      raw_lint_severities=_;
+      untyped;
+    } = options_flags.flowconfig_flags in
+    file_options ~root ~no_flowlib ~temp_dir ~includes ~ignores ~libs ~untyped flowconfig
   in
   let lint_severities = parse_lints_flag
     (FlowConfig.lint_severities flowconfig) options_flags.flowconfig_flags.raw_lint_severities

src/commands/config/flowConfig.ml

@@ -300,8 +300,10 @@ module Opts = struct
 end
 
 type config = {
-  (* file blacklist *)
+  (* completely ignored files (both module resolving and typing) *)
   ignores: string list;
+  (* files that should be treated as untyped *)
+  untyped: string list;
   (* non-root include paths *)
   includes: string list;
   (* library paths. no wildcards *)
@@ -325,6 +327,9 @@ end = struct
   let ignores o ignores =
     List.iter (fun ex -> (fprintf o "%s\n" ex)) ignores
 
+  let untyped o untyped =
+    List.iter (fun ex -> (fprintf o "%s\n" ex)) untyped
+
   let includes o includes =
     List.iter (fun inc -> (fprintf o "%s\n" inc)) includes
 
@@ -374,10 +379,18 @@ end = struct
          (string_of_kind kind)))
       strict_mode
 
+  let section_if_nonempty o header f = function
+    | [] -> ()
+    | xs ->
+      section_header o header;
+      f o xs;
+      fprintf o "\n"
+
   let config o config =
     section_header o "ignore";
     ignores o config.ignores;
     fprintf o "\n";
+    section_if_nonempty o "untyped" untyped config.untyped;
     section_header o "include";
     includes o config.includes;
     fprintf o "\n";
@@ -396,6 +409,7 @@ end
 
 let empty_config = {
   ignores = [];
+  untyped = [];
   includes = [];
   libs = [];
   lint_severities = LintSettings.default_severities;
@@ -445,6 +459,10 @@ let parse_ignores config lines =
   let ignores = trim_lines lines in
   { config with ignores; }
 
+let parse_untyped config lines =
+  let untyped = trim_lines lines in
+  { config with untyped; }
+
 let parse_options config lines =
   let open Opts in
   let (>>=) = Core_result.(>>=) in
@@ -886,6 +904,7 @@ let parse_section config ((section_ln, section), lines) =
   | "lints", _ -> parse_lints config lines
   | "strict", _ -> parse_strict config lines
   | "options", _ -> parse_options config lines
+  | "untyped", _ -> parse_untyped config lines
   | "version", _ -> parse_version config lines
   | _ -> error section_ln (spf "Unsupported config section: \"%s\"" section)
 
@@ -911,13 +930,15 @@ let read filename =
     |> List.filter is_not_comment in
   parse empty_config lines
 
-let init ~ignores ~includes ~libs ~options ~lints =
+let init ~ignores ~untyped ~includes ~libs ~options ~lints =
   let ignores_lines = List.map (fun s -> (1, s)) ignores in
+  let untyped_lines = List.map (fun s -> (1, s)) untyped in
   let includes_lines = List.map (fun s -> (1, s)) includes in
   let options_lines = List.map (fun s -> (1, s)) options in
   let lib_lines = List.map (fun s -> (1, s)) libs in
   let lint_lines = List.map (fun s -> (1, s)) lints in
   let config = parse_ignores empty_config ignores_lines in
+  let config = parse_untyped config untyped_lines in
   let config = parse_includes config includes_lines in
   let config = parse_options config options_lines in
   let config = parse_libs config lib_lines in
@@ -944,8 +965,10 @@ let restore (filename, config) = cache := Some (filename, config)
 
 (* Accessors *)
 
-(* file blacklist *)
+(* completely ignored files (both module resolving and typing) *)
 let ignores config = config.ignores
+(* files that should be treated as untyped *)
+let untyped config = config.untyped
 (* non-root include paths *)
 let includes config = config.includes
 (* library paths. no wildcards *)

src/commands/config/flowConfig.mli

@@ -12,6 +12,7 @@ val empty_config: config
 
 val init:
   ignores: string list ->
+  untyped: string list ->
   includes: string list ->
   libs: string list ->
   options: string list ->
@@ -23,8 +24,10 @@ val restore: string * config -> unit
 
 (* Accessors *)
 
-(* file blacklist *)
+(* completely ignored files (both module resolving and typing) *)
 val ignores: config -> string list
+(* files that should be treated as untyped *)
+val untyped: config -> string list
 (* non-root include paths *)
 val includes: config -> string list
 (* library paths. no wildcards *)

src/commands/configCommands.ml

@@ -40,6 +40,7 @@ module Init = struct
     | Some str -> Str.split (Str.regexp ";") str
     in
     let ignores = flowconfig_flags.CommandUtils.ignores in
+    let untyped = flowconfig_flags.CommandUtils.untyped in
     let includes = flowconfig_flags.CommandUtils.includes in
     let libs = flowconfig_flags.CommandUtils.libs in
     let lints = flowconfig_flags.CommandUtils.raw_lint_severities in
@@ -51,7 +52,7 @@ module Init = struct
       FlowExitStatus.(exit ~msg Invalid_flowconfig)
     end;
 
-    let config = FlowConfig.init ~ignores ~includes ~libs ~options ~lints in
+    let config = FlowConfig.init ~ignores ~untyped ~includes ~libs ~options ~lints in
 
     let out = Sys_utils.open_out_no_fail file in
     FlowConfig.write config out;

src/commands/genFlowFilesCommand.ml

@@ -55,6 +55,7 @@ let spec = {
     |> strip_root_flag
     |> ignore_flag
     |> include_flag
+    |> untyped_flag
     |> from_flag
     |> anon "src" (required string)
         ~doc:"The path to a file or directory to generate .js.flow files for"
@@ -85,7 +86,7 @@ let write_file strip_root root content perm src_file_path dest_file_path =
   Unix.close fd
 
 let main option_values root error_flags strip_root ignore_flag
-  include_flag from src out_dir () = (
+  include_flag untyped_flag from src out_dir () = (
   FlowEventLogger.set_from from;
   let src = expand_path src in
   let root = guess_root (
@@ -111,7 +112,7 @@ let main option_values root error_flags strip_root ignore_flag
         FlowExitStatus.exit ~msg FlowExitStatus.Commandline_usage_error
       );
 
-      let options = LsCommand.make_options ~root ~ignore_flag ~include_flag in
+      let options = LsCommand.make_options ~root ~ignore_flag ~include_flag ~untyped_flag in
       let _, libs = Files.init options in
       let next_files =
         LsCommand.get_ls_files ~root ~all:false ~options ~libs ~imaginary:false (Some src)

src/commands/lsCommand.ml

@@ -25,6 +25,7 @@ let spec = {
     |> strip_root_flag
     |> ignore_flag
     |> include_flag
+    |> untyped_flag
     |> root_flag
     |> json_flags
     |> from_flag
@@ -106,13 +107,15 @@ let rec iter_get_next ~f get_next =
       List.iter f result;
       iter_get_next ~f get_next
 
-let make_options ~root ~ignore_flag ~include_flag =
+let make_options ~root ~ignore_flag ~include_flag ~untyped_flag =
   let flowconfig = FlowConfig.get (Server_files_js.config_file root) in
   let temp_dir = FlowConfig.temp_dir flowconfig in
   let includes = CommandUtils.list_of_string_arg include_flag in
   let ignores = CommandUtils.list_of_string_arg ignore_flag in
+  let untyped = CommandUtils.list_of_string_arg untyped_flag in
   let libs = [] in
-  CommandUtils.file_options ~root ~no_flowlib:true ~temp_dir ~ignores ~includes ~libs flowconfig
+  CommandUtils.file_options flowconfig
+    ~root ~no_flowlib:true ~temp_dir ~ignores ~includes ~libs ~untyped
 
 (* The problem with Files.wanted is that it says yes to everything except ignored files and libs.
  * So implicitly ignored files (like files in another directory) pass the Files.wanted check *)
@@ -172,7 +175,7 @@ let get_next_append_const get_next const =
       ret
 
 let main
-  strip_root ignore_flag include_flag root_flag json pretty from all imaginary reason
+  strip_root ignore_flag include_flag untyped_flag root_flag json pretty from all imaginary reason
   input_file root_or_files () =
 
   let files_or_dirs = get_filenames_from_input ~allow_imaginary:true input_file root_or_files in
@@ -192,7 +195,7 @@ let main
       | _ -> None)
   ) in
 
-  let options = make_options ~root ~ignore_flag ~include_flag in
+  let options = make_options ~root ~ignore_flag ~include_flag ~untyped_flag in
   (* Turn on --no-flowlib by default, so that flow ls never reports flowlib files *)
   let options = { options with Files.default_lib_dir = None; } in
   let _, libs = Files.init options in

src/common/files.ml

@@ -10,6 +10,7 @@
 type options = {
   default_lib_dir: Path.t option;
   ignores: (string * Str.regexp) list;
+  untyped: (string * Str.regexp) list;
   includes: Path_matcher.t;
   lib_paths: Path.t list;
   module_file_exts: SSet.t;
@@ -19,6 +20,7 @@ type options = {
 
 let default_lib_dir options = options.default_lib_dir
 let ignores options = options.ignores
+let untyped options = options.untyped
 let includes options = options.includes
 let lib_paths options = options.lib_paths
 let module_file_exts options = options.module_file_exts
@@ -289,6 +291,15 @@ let is_ignored (options: options) =
     let path = Sys_utils.normalize_filename_dir_sep path in
     List.exists (fun rx -> Str.string_match rx path 0) list
 
+(* true if a file path matches an [untyped] entry in config *)
+let is_untyped (options: options) =
+  let list = List.map snd options.untyped in
+  fun path ->
+    (* On Windows, the path may use \ instead of /, but let's standardize the
+     * ignore regex to use / *)
+    let path = Sys_utils.normalize_filename_dir_sep path in
+    List.exists (fun rx -> Str.string_match rx path 0) list
+
 (* true if a file path matches an [include] path in config *)
 let is_included options f =
   Path_matcher.matches options.includes f

src/common/files.mli

@@ -10,6 +10,7 @@
 type options = {
   default_lib_dir: Path.t option;
   ignores: (string * Str.regexp) list;
+  untyped: (string * Str.regexp) list;
   includes: Path_matcher.t;
   lib_paths: Path.t list;
   module_file_exts: SSet.t;
@@ -19,6 +20,7 @@ type options = {
 
 val default_lib_dir: options -> Path.t option
 val ignores: options -> (string * Str.regexp) list
+val untyped: options -> (string * Str.regexp) list
 val includes: options -> Path_matcher.t
 val lib_paths: options -> Path.t list
 val module_file_exts: options -> SSet.t
@@ -38,6 +40,8 @@ val is_flow_file: options: options -> string -> bool
 
 (* true if a file path matches an [ignore] entry in config *)
 val is_ignored: options -> string -> bool
+(* true if a file path matches an [untyped] entry in config *)
+val is_untyped: options -> string -> bool
 (* true if a file path matches an [include] path in config *)
 val is_included: options -> string -> bool