Skip to content

Commit

Permalink
feat: fixes bug when writing to many files concurrently
Browse files Browse the repository at this point in the history
Basically there's a bug in the vectorized i/o which I think may be
behind the problems I've seen with Blink sending POST bodies.

This goes back to regular write-ops for non-vectorized inputs (which
should be plenty!)
  • Loading branch information
leostera committed Nov 13, 2024
1 parent 310a486 commit 47c529c
Show file tree
Hide file tree
Showing 5 changed files with 80 additions and 2 deletions.
17 changes: 15 additions & 2 deletions packages/riot-stdlib/file.ml
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,15 @@ end

let to_reader t = Rio.Reader.of_read_src (module Read) t

let read_to_string path =
if not (exists path) then Error (`File_not_found path)
else
let file = open_read path in
let reader = to_reader file in
let buf = Buffer.create 512 in
let* _ = Rio.read_to_end reader ~buf in
Ok (Buffer.contents buf)

module Write = struct
type t = write_file

Expand All @@ -64,10 +73,14 @@ module Write = struct
| Error err -> Error err

let write t ~buf =
let bufs = Rio.Iovec.from_string buf in
write_owned_vectored t ~bufs
File.write t.fd ~pos:0 ~len:(String.length buf) (Bytes.unsafe_of_string buf)

let flush _t = Ok ()
end

let to_writer t = Rio.Writer.of_write_src (module Write) t

let write ?permissions path ~content:buf =
let file = open_write ?permissions path in
let dst = to_writer file in
Rio.write_all dst ~buf
1 change: 1 addition & 0 deletions packages/riot-stdlib/telemetry_app.ml
Original file line number Diff line number Diff line change
Expand Up @@ -25,5 +25,6 @@ module Dispatcher = struct
end

let start () = Dispatcher.start_link ()

let emit = Dispatcher.emit
let attach = Telemetry.attach
8 changes: 8 additions & 0 deletions packages/riot/riot.mli
Original file line number Diff line number Diff line change
Expand Up @@ -667,6 +667,14 @@ module File : sig
val to_reader : read_file -> read_file IO.Reader.t
val to_writer : write_file -> write_file IO.Writer.t
val exists : string -> bool

val write : ?permissions:int -> string -> content:string -> (unit, [> `Noop ]) IO.io_result
val read_to_string :
string ->
( string,
[> `File_not_found of string
] )
IO.io_result
end

module Net : sig
Expand Down
50 changes: 50 additions & 0 deletions test/concurrent_file_reads.ml
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
open Riot

type Message.t += ContentMatch of string | Error of string

let () =
Riot.run_with_status ~on_error:(fun _ -> 1) @@ fun () ->
let _ = Logger.start () in
Logger.set_log_level (Some Debug);
let main = self () in
let proc_count = 10_000 in
let _ =
spawn (fun () ->
for id = 0 to proc_count do
spawn (fun () ->
let dir = Filename.temp_dir "_riot_test" (Int.to_string id) in
Sys.chdir dir;
let filename = dir ^ "/" ^ "test_" ^ Int.to_string id in
let content = filename in
let () = File.write filename ~content |> Result.get_ok in
let msg =
match File.read_to_string filename with
| Ok actual ->
if String.equal actual content then ContentMatch filename
else
Error
(Format.sprintf
"%d: the content did not match! %S != %S\n\
filename: %s" id actual content filename)
| Error (`File_not_found file) ->
let cwd = Sys.getcwd () in
Error
(Format.sprintf
"%d: file not found: %S in %S because cwd is %S" id
file dir cwd)
| Error (#IO.io_error as e) ->
let err = Format.asprintf "%d: error: %a" id IO.pp_err e in
Error err
in
send main msg)
|> ignore
done)
in

for _i = 0 to proc_count do
match receive_any () with
| ContentMatch _filename -> ()
| Error str -> failwith str
| _ -> ()
done;
Ok 0
6 changes: 6 additions & 0 deletions test/dune
Original file line number Diff line number Diff line change
Expand Up @@ -205,3 +205,9 @@
(name terminate_when_main_terminates_test)
(modules terminate_when_main_terminates_test)
(libraries riot))

(test
(package riot)
(name concurrent_file_reads)
(modules concurrent_file_reads)
(libraries riot))

0 comments on commit 47c529c

Please sign in to comment.