Psv: move constants (max_row_count, buffer_size, etc.) into config

data/sea/52-psv.h

@@ -23,6 +23,11 @@ typedef struct {
     const size_t  facts_limit;   /* maximum number of facts per entity-attribute */
     const ibool_t has_drop; /* whether to write discarded facts to output or drop log */
 
+    /* constants */
+    const size_t psv_input_buffer_size;
+    const size_t psv_output_buffer_size;
+    const size_t psv_max_row_count;
+
 } psv_config_t;
 
 typedef struct {
@@ -67,7 +72,7 @@ typedef struct {
 
 
 /* forward declarations for functions, implemented by generated code */
-static ifleet_t * INLINE psv_alloc_fleet (iint_t max_chord_count);
+static ifleet_t * INLINE psv_alloc_fleet (const iint_t max_chord_count, const iint_t max_row_count);
 
 static void INLINE psv_collect_fleet (ifleet_t *fleet);
 
@@ -89,7 +94,8 @@ static ierror_loc_t INLINE psv_read_fact
   , const char   *time_ptr
   , const size_t  time_size
   , ifleet_t     *fleet
-  , const size_t  max_ent_attr_count);
+  , const size_t  facts_limit
+  , const iint_t  max_row_count );
 #else
 static ierror_loc_t INLINE psv_read_fact
   ( const char   *entity_ptr
@@ -99,7 +105,8 @@ static ierror_loc_t INLINE psv_read_fact
   , const char   *time_ptr
   , const size_t  time_size
   , ifleet_t     *fleet
-  , const size_t  max_ent_attr_count);
+  , const size_t  facts_limit
+  , const iint_t  max_row_count );
 #endif
 
 static ierror_msg_t INLINE psv_write_output
@@ -276,7 +283,7 @@ static ierror_loc_t psv_write_dropped_entity_cur (psv_state_t *s, const ierror_l
     return 0;
 }
 
-static ierror_loc_t psv_read_buffer (psv_state_t *s, const size_t facts_limit)
+static ierror_loc_t psv_read_buffer (psv_state_t *s, const size_t facts_limit, const iint_t max_row_count)
 {
     ierror_loc_t error;
 
@@ -417,11 +424,11 @@ static ierror_loc_t psv_read_buffer (psv_state_t *s, const size_t facts_limit)
         }
 
 #if ICICLE_PSV_INPUT_SPARSE
-        error = psv_read_fact (entity_ptr, entity_size, attrib_ptr, attrib_size, value_ptr, value_size, time_ptr, time_size, s->fleet, facts_limit);
+        error = psv_read_fact (entity_ptr, entity_size, attrib_ptr, attrib_size, value_ptr, value_size, time_ptr, time_size, s->fleet, facts_limit, max_row_count);
         if (error)
             goto on_error;
 #else
-        error = psv_read_fact (entity_ptr, entity_size, value_ptr, value_size, time_ptr, time_size, s->fleet, facts_limit);
+        error = psv_read_fact (entity_ptr, entity_size, value_ptr, value_size, time_ptr, time_size, s->fleet, facts_limit, max_row_count);
         if (error)
           goto on_error;
 #endif
@@ -450,7 +457,7 @@ static ierror_loc_t psv_read_whole_buffer (psv_config_t *cfg, psv_state_t *s)
     ierror_loc_t error      = 0;
 
     while (keep_going) {
-        error = psv_read_buffer (s, cfg->facts_limit);
+        error = psv_read_buffer (s, cfg->facts_limit, cfg->psv_max_row_count);
 
         if (error) {
             /* try to skip to the next entity, if there is no new entity
@@ -584,11 +591,11 @@ void psv_snapshot (piano_t *piano, psv_config_t *cfg)
         return;
     }
 
-    ifleet_t *fleet = psv_alloc_fleet (chord_file.max_chord_count);
+    ifleet_t *fleet = psv_alloc_fleet (chord_file.max_chord_count, cfg->psv_max_row_count);
 
-    char *input_ptr  = calloc (psv_input_buffer_size + 1, 1);
-    char *entity_cur = calloc (psv_input_buffer_size + 1, 1);
-    char *output_ptr = calloc (psv_output_buffer_size + 1, 1);
+    char *input_ptr  = calloc (cfg->psv_input_buffer_size + 1, 1);
+    char *entity_cur = calloc (cfg->psv_input_buffer_size + 1, 1);
+    char *output_ptr = calloc (cfg->psv_output_buffer_size + 1, 1);
 
     static const psv_state_t empty_state;
     psv_state_t state = empty_state;
@@ -597,7 +604,7 @@ void psv_snapshot (piano_t *piano, psv_config_t *cfg)
     state.input_ptr    = input_ptr;
     state.entity_cur   = entity_cur;
     state.output_start = output_ptr;
-    state.output_end   = output_ptr + psv_output_buffer_size - 1;
+    state.output_end   = output_ptr + cfg->psv_output_buffer_size - 1;
     state.output_ptr   = output_ptr;
     state.fleet        = fleet;
     state.output_fd    = output_fd;
@@ -612,7 +619,7 @@ void psv_snapshot (piano_t *piano, psv_config_t *cfg)
 
         size_t bytes_read = read ( input_fd
                                  , input_ptr  + input_offset
-                                 , psv_input_buffer_size - input_offset );
+                                 , cfg->psv_input_buffer_size - input_offset );
 
         if (bytes_read == psv_read_error) {
             cfg->error = ierror_msg_alloc ("error reading input", 0, 0);

data/sea/54-zebra.h

@@ -13,6 +13,7 @@ typedef struct {
     const char  *error;
     iint_t fact_count;
     iint_t entity_count;
+    const size_t output_buffer_size;
 
 } zebra_config_t;
 
@@ -189,11 +190,12 @@ zebra_state_t *zebra_alloc_state (piano_t *piano, zebra_config_t *cfg)
         max_chord_count = 1;
     }
 
-    ifleet_t *fleet = psv_alloc_fleet (max_chord_count);
-    char *output_ptr = calloc (psv_output_buffer_size + 1, 1);
+    /* max_row_count is unused */
+    ifleet_t *fleet = psv_alloc_fleet (max_chord_count, 0);
+    char *output_ptr = calloc (cfg->output_buffer_size + 1, 1);
 
     state->output_start = output_ptr;
-    state->output_end   = output_ptr + psv_output_buffer_size - 1;
+    state->output_end   = output_ptr + cfg->output_buffer_size - 1;
     state->output_ptr   = output_ptr;
     state->fleet        = fleet;
     state->output_fd    = output_fd;

icicle-compiler/icicle.cabal

@@ -220,6 +220,28 @@ test-suite test
 
   hs-source-dirs:
                        test
+  other-modules:
+                       -- Icicle.Test.Sea.Bindings
+
+  include-dirs:
+                       data/sea
+
+  install-includes:
+                       00-includes.h
+                       05-error.h
+                       06-segv.h
+                       20-simple.h
+                       21-time.h
+                       30-array.h
+                       31-buffer.h
+                       40-grisu2-powers.h
+                       41-grisu2.h
+                       42-text-conversion.h
+                       50-chord.h
+                       51-piano.h
+                       52-psv.h
+                       53-zebra-data.h
+                       54-zebra.h
 
   build-depends:
                        base                            >= 3          && < 5
@@ -235,7 +257,10 @@ test-suite test
                      , ambiata-jetski
                      , ambiata-x-eithert
                      , ambiata-x-file-embed
+                     , ambiata-anemone
+                     , ambiata-zebra
                      , bifunctors                      >= 4.2        && < 5.4
+                     , bindings-DSL                    >= 1.0.0      && <= 1.0.23
                      , aeson
                      , bytestring
                      , containers
@@ -274,6 +299,7 @@ test-suite test-io
                     , icicle-source
                     , icicle-core
                     , icicle
+                    , ambiata-zebra
                     , ambiata-disorder-core
                     , ambiata-disorder-corpus
                     , ambiata-p

icicle-compiler/main/icicle-make.hs

@@ -111,7 +111,7 @@ main =
         PsvOutputConfig mode PsvOutputSparse defaultOutputMissing
 
       format =
-        FormatPsv (psvDefaultConstants inputConfig outputConfig)
+        FormatPsv (PsvConfig inputConfig outputConfig)
 
       inputOptions =
         InputOpts AllowDupTime $ tombstonesOfDictionary loadedDictionary

icicle-compiler/main/icicle.hs

@@ -177,13 +177,9 @@ pChord :: Parser (Scope ())
 pChord =
   flag' (ScopeChord ()) (long "chord")
 
-defaultFactsLimit :: Int
-defaultFactsLimit =
-  1024 * 1024
-
 pLimit :: Parser Int
 pLimit =
-  flip option (long "facts-limit" <> value defaultFactsLimit) $
+  flip option (long "facts-limit" <> value defaultPsvFactsLimit) $
     readerAsk >>= \s -> case readMaybe s of
       Just i  -> return i
       Nothing -> readerError "--facts-limit NUMBER"
@@ -211,6 +207,7 @@ runCommand = \case
 
     end <- liftIO getCurrentTime
     let secs = realToFrac (end `diffUTCTime` start) :: Double
+
     liftIO (printf "icicle: compilation time = %.2fs\n" secs)
 
   IcicleQuery q -> do

icicle-compiler/src/Icicle/Command.hs

@@ -221,11 +221,11 @@ mkQueryFleet input chord source = do
   -- we actually only need to differentiate between psv/zebra, make this better
   let format = case inputFormat input of
         InputSparsePsv ->
-          FormatPsv $ psvDefaultConstants
+          FormatPsv $ PsvConfig
             (PsvInputConfig Chords PsvInputSparse)
             (PsvOutputConfig Chords PsvOutputSparse defaultOutputMissing)
         InputDensePsv ->
-          FormatPsv $ psvDefaultConstants
+          FormatPsv $ PsvConfig
             (PsvInputConfig Chords PsvInputSparse)
             (PsvOutputConfig Chords PsvOutputSparse defaultOutputMissing)
         InputZebra ->
@@ -272,13 +272,14 @@ loadDictionary path iformat oformat0 scope =
 
     oconfig =
       PsvOutputConfig mode oformat defaultOutputMissing
+
   in
     case iformat of
       InputSparsePsv -> do
         d <- firstEitherT IcicleDictionaryImportError $
           Toml.loadDictionary Source.optionSmallData Toml.ImplicitPrelude path
 
-        let f = FormatPsv $ psvDefaultConstants
+        let f = FormatPsv $ PsvConfig
                   (PsvInputConfig mode PsvInputSparse)
                   oconfig
 
@@ -288,7 +289,7 @@ loadDictionary path iformat oformat0 scope =
         (d, dense) <- firstEitherT IcicleDictionaryImportError $
           Toml.loadDenseDictionary Source.optionSmallData Toml.ImplicitPrelude path Nothing
 
-        let f = FormatPsv $ psvDefaultConstants
+        let f = FormatPsv $ PsvConfig
                   (PsvInputConfig mode (PsvInputDense dense (denseSelectedFeed dense)))
                   oconfig
 
@@ -338,12 +339,12 @@ runPsvQuery b = do
       output     = queryOutputPath   b
       dropPath   = queryDropPath     b
       chordPath  = queryChordPath    b
-      limit      = queryFactsLimit   b
       discard    = queryUseDrop b
+      constants  = defaultPsvConstants { psvFactsLimit = queryFactsLimit b }
 
   start <- liftIO getCurrentTime
   stats <- firstEitherT IcicleSeaError
-         $ seaPsvSnapshotFilePath fleet input output dropPath chordPath limit discard
+         $ seaPsvSnapshotFilePath fleet input output dropPath chordPath discard constants
   end   <- liftIO getCurrentTime
   size  <- liftIO (withFile input ReadMode hFileSize)
 

icicle-compiler/src/Icicle/Sea/Eval.hs

@@ -80,11 +80,12 @@ seaZebraSnapshotFd :: SeaFleet ZebraState
                    -> Maybe Posix.Fd
                    -> EitherT SeaError IO ZebraStats
 seaZebraSnapshotFd fleet input output mchords = do
-  withWords 6 $ \pState -> do
+  withWords 7 $ \pState -> do
   input_path <- liftIO $ newCString input
   pokeWordOff pState 0 input_path
   pokeWordOff pState 1 output
   pokeWordOff pState 2 (fromMaybe 0 mchords)
+  pokeWordOff pState 6 (defaultPsvOutputBufferSize)
 
   sfSnapshot fleet pState
 
@@ -110,10 +111,10 @@ seaPsvSnapshotFilePath :: SeaFleet PsvState
                        -> FilePath
                        -> FilePath
                        -> Maybe FilePath
-                       -> Int
                        -> FlagUseDrop
+                       -> PsvConstants
                        -> EitherT SeaError IO PsvStats
-seaPsvSnapshotFilePath fleet input output dropped mchords limit discard = do
+seaPsvSnapshotFilePath fleet input output dropped mchords discard conf = do
   bracketEitherT' (liftIO $ Posix.openFd input Posix.ReadOnly Nothing Posix.defaultFileFlags)
                   (liftIO . Posix.closeFd) $ \ifd -> do
   bracketEitherT' (liftIO $ Posix.createFile output (Posix.CMode 0O644))
@@ -122,7 +123,7 @@ seaPsvSnapshotFilePath fleet input output dropped mchords limit discard = do
                   (liftIO . Posix.closeFd) $ \dfd -> do
   bracketEitherT' (liftIO $ maybeOpen mchords)
                   (liftIO . maybeClose) $ \mcfd -> do
-  seaPsvSnapshotFd fleet ifd ofd dfd mcfd limit discard
+  seaPsvSnapshotFd fleet ifd ofd dfd mcfd discard conf
 
 
 
@@ -131,18 +132,21 @@ seaPsvSnapshotFd :: SeaFleet PsvState
                  -> Posix.Fd
                  -> Posix.Fd
                  -> Maybe Posix.Fd
-                 -> Int
                  -> FlagUseDrop
+                 -> PsvConstants
                  -> EitherT SeaError IO PsvStats
-seaPsvSnapshotFd fleet input output dropped mchords limit discard =
-  withWords 9 $ \pState -> do
-
-  pokeWordOff pState 0 input
-  pokeWordOff pState 1 output
-  pokeWordOff pState 2 dropped
-  pokeWordOff pState 3 (fromMaybe 0 mchords)
-  pokeWordOff pState 7 limit
-  pokeWordOff pState 8 (discard == FlagUseDropFile)
+seaPsvSnapshotFd fleet input output dropped mchords discard conf =
+  withWords 12 $ \pState -> do
+
+  pokeWordOff pState 0  input
+  pokeWordOff pState 1  output
+  pokeWordOff pState 2  dropped
+  pokeWordOff pState 3  (fromMaybe 0 mchords)
+  pokeWordOff pState 7  (psvFactsLimit conf)
+  pokeWordOff pState 8  (discard == FlagUseDropFile)
+  pokeWordOff pState 9  (psvInputBufferSize conf)
+  pokeWordOff pState 10 (psvOutputBufferSize conf)
+  pokeWordOff pState 11 (psvMaxRowCount conf)
 
   sfSnapshot fleet pState
 

icicle-compiler/src/Icicle/Sea/Eval/Base.hs

@@ -302,19 +302,13 @@ codeOfPrograms input attributes programs = do
 
   case input of
     NoInput -> do
-      let consts = seaOfDefaultConstants
       pure . textOfDoc . vsep
         $ [ "#define ICICLE_NO_INPUT 1"
-          , consts
           , seaPreamble
           , defs
           ] <> progs
     HasInput format opts _ -> do
       doc <- seaOfDriver format opts attributes states
-      let cnst = case format of
-                   FormatPsv conf
-                     -> seaOfConstants conf
-                   _ -> seaOfDefaultConstants
       let def  = case format of
                    FormatPsv conf
                      -> vsep [ defOfPsvInput (psvInputConfig conf), defOfPsvOutput (psvOutputConfig conf) ]
@@ -323,7 +317,7 @@ codeOfPrograms input attributes programs = do
                              , "#define ICICLE_ZEBRA 1"
                              ]
 
-      pure . textOfDoc . vsep $ [def, cnst, seaPreamble, defs] <> progs <> ["", doc]
+      pure . textOfDoc . vsep $ [def, seaPreamble, defs] <> progs <> ["", doc]
 
 textOfDoc :: Doc -> Text
 textOfDoc doc = T.pack (displayS (renderPretty 0.8 80 (pretty doc)) "")