VEModel test and walkthrough updates for exporters

sources/framework/VEModel/inst/advanced/04-extract.R

@@ -367,7 +367,7 @@ db.connection <- file.path(results$resultsPath,mwr$setting("OutputDir"),"my-db.s
 # if ( file.exists(db.connection) ) unlink(db.connection)
 # Alternatively, add overwrite=TRUE to dbWriteTable below
 
-mydb <- dbConnect(RSQLite::SQLite(), db.connection)
+mydb <- DBI::dbConnect(RSQLite::SQLite(), db.connection)
 
 # Put the extracted data into the new database
 for ( ve.table in names(extracted.df) ) {

sources/framework/VEModel/inst/walkthrough/01-install.R

@@ -1,4 +1,4 @@
-### install.R
+### 01-install.R
 #   Walk through model installation
 
 # Load VEModel package (in effect, the visioneval environment)
@@ -67,7 +67,7 @@ rpat <- installModel("VERPAT",modelPath="MYRPAT",confirm=FALSE) # VERPAT base va
 
 # See what we've got
 message('\nExpect to see: "MYRPAT", "VERSPM-base", "VERSPM-pop", and "VERSPM-run"')
-print(dir("models"))
+openModel()
 
 # Installed these models:
 # Once a model is loaded you can 'print' it to get information about it
@@ -84,7 +84,7 @@ print(dir("models"))
 
 # And you can get rid of the extra model like this (careful that you're deleting the right one!)
 # It's probably better to use File explorer (Windows) or Finder (Mac)
-unlink("models/VERSPM-base(2)",recursive=TRUE)
+unlink("models/VERSPM-base(1)",recursive=TRUE)
 
 # opening models (e.g. in a new R session, after they are installed)
 # you can always re-open a model - it just creates a new R object for manipulating it

sources/framework/VEModel/inst/walkthrough/02-running.R

@@ -1,4 +1,4 @@
-### running.R
+### 02-running.R
 #   Walk through running models
 
 # Load VEModel package (in effect, the visioneval environment)
@@ -31,5 +31,4 @@ mod.pop <- openModel("VERSPM-pop")
 mod.pop$run() # just run it - one stage at a time
 
 # Continue with 03-extract.R to learn about the parts of a VisionEval model
-# See model-stages.R for more information on model stages
-# See scenarios.R for more information on scenarios
+# See scenarios.R for more information on model stages and scenarios

sources/framework/VEModel/inst/walkthrough/03A-advanced-export.R

@@ -0,0 +1,200 @@
+### 03A-advanced-extract.R
+#   Examples of retrieving raw data from a finished model run
+
+####################
+# SAMPLE MODEL SETUP
+####################
+
+require(VEModel) # just in case it's not already out there...
+mwr <- openModel("VERSPM-run") # Run Install.R to install/run that model
+results <- mwr$run()
+print(results)
+
+########################
+# CHANGING DISPLAY UNITS
+########################
+
+# You can create a file called "display_units.csv" and put it in your
+# model's "defs" folder. Then the fields listed there can have their
+# units automatically converted when you extract them.
+
+# Here, we'll set up the display_units file with a useful conversion
+# This shows how to construct a DisplayUnitsFile
+# Practically speaking, you can do this once in your life and drop it
+# into your model's "defs" directory.
+# You don't have to do it like this: you can also build it by hand
+results$select()$all()
+un <- results$list(details=TRUE)[,c("Group","Table","Name","Units")]
+spd <- un[ grepl("MI/",un$Units)&grepl("sp",un$Name,ignore.case=TRUE), ]
+spd$DisplayUnits <- "MI/HR"
+print(spd)
+
+# Put the display_units file in a useful place (model 'defs' folder)
+# This file will be automatically used during export!
+display_units_file <- file.path(
+  mwr$modelPath,           # folder for model, inside the "models" folder
+  mwr$setting("ParamDir"), # "defs" by default
+  mwr$setting("DisplayUnitsFile") # defaults to 'display_units.csv'
+)
+cat(display_units_file,"\n")
+write.csv(spd,file=display_units_file,row.names=FALSE)
+
+# Select speed fields...
+selected <- results$select()$find(pattern="speed",Group="Year",Table="Marea",select=TRUE)
+
+# Add the key fields
+selected <- selected$addkeys() # Forces tables to have basic geography fields plus "Id"
+print(results$units())
+
+# Using DISPLAY units
+results$export(selection=selected,connection=list(TablePrefix="DisplayUnits"),convertUnits=TRUE)  # Using Display Units
+# Using DATASTORE units
+results$export(selection=selected,connection=list(TablePrefix="Datastore"),convertUnits=FALSE)  # Using DATASTORE units
+
+# Show output files
+print(mwr$dir(output=TRUE,all.files=TRUE))
+
+#########################
+# ACCESSING EXPORTED DATA
+#########################
+
+# Export to a different Database name and save the exporter for further investigation
+# (Timestamp goes on the file name)
+exporter <- results$export("sqlite",connection=list(Database="My-SQLite-Database"))
+exporter$list()
+
+# See the exported SQLite databases
+print(mwr$dir(output=TRUE,all.files=TRUE))
+
+# Read the data back from the export ("extracts" the exported results)
+all.the.data <- exporter$data()       # Returns a list of data.frames by reading back what you exported
+all.the.data.tables <- exporter$data(format="data.table") # If you would rather work with data.tables
+all.the.tibbles <- exporter$data(format=tibble::tibble)   # Any function that knows about data.frames will work
+rm( all.the.data, all.the.data.tables, all.the.tibbles)
+
+# If you have a function that knows about lists of data.frames, you can use that too:
+# additional arguments sent to "data" are passed to the formatting function.
+# The exportPath function returns the absolute path to the model's Output Directory
+excel.workbook.name <- results$export()$data(
+  formatList=TRUE,
+  format=writexl::write_xlsx,
+  path=file.path(mwr$exportPath(),"My-Excel-Data.xlsx")
+)
+mwr$dir(output=TRUE,all.files=TRUE)
+
+# The "extract" function also understands the "format" option, so you
+# can run and extract your model in a single line of code
+openModel("VERSPM-run")$run()$extract(
+  formatList=TRUE,
+  format=writexl::write_xlsx,
+  path=file.path(mwr$exportPath(),"All-in-one-Excel.xlsx")
+)
+
+# If you're exporting to one of the built-in formatters, the "export" function is more direct:
+exporter <- openModel("VERSPM-run")$run()$export("sqlite") # into SQLite
+
+# And you can still move the exported data around
+# This line copies everything from SQLite to Excel
+exporter$data(
+  formatList=TRUE,
+  format=writexl::write_xlsx,
+  path=file.path(mwr$exportPath(),"Back-from-SQLite.xlsx")
+)
+
+rm(exporter) # will close, eventually, the SQLite databaase
+
+mwr$dir(output=TRUE,all.files=TRUE)
+
+############################
+# PARTITIONING EXPORTED DATA
+############################
+
+# The default export creates one table per scenario (model stage) per group (Global,
+# Year) per table type (Household, Worker, Vehicle)
+
+# You can "partition" exported data differently depending on your analysis needs. Data can
+# be partitioned into folders (what the CSV export does), or the partition can be written
+# into the table names (which is the default for SQLite)
+
+# To make a partition, you just list a field in the output data (usually a geography, or
+# the scenario year, or the scenario name) and explain whether you want the partitioned
+# table identified by a "folder" or coded in the table "name". Data from any source with
+# the same partition ends up in the same table (so you can accumulate every year of every
+# scenario in a single "Household" table, but keep the Global group tables with the same
+# name separate).
+
+# Here are some examples of partitioning:
+
+# Clear ALL the outputs in a hurry so we can start fresh:
+mwr$clear(force=TRUE)
+
+# This partition will put all the scenarios and years into single tables (the Global
+# group is always kept separate, using a "folder" by default)
+partition = c(Global="folder")
+exporter <- mwr$results()$export(partition=partition)
+# Yields a single CSV file for each table with all the Years and Scenarios together
+# The Global group will end up in a sub-folder.
+
+# show the output files
+mwr$dir(output=TRUE,all.files=TRUE)
+
+# Show all the table names exported
+print(exporter$list())
+# Show what's in each of the tables (metadata)
+print(exporter$list(namesOnly=FALSE))
+
+# "folders" become prefixed "names" in SQL:
+results <- mwr$results()
+sqlexporter <- results$export("sql",partition=c(Global="folder"),connection=(Database="All-in-one-Tables.sqlite"))
+print(sqlexporter$list())
+
+#################################
+# CONNECTING TO A "REAL" DATABASE
+#################################
+
+# You can use any DBI connection for export to databases like MySQL, MariaDB, SQL Server,
+# or PostgreSQL (or through ODBC connections, though those are messier to set up).
+# ODBC will get you Microsoft Access).
+# Read the R documentation for DBI drivers (https://dbi.r-dbi.org)
+# Your setup will look like the following:
+
+mariadb <- list(
+  driver = "sql", # send into DBI driver
+  package = "RMariaDB",
+  drv = RMariaDB::MariaDB(), # or drv="RMariaDB::MariaDB()" - character string will be parsed
+  Timestamp = "prefix",
+  DBIConfig = list(
+    # items here are anything you would pass to dbConnect for that database
+    dbname = "visioneval", # Adjust for local database name / user / password
+    user = "visioneval",
+    password = "showme"
+  )
+)
+
+# Then just do this
+mwr$results()$export(connection=mariadb)
+
+# You can set up some or all of the connection in your model's visioneval.cnf
+# (or in the runtime visioneval.cnf for "all model" defaults. You can either
+# create a new exporter name, or redefine defaults for an existing one.
+
+# This block in visioneval.cnf makes "mysql" use your local database (uncomment it!)
+# FYI the MariaDB driver will also work for branded MySQL databases
+
+# Exporters:
+#   mysql:
+#     Connection:
+#       driver: sql
+#       package = RMariaDB,       # will require the package, which must be installed
+#       drv = RMariaDB::MariaDB() # character string will be parsed
+#       Timestamp: prefix         # Timestamp each table at the beginning of its name
+#       DBIConfig:
+#         dbname: visioneval      # Adjust for local database name / user / password
+#         user: visioneval
+#         password: showme
+#     Partition:                  # Do it however you like
+#         Global: path
+#         Year: name              # Puts all scenarios into one table per Year
+#
+# Then when you do the following in R, it will use your database:
+#    mwr$run()$export("mysql")

sources/framework/VEModel/inst/walkthrough/04-scenarios.R

@@ -25,11 +25,20 @@ mod.scenarios <- installModel("VERSPM",var="scenarios-ms",modelPath="VERSPM-scen
 
 mod.scenarios$dir(scenarios=TRUE,all.files=TRUE) # Look at the elements of the setup
 
+# The visioneval.cnf configuration defines the base model
+# the scenarios\visioneval.cnf file defines additional scenarios
+# Setting up scenarios is beyond the scope of this walkthrough
+# See https://visioneval.org/docs for more information
+
 # Run the model
+
+# mod.scenarios$plan(workers=3) # Use three cores to run scenarios in parallel
+# RAM, not number of CPUs, is the limiting factor on efficiency.
 mod.scenarios$run()
 
 # Check that everything ran correctly
-print(mod.scenarios)
+print(mod.scenarios) # summary information
+print(mod.scenarios$results()) # More detailed information
 
 # Details can be interesting
 print(mod.scenarios,details=TRUE)

sources/framework/VEModel/inst/walkthrough/05-queries.R

@@ -18,8 +18,10 @@ mod.scenarios <- if ( "VERSPM-scenarios" %in% dir("models") ) {
 } else {
   installModel("VERSPM",var="scenarios-ms",modelPath="VERSPM-scenarios",confirm=FALSE)
 }
-mod.scenarios$run()       # will do nothing if you just ran the model
+mod.scenarios$plan(workers=3)     # Adjust how many based on CPUs and RAM available
+mod.scenarios$run()               # will do nothing if you already ran the model
 print(mod.scenarios,details=TRUE) # without 'details' just says how many scenarios...
+mod.scenarios$clear(outputOnly=TRUE,force=TRUE)
 
 #######################
 # BASIC QUERY OPERATION
@@ -32,7 +34,7 @@ print(mod.scenarios$query())
 # and to read some basic documentation.
 
 # A query is just an R script that defines a list of query specifications.
-# Each specification describes one resulting metric to evaluate for each scenario (model stage).
+# Each specification describes one metric to evaluate for each scenario (model stage).
 
 # Open the query and run it on the scenario results
 qry <- mod.scenarios$query("VERSPM-scenarios") # open the query
@@ -65,40 +67,23 @@ print(
 # Still no outputs - extract produces a data.frame for further processing
 mod.scenarios$dir(output=TRUE)
 
-# Export (instead of, or in addition to, extract) to generate a file
-# Currently, only .csv is supported. But you can save earlier q.df in any
-# file format that supports tables and has an R package - see below
+###################
+# EXPORTING QUERIES
+###################
 
-qry$export(format="csv") # Default CSV file name in output directory
-mod.scenarios$dir(outputs=TRUE,all.files=TRUE) # Now we have output in a .csv file
-
-#################################
-# WRITING TO AN EXTERNAL DATABASE
-#################################
-
-# You can also write the extract data.frame into an external database
-# See 03-extract.R for how to export raw model results to a database
-
-require(RSQLite)  # you may need to install RSQLite and its dependencies
-
-# Open DBI connection
-dbPath <- file.path(mod.scenarios$modelResults,mod.scenarios$setting("OutputDir"))
-if ( ! dir.exists(dbPath) ) dir.create(dbPath,recursive=TRUE)
-
-db.name <- paste0("VERSPM-scenarios_Database.sqlite")
+# Export (instead of, or in addition to, extract) to generate a file with the query
+# results. The export works the same as for the raw data, except that the partition is
+# ignored and you get a single table (either Long or Wide format)
 
-db.connection <- file.path(dbPath,db.name)
-
-mydb <- DBI::dbConnect(RSQLite::SQLite(), db.connection)
-
-# Make a table name for the query results
-table.name <- paste0("Query_",format(Sys.time(),"%Y_%m_%d-%H_%M"))
-
-# We could have used q.df from above, but we'll just regenerate it
-dbWriteTable( mydb, table.name , qry$extract(), overwrite=TRUE )
+qry$export() # Default CSV file name in output directory
+mod.scenarios$dir(outputs=TRUE,all.files=TRUE) # Now we have output in a .csv file
 
-# Finally, clean up the database connection
-DBI::dbDisconnect(mydb)
+qry$export("sqlite") # Put the query extraction into an SQLite Database
 
-# See that there is now an SQLite database
-mod.scenarios$dir(outputs=TRUE,all.files=TRUE)
+# Or put the query results into Excel directly:
+qry$export("data.frame")$data(
+  format=writexl::write_xlsx,
+  path=file.path(mod.scenarios$exportPath(),"Query-Output.xlsx")
+)
+# if you were paying close attention before, you don't need the "formatList" parameter for
+# Excel output here because queries generate a single data.frame in each case.