From 276d1f628e2b28bc1548ab8e403fbb9996152ace Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Tue, 21 Sep 2021 16:34:41 -0400 Subject: [PATCH 01/10] docs, #44: move wiki content over to user manual --- docs/manual/404.html | 308 ++++++++++++ docs/manual/SSMSE.html | 325 +++++++------ ...a-custom-management-strategyprocedure.html | 144 +++--- docs/manual/glossary-of-mse-terms.html | 326 +++++++++++++ docs/manual/helper.html | 88 ++-- docs/manual/index.html | 95 ++-- docs/manual/intro.html | 96 ++-- .../anchor-sections-1.0.1/anchor-sections.css | 5 + .../anchor-sections-1.0.1/anchor-sections.js | 33 ++ .../gitbook-2.6.7/css/plugin-bookdown.css | 6 + .../gitbook-2.6.7/css/plugin-fontsettings.css | 11 + docs/manual/libs/gitbook-2.6.7/css/style.css | 9 +- .../libs/gitbook-2.6.7/js/plugin-search.js | 93 +++- .../libs/gitbook-2.6.7/js/plugin-sharing.js | 15 +- .../libs/header-attrs-2.11/header-attrs.js | 12 + .../libs/jquery-3.6.0/jquery-3.6.0.min.js | 2 + docs/manual/output.html | 94 ++-- ...otting-output-and-performance-metrics.html | 88 ++-- docs/manual/reference-keys.txt | 1 + docs/manual/search_index.json | 2 +- docs/manual/simple.html | 456 +++++++++--------- inst/bookdown/06-glossary.Rmd | 21 + inst/bookdown/README.md | 3 + 23 files changed, 1592 insertions(+), 641 deletions(-) create mode 100644 docs/manual/404.html create mode 100644 docs/manual/glossary-of-mse-terms.html create mode 100644 docs/manual/libs/anchor-sections-1.0.1/anchor-sections.css create mode 100644 docs/manual/libs/anchor-sections-1.0.1/anchor-sections.js create mode 100644 docs/manual/libs/header-attrs-2.11/header-attrs.js create mode 100644 docs/manual/libs/jquery-3.6.0/jquery-3.6.0.min.js create mode 100644 inst/bookdown/06-glossary.Rmd create mode 100644 inst/bookdown/README.md diff --git a/docs/manual/404.html b/docs/manual/404.html new file mode 100644 index 00000000..2f56328e --- /dev/null +++ b/docs/manual/404.html @@ -0,0 +1,308 @@ + + + + + + + Page not found | SSMSE user manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

Page not found

+

The page you requested cannot be found (perhaps it was moved or renamed).

+

You may want to try searching to find the page's new location, or use +the table of contents to find the page you are looking for.

+
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/manual/SSMSE.html b/docs/manual/SSMSE.html index 874eacf9..8920fa0f 100644 --- a/docs/manual/SSMSE.html +++ b/docs/manual/SSMSE.html @@ -6,7 +6,7 @@ 4 Options for run_SSMSE | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,18 +223,18 @@

-
+

4 Options for run_SSMSE

Many inputs are possible for the run_SSMSE() option. Here, we will describe some of the options available. For detailed documentation, type ?SSMSE::run_SSMSE() into the R console.

-
+

4.1 Scenarios in SSMSE

Note that multiple scenarios can be called in run_SSMSE(), often through vector inputs to run_SSMSE(). Below, we will describe inputs needed to run 1 scenario.

-
+

4.2 Operating model

The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = "cod". Otherwise, the path to the OM model should be specified in OM_in_dir_vec.

-
+

4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM)

The management strategy (and EM) can be specified in one of two ways:

    @@ -224,73 +242,73 @@

    4.3 The Management Strategy/proce
  1. Using a custom procedure via a function in R

In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet.

-
+

4.3.1 Specify the Management Strategy in a SS model

An SS model can be set up as the EM for the MSE. To use this option, specify "EM" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify "cod" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec.

Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user.

-
+

4.3.2 Using a custom management strategy/procedure

Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year:

- +
constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, 
+                              frac_catch = 0.5, ...) { # need to include ... to allow function to work
+  # set catch the same as the previous year (sampled catch).
+  # catch is in the same units as the operating model, in this case it is in
+  # biomass.
+  catch <- data.frame(
+    year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward
+    seas = 1, # hard coded from looking at model 
+    fleet = 1,  # hard coded from looking at model
+    catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch,
+    catch_se = 0.05) # hard coded from looking at model
+  catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL
+  catch_F <- NULL # catch in terms of F, can be left as NULL.
+  discards <- NULL # discards can be left as NULL if there are no discards
+  catch_list <- list(catch = catch,
+                     catch_bio = catch_bio, 
+                     catch_F = catch_F,
+                     discards = discards)
+}

Let’s assume this function is saved in a file within the working directory named constant_catch_MS.R.

This function can then be used in a call to run_SSMSE():

- +
# define sample structure
+datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE")
+sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning
+sample_struct$lencomp <- NULL # don't use length sampling
+
+# run the SSMSE routine
+run_result_custom <- run_SSMSE(
+                              scen_name_vec = "constant-catch",
+                              out_dir_scen_vec = "my_results",
+                              iter_vec = 1,
+                              OM_name_vec = "cod",
+                              OM_in_dir_vec = NULL,
+                              MS_vec = "constant_catch_MS", # use custom fun
+                              custom_MS_source = "constant_catch_MS.R",
+                              use_SS_boot_vec = TRUE,
+                              nyrs_vec = 6,
+                              nyrs_assess_vec = 3,
+                              run_EM_last_yr = FALSE,
+                              run_parallel = FALSE,
+                              sample_struct_list = list(sample_struct),
+                              seed = 12345)
-
+

4.4 Sampling options

Currently, the only available sampling option is to use the bootstrapping module within SS itself. This means specifying use_SS_boot_vec = TRUE. Details on how sampling is done using the bootstrapping module in SS is available in the “Bootstrap Data Files” section of the SS user manual.

Users also need to specify how and which data types should be sampled for each future year in the simulation in sample_struct_list. sample_struct_list is a list of lists. The first level is a list for each scenario; then, for the scenario, there is a list of dataframes, each of which specifying which years and fleets of data should be sampled in the future as well as which standard errors or sample sizes should be used.

The helper function create_sample_struct() can be used to help users generate the list of dataframes for a scenario. See an example of this function’s use in the simple example or by typing ?SSMSE::create_sample_struct() into the R console.

Users can specify the sampling during the historical period of the model through the sample_struct_hist input to run_SSMSE. This is an optional input and has the same structure as sample_struct_list, but the years will be before the original end year of the OM.

-
+

4.5 Future changes to the operating model (#future)

By default, SSMSE will simply extend the structure of the OM into the future using the same parameter values as in the last year of the model. However, the user may want to allow changes in parameter values to occur as the OM is extended forward in time. Users can input values into the future_om_list to accomplish this. This input is a list of lists. For the first level of lists, each represents a separate change to make. Within the first level, there are 4 list components that outline the details of the change to make. There are 2 main choices: 1) to specify model changes by telling SSMSE how values should be sampled or 2) to input your own custom values for future values of a given parameter.

-
+

4.5.1 The strucutre of future_om_list

For example, here is an example list containing just one future change for the model. It shows that the model should be changed to 4.5 in year 103 and afterwards:

- +
my_future_om_list <- create_future_om_list(list_length = 1)
+my_future_om_list
[[1]]
 [[1]]$pars
 [1] "SizeSel_P_3_Fishery(1)"
@@ -302,44 +320,42 @@ 
4.5.1 The strucutre of futu
 [1] "model_change"
 
 [[1]]$input
-  first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param
-1                 NA                NA              102                103     mean
-    method value
-1 absolute   4.5
- + first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value +1 NA NA 102 103 mean absolute 4.5 +
length(my_future_om_list) # note has length 1 b/c 1 change
[1] 1
- +
length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change
[1] 4

Note there is just one change specified here. For the change, there are four list items that are required for any specified change.

    -
  1. The first list item is named “pars”. It contains a vector of parameter name(s) to apply the change to. The names should be the same as the names in r4ss::SS_read_pars() or can include the following:
    2. +
  2. The first list item is named “pars.” It contains a vector of parameter name(s) to apply the change to. The names should be the same as the names in r4ss::SS_read_pars() or can include the following:
  • "rec_devs" - For specifying recruitment deviations
  • "impl_error - For specifying implementation error in transforming a management procedure’s specified future catch into realized catch.
    • -
  • "all" - Apply the change to all parameters in the model, with the exception of “SR_sigmaR”, “SR_regime”, “SR_autocorr”, and “impl_error” parameters. Changing the first 3 stock recruitment related parameters would conflict with changes in recruitment deviations. Since implementation error is not a parameter specified in the control file and SSMSE does not rely on the implementation error parameter created through the forecasting file, including the implementation error in “all” did not seem appropriate.
    • +
  • "all" - Apply the change to all parameters in the model, with the exception of “SR_sigmaR,” “SR_regime,” “SR_autocorr,” and “impl_error” parameters. Changing the first 3 stock recruitment related parameters would conflict with changes in recruitment deviations. Since implementation error is not a parameter specified in the control file and SSMSE does not rely on the implementation error parameter created through the forecasting file, including the implementation error in “all” did not seem appropriate.

In this case, we just want to apply the change to the SizeSel_P_3_Fishery(1) parameter of the model.

    -

  1. The second item is “scen”, which contains a vector of information about how to apply the changes within and across scenarios. The first value is an option to specify how the change should be applied among scenarios, either “randomize” (use a different value for each iteration of each scenario) or “replicate” (use the same set of values across scenarios for the same number iteration, but each value will be different across iterations within the same scenario). Note that the same values will be applied across iterations and scenarios if there isn’t any stochasticity in the values being drawn (e.g., standard deviation set at 0), regardless if “randomize” or “replicate” is chosen. In the example above, there is no stochasticity, so specifying randomize or replicate does not matter. Following the first value are the names of the scenarios to apply the change to. If the change should be applied to all of the scenarios, “all” can be used in place of naming every scenario. In this example, The change will only be applied to the scenario named “scen2”, so the input for “scen” is c("randomize", "scen2").

    2. -

  2. The “pattern” is a vector of character inputs. The first value should be either “model_change”, meaning that SSMSE will calculate the change values, or “custom” which allows the user to directly put in the values that they want in the model. In this case, “model_change” is used. A second input can also be added when the “model_change” pattern is used, which specifies the distribution to pull from. This could be “normal” or “lognormal”, but if no input is provided, a normal distribution will be used for sampling.

    3. -

  3. The fourth item, named “input”, is a dataframe, which contains different column name values depending on if the “model_change” pattern is used or if the “custom” pattern is used. For the model_change options, a dataframe in input specifies a change for the parameter of a distribution. Because we are using the model change option, we have the following columns:

    4. +

  4. The second item is “scen,” which contains a vector of information about how to apply the changes within and across scenarios. The first value is an option to specify how the change should be applied among scenarios, either “randomize” (use a different value for each iteration of each scenario) or “replicate” (use the same set of values across scenarios for the same number iteration, but each value will be different across iterations within the same scenario). Note that the same values will be applied across iterations and scenarios if there isn’t any stochasticity in the values being drawn (e.g., standard deviation set at 0), regardless if “randomize” or “replicate” is chosen. In the example above, there is no stochasticity, so specifying randomize or replicate does not matter. Following the first value are the names of the scenarios to apply the change to. If the change should be applied to all of the scenarios, “all” can be used in place of naming every scenario. In this example, The change will only be applied to the scenario named “scen2,” so the input for “scen” is c("randomize", "scen2").

    5. +

  5. The “pattern” is a vector of character inputs. The first value should be either “model_change,” meaning that SSMSE will calculate the change values, or “custom” which allows the user to directly put in the values that they want in the model. In this case, “model_change” is used. A second input can also be added when the “model_change” pattern is used, which specifies the distribution to pull from. This could be “normal” or “lognormal,” but if no input is provided, a normal distribution will be used for sampling.

    6. +

  6. The fourth item, named “input,” is a dataframe, which contains different column name values depending on if the “model_change” pattern is used or if the “custom” pattern is used. For the model_change options, a dataframe in input specifies a change for the parameter of a distribution. Because we are using the model change option, we have the following columns:

  • first_yr_averaging: The first year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used.
  • last_yr_averaging: The last year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used.
  • last_yr_orig_val: The last year of the future deviations with the original model value. This value will not be changed, but the following year’s value will be.
  • first_yr_final_val: The first year where the final value as specified will be reached. If no future changes are made, the final value will continue forward in the OM through all projected years. For step changes, the first_yr_final_val is just 1 year after the last_yr_orig_val. However, if a more gradual adjustment toward the final value is desired, this could be any number of years after the original value.
    • -
  • ts_param: The sampling parameter to modify. Options are “mean”, “ar_1_phi” (if using autocorrelation), “sd”, and “cv”. If not included in the dataframe, “mean” is assumed to be the same as in the previous model year, sd is assumed to be 0, and we assume no autocorrelation (as specified through an autoregressive AR1 process). Phi should be between -1 and 1 (this creates a stationary process) and if phi = 1, this creates a random walk (as random walk is a special case of an AR1 model). Note that both sd and cv cannot be specified within the same input dataframe.
    • -
  • method: How to apply the change relative to the historical (if specified) or previous year’s value (if first_yr_averaging and last_yr_averaging for that row are NA). Options are “multiplicative”, “additive”, and “absolute”. This example uses “absolute”, meaning that the number in “value” is directly used.
    • +
  • ts_param: The sampling parameter to modify. Options are “mean,” “ar_1_phi” (if using autocorrelation), “sd,” and “cv.” If not included in the dataframe, “mean” is assumed to be the same as in the previous model year, sd is assumed to be 0, and we assume no autocorrelation (as specified through an autoregressive AR1 process). Phi should be between -1 and 1 (this creates a stationary process) and if phi = 1, this creates a random walk (as random walk is a special case of an AR1 model). Note that both sd and cv cannot be specified within the same input dataframe.
    • +
  • method: How to apply the change relative to the historical (if specified) or previous year’s value (if first_yr_averaging and last_yr_averaging for that row are NA). Options are “multiplicative,” “additive,” and “absolute.” This example uses “absolute,” meaning that the number in “value” is directly used.
  • value. The value of the parameter change. This may be NA if historical averaging is used.
-
+

4.5.2 Example of future_om_list denoting a gradual change

Suppose we wanted to change the value of “SizeSel_P_3_Fishery(1)” in scen2 over 4 years after year 102 to arrive at a value of 4.5 in year 106. We can do this by using my_future_om_list, but changing the value in the first row of first_yr_final_val to 106:

- +
my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106
+my_future_om_list[[1]]
$pars
 [1] "SizeSel_P_3_Fishery(1)"
 
@@ -350,25 +366,23 @@ 
4.5.2 Example of future_om_list d
 [1] "model_change"
 
 $input
-  first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param
-1                 NA                NA              102                106     mean
-    method value
-1 absolute   4.5
+ first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value +1 NA NA 102 106 mean absolute 4.5
-
+

4.5.3 Example of future_om_list with a random deviations

Suppose we now wanted the value of “SizeSel_P_3_Fishery(1)” to change randomly according to a normal distribution with a standard deviation of 0.1 around a mean of 4.5 from 104 onwards. This can be done by adding a line specifying a change in standard deviation (which for now, has been assumed to be 0) to the data frame:

- +
new_vals <- data.frame(first_yr_averaging = NA,
+                       last_yr_averaging  = NA, 
+                       last_yr_orig_val   = 103,
+                       first_yr_final_val = 104, 
+                       ts_param = "sd", 
+                       method = "absolute",
+                       value = 0.1)
+
+my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]],
+                                           new_vals)
+my_future_om_list[[1]]
$pars
 [1] "SizeSel_P_3_Fishery(1)"
 
@@ -379,38 +393,35 @@ 
4.5.3 Example of future_om_list w
 [1] "model_change"
 
 $input
-  first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param
-1                 NA                NA              102                106     mean
-2                 NA                NA              103                104       sd
-    method value
-1 absolute   4.5
-2 absolute   0.1
+ first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value +1 NA NA 102 106 mean absolute 4.5 +2 NA NA 103 104 sd absolute 0.1

Note that the last_yr_orig_val and first_yr_final_val are different than the line for the mean, which is allowed.

-
+

4.5.4 Example of using historical values for determining parameter values

This example applies random annual deviations to all parameters for scenarios scen2 and scen3.

- +
future_om_list_2 <- vector(mode = "list", length = 1)
+future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4))
+names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input")
+
+future_om_list_2[[1]][["pars"]] <- "all"
+future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3")
+future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist
+future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1),
+                                               last_yr_averaging = c(100, 100),
+                                               last_yr_orig_val = c(100, 100),
+                                               first_yr_final_val = c(101, 101), 
+                                               ts_param = c("cv", "mean"),
+                                               method = c("absolute", "multiplier"), 
+                                               value = c(0.1, 1))

Note that the choice of year range for historical values does not matter unless the parameter is already time-varying in the original operating model or has become time varying through a previous change. Otherwise, the base model parameter will be applied. If no historical years are included, then the base parameter value will be the basis of comparison for relative changes (i.e., method = multiplier or additive).

-
+

4.5.5 Example using “custom” pattern instead of “model_change”

- +
custom_future_om_list <- create_future_om_list(example_type = "custom", 
+                                               list_length = 1)
+custom_future_om_list
[[1]]
 [[1]]$pars
 [1] "impl_error"
@@ -513,7 +524,7 @@ 
4.5.5 Example using “custom”
 88 impl_error scen3    3 106  1.10
 89 impl_error scen3    4 106  1.10
 90 impl_error scen3    5 106  1.10
-

The inputs required for pattern = custom are similar to using pattern = model_change, but there is no need to specify a distribution as the second vector input of “pattern” and the column names in input are different. Also that the change is “randomized”, which indicates that a value for each scenario and each iteration are necessary, whereas if the first value of “scen” was “replicate”, separate values for each scenario should not be specified, but rather a single value for “all” should be used. The columns in the dataframe are:

+

The inputs required for pattern = custom are similar to using pattern = model_change, but there is no need to specify a distribution as the second vector input of “pattern” and the column names in input are different. Also that the change is “randomized,” which indicates that a value for each scenario and each iteration are necessary, whereas if the first value of “scen” was “replicate,” separate values for each scenario should not be specified, but rather a single value for “all” should be used. The columns in the dataframe are:

  • par: the parameter name or “all” if it should be applied to all parameters in the “pars” input
  • scen: which scenario the value should apply to, or “all” if “replicate” is used as the scenario input
    • @@ -522,26 +533,26 @@

    4.5.5 Example using “custom”
  • value: the value to apply to the model

-
+

4.5.6 How is the operating modified to accomodate changes as specified in the future_OM_list?

All changes are made by converting the parameter(s) with changes to be time varing by using additive annual parameter deviations. Because this is an operating model, Stock Synthesis is not run with estimation, so to get the changes into the OM, values (drawn or calculated in model_changes or specified by the user using “custom”) are directly input as parameter deviations into the ss.par file.

If an OM already contains time varying parameters, these parameters will also be converted to additive parameter deviations before applying the changes specified in the future_OM_list.

-
+

4.5.7 Example of specifying recruitment deviations

This shows some code for putting recruitment deviations with a mean of 0 and the same standard deviation as the historical recruitment deviations in years 1 to 100:

- +
template_mod_change <- create_future_om_list(list_length = 1)
+rec_dev_specify <- template_mod_change[[1]]
+rec_dev_specify$pars <- "rec_devs"
+rec_dev_specify$scen <- c("replicate", "all")
+rec_dev_specify$input$first_yr_averaging <- 1
+rec_dev_specify$input$last_yr_averaging <- 100
+rec_dev_specify$input$last_yr_orig_val <- 100
+rec_dev_specify$input$first_yr_final_val <- 101
+rec_dev_specify$input$ts_param <- "sd"
+rec_dev_specify$input$value <- NA
+rec_dev_list <- list(rec_dev_specify)
+rec_dev_list
[[1]]
 [[1]]$pars
 [1] "rec_devs"
@@ -553,10 +564,8 @@ 
4.5.7 Example of specifying recru
 [1] "model_change"
 
 [[1]]$input
-  first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param
-1                  1               100              100                101       sd
-    method value
-1 absolute    NA
+ first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value +1 1 100 100 101 sd absolute NA
@@ -571,7 +580,6 @@

4.5.7 Example of specifying recru

- @@ -590,6 +598,7 @@

4.5.7 Example of specifying recru "weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -610,6 +619,10 @@

4.5.7 Example of specifying recru "text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html b/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html index f3d943e6..af4a21a5 100644 --- a/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html +++ b/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html @@ -6,7 +6,7 @@ 3 Advanced options: use a custom management strategy/procedure | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,42 +223,42 @@

-
+

3 Advanced options: use a custom management strategy/procedure

Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year:

- +
constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, 
+                              frac_catch = 0.5, ...) { # need to include ... to allow function to work
+  # set catch the same as the previous year (sampled catch).
+  # catch is in the same units as the operating model, in this case it is in
+  # biomass.
+  catch <- data.frame(
+    year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward
+    seas = 1, # hard coded from looking at model 
+    fleet = 1,  # hard coded from looking at model
+    catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch,
+    catch_se = 0.05) # hard coded from looking at model
+  catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL
+  catch_F <- NULL # catch in terms of F, can be left as NULL.
+  discards <- NULL # discards can be left as NULL if there are no discards
+  catch_list <- list(catch = catch,
+                     catch_bio = catch_bio, 
+                     catch_F = catch_F,
+                     discards = discards)
+}

The function should be created in a separate file. In this case, assume this function is available in a file custom_funs.R.

This function can then be used in a call to run_SSMSE:

- +
run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario
+                out_dir_scen_vec = run_res_path, # directory in which to run the scenario
+                iter_vec = 1,
+                OM_name_vec = "cod", # specify directories instead
+                OM_in_dir_vec = NULL,
+                MS_vec = "constant_catch_MS", # use the custom function
+                custom_MS_source = "custom_funs.R", # File where the custom function is available.
+                nyrs_vec = 6,        # Years to project OM forward
+                nyrs_assess_vec = 3, # Years between assessments
+                future_om_list = future_om_list_recdevs_sel,
+                sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS.
+                seed = 12345) #Set a fixed integer seed that allows replication
@@ -253,7 +271,6 @@

3 Advanced options: use a custom

- @@ -272,6 +289,7 @@

3 Advanced options: use a custom "weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -292,6 +310,10 @@

3 Advanced options: use a custom "text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/glossary-of-mse-terms.html b/docs/manual/glossary-of-mse-terms.html new file mode 100644 index 00000000..5d71dc5d --- /dev/null +++ b/docs/manual/glossary-of-mse-terms.html @@ -0,0 +1,326 @@ + + + + + + + 8 Glossary of MSE terms | SSMSE user manual + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+
+ + +
+
+ +
+
+

8 Glossary of MSE terms

+

This glossary is based off of Rademeyer et al. (2007) and Punt et al. (2016).

+
    +
  • Assessment error - Error that occurs during the process of conducting an assessment, specifically error “which inform the catch control rule that is being evaluated using the MSE: management advice for any system is based on uncertain data. Consequently, the data that inform catch control rules need to be generated in a manner which is as realistic as possible. Uncertainty arises when the model used for conducting assessments and providing management advice differs from the operating model, or the data are too noisy to estimate all key parameters reliably (Punt et al. 2016).”
    • +
  • Assessment model (AM) - A fitted and well scrutinized Stock Synthesis stock assessment model. The AM is input for SSMSE so that it can build the operating model.
    • +
  • Bootstrapped dataset - After running an SS model (and if the user turns on the option in the starter file), the output file data.ss_new contains a “dataset” that has the same form as the input data.ss file, but instead of the input values, contains a sampled dataset. This is derived from using the expected values given the parameter estimates and model structure (see glossary entry on Expected values), uncertainty either estimated or input in the model, and an assumed distribution for sampling. As many bootstrapped data sets as the user desires can be output from an SS model. These “bootstrap datasets” are the third or greater set of values in the data.ss_new file.
    • +
  • Estimation model (EM) - This refers to the model used within the MSE procedure to represent the stock assessment process. Note that an estimation model proxy could also be used to represent the stock assessment process.
    • +
  • Expected values - After running an SS model (and if the user turns on teh option in the starter file), the output file data.ss_new contains a “dataset” that has the same form as the input data.ss file, but instead of the input values, contains the expected values given the parameter estimates and model structure. This “expected values dataset” is the second set of values in the data.ss_new file.
    • +
  • Implementation error - Also called implementation uncertainty or outcome uncertainty. Broadly, this includes error of implementing the management action(s), which is often not done perfectly. Definition from Punt et al. (2016): “The most obvious form of this type of uncertainty is when catches are +not the same as the TACs – typically more is taken or the decision-makers do not implement the TACs suggested by the management strategy. However, there are many other sources of outcome uncertainty, such as that associated with catch limits set for recreational fisheries and regulating discards.”
    • +
  • Index - An index of abundance. In SS files, this is sometimes used interchangably with CPUE. Plural, indices.
    • +
  • Management strategy - Following Punt et al. (2016), there are 2 types: model-based management strategies and empirical management strategies. Ideally, SSMSE will allow for users to use either of these management strategies. Model-based management strategies include conducting a stock assessment and using output for determining harvest control rules. Empirical management strategies do not involve conducting a stock assessment but rather setting regulations from data (although summarization is possible). In some cases, management strategies are computationally intensive/time consuming to formally include in the context of an MSE; to speed up the process, it is commen to use management strategy proxies, typically an assumed error distribution about the operating model values. Management strategies are also known as management procedures.
    • +
  • Model uncertainty - Definition from Punt et al. (2016): “the form of relationships within an operating model will always be subject to uncertainty. The simplest type of model uncertainty involves, for example, whether the stock–recruitment relationship is Beverton–Holt or Ricker, whether a fixed value for a model parameter is correct, or whether fishery selectivity is asymptotic or dome-shaped. However, there are other more complicated types of model structure uncertainty such as how many stocks are present in the area modelled, the error structure of the data used for assessment purposes, the impact of future climate change on biological relationships such as the stock–recruitment function, and ecosystem impacts on biological and fishery processes.”
    • +
  • Observation error - Error that results from not observing the true dynamics of the system. See also bootstrapped dataset.
    • +
  • Operating model (OM) - This is a Stock Synthesis model that defines the assumed true dynamics of the population and its associated fisheries for the purposes of management strategy evaluation. Most often more than one operating model is necessary in order to adequately characterize the uncertainty in the true dynamics of the system. The Stock Synthesis OM may also define how sampling from the true dynamics is done, as SS produces expected values and, if desired, bootstrapped data sets based on sampling assumptions implicit in the model. For more information, see the glossary entries on Expected values and bootstrapped dataset
    • +
  • Parameter uncertainty - Definition from Punt et al. (2016): “many operating models are fit to the data available, but the values estimated for the parameters of those operating models (e.g. fishery selectivity-at-age, the parameters of the stock–recruitment relationship and historical deviations in recruitment about the stock–recruitment relationship) are subject to error.”
    • +
  • Performance statistics - Definition from Rademeyer et al. (2007): “Statistics that summarize different aspects of the results of a simulation trial used to evaluate how well a specific [management strategy] achieves some or all of the general objectives for management for a particular scenario.” Performance statistics usually fall into one of three categories: catch-related, stability related, or risk related.
    • +
  • Process uncertainty - Definition from Punt et al. (2016): “variation (usually assumed to be random, though sometimes incorporating autocorrelation) in parameters often considered fixed in stock assessments such as natural mortality, future recruitment about a stock–recruitment relationship and selectivity.”
    • +
  • Stock Synthesis (SS, SS3) - A integrative, general population dynamics modeling program used to assess the effects of fishing on population. Available at the Stock Synthesis website
    • +
  • Uncertainty - Incorporating uncertainty into an MSE procedure is extremely important. There are several potential sources of uncertainty, which we have divided as done in Punt et al. (2016) in addition to adding observation error. For more details, see the glossary entries on Process Uncertainty, Parameter uncertainty, Model uncertainty, Assessment error, Implementation error, and observation error. Punt et al. (2016) suggest that MSE should at least consider process uncertainty (particularly deviations from the stock recruitment relationship), parameter uncertainty (particularly which relates to productivity and stock size), and observation error. Note that Rademeyer et al. (2007) divides error into estimation error, implementation error, observation error and process error, which could be used instead. These may be more natural divisions in sources of error.
    • +
+ +
+
+ +
+
+
+ + +
+
+ + + + + + + + + + + + + + + diff --git a/docs/manual/helper.html b/docs/manual/helper.html index 0a5fe25f..110ca522 100644 --- a/docs/manual/helper.html +++ b/docs/manual/helper.html @@ -6,7 +6,7 @@ 5 Helper functions in SSMSE | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,19 +223,19 @@

-
+

5 Helper functions in SSMSE

run_SSMSE() requires some detailed inputs that are time consuming to make. In the interest of automating routines as much as possible, helper functions are available in SSMSE.

-
+

5.1 Creating multiple operating models with develop_OMs()

This function allows users to change a parameter in a Stock Synthesis OM and, if desired, refit the operating model to data. Typically, users will want to create multiple operating models to use in different scenarios in order to account for uncertainties in parameter values, such as biological parameters relating to natural mortality or growth.

-
+

5.2 Set up sampling for a scenario with create_sample_struct()

run_SSMSE() requires the input sample_struct, which outlines the future sampling structure to use to generate data sets. This object could be difficult to create manually and users may want to just continue sampling as previously done in an existing SS model. create_sample_struct() will use the time patterns in sampling for different data types found in an SS data file and extend it forward in time. If no pattern is found, the function will return NAs and provide a warning to the user. If any NAs are returned (except to indicate that an entire data type is missing), the user will need to remove or fill in the NA value before usign the list as an input to run_SSMSE().

See use of create_sample_struct() in simple example.

-
+

5.3 Get examples of the future_om_list input with create_future_om_list()

Users can modify the future structure of their operating model through the future_om_list input to run_SSMSE. The structure and names of this object matter, so examples are provided by calling create_future_om_list().

See more details about the specification of future_om_list objects.

@@ -234,7 +252,6 @@

5.3 Get examples of the fut

- @@ -253,6 +270,7 @@

5.3 Get examples of the fut "weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -273,6 +291,10 @@

5.3 Get examples of the fut "text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/index.html b/docs/manual/index.html index 6fe7e242..7509ad16 100644 --- a/docs/manual/index.html +++ b/docs/manual/index.html @@ -6,7 +6,7 @@ SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -208,28 +226,27 @@

Preface

-Logo for the SSMSE package +

Logo for the SSMSE package

This is the user manual for SSMSE, an R package for Management Strategy Evaluation with Stock Synthesis Operating models. This document is still a work in progress! If you would like to suggest changes, the R markdown files used to generate the user manual are available in a bookdown folder within the SSMSE repository. Please feel free to open an issue or submit a pull request to suggest changes.

R session information

- -
R version 4.0.2 Patched (2020-07-15 r78861)
+
sessionInfo()

+
R version 4.1.1 (2021-08-10)
 Platform: x86_64-w64-mingw32/x64 (64-bit)
-Running under: Windows 10 x64 (build 18363)
+Running under: Windows 10 x64 (build 19042)
 
 Matrix products: default
 
 locale:
-[1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252   
-[3] LC_MONETARY=English_United States.1252 LC_NUMERIC=C                          
-[5] LC_TIME=English_United States.1252    
+[1] LC_COLLATE=English_United States.1252  LC_CTYPE=English_United States.1252    LC_MONETARY=English_United States.1252
+[4] LC_NUMERIC=C                           LC_TIME=English_United States.1252    
 
 attached base packages:
 [1] stats     graphics  grDevices utils     datasets  methods   base     
@@ -249,7 +266,6 @@ 
R session information
- @@ -268,6 +284,7 @@

R session information

"weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -288,6 +305,10 @@

R session information

"text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/intro.html b/docs/manual/intro.html index 395f747e..2d9c0d24 100644 --- a/docs/manual/intro.html +++ b/docs/manual/intro.html @@ -6,7 +6,7 @@ 1 Introduction | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,13 +223,13 @@

-
+

1 Introduction

-
+

1.1 Purpose

SSMSE allows users to directly use Stock Synthesis (SS) as an operating model (OM) in an Management Strategy Evaluation (MSE). The approach requires a conditioned Stock Synthesis model, which is treated as the OM.

-
+

1.2 Functions in SSMSE

The main functions users can call in SSMSE are:

@@ -314,21 +332,21 @@

1.2 Functions in SSMSE

-
+

1.3 Brief description of the SSMSE MSE simulation procedure

-
+

1.3.1 Conditioning the OM and sampling from the OM

For each scenario, SSMSE starts with the user providing a fitted Stock Synthesis model (or selecting an model from the SSMSE package) to use as an OM. For each iteration of the scenario, SSMSE turns the SS fitted model into an OM and runs it once with no estimation with Stock Synthesis in order to get the “true” values and a bootstrapped data set from SS. Note that any modifications to the OM’s parameters specified by the users as it is extended forward in time are also applied.

-
+

1.3.2 First run of the management strategy in the MSE simulation

The bootstrapped dataset is then used in a Management strategy to project catch by fleet to use for the next \(n\) years, where \(n\) is the number of years between assessments.

-
+

1.3.3 Feedback from managment strategy into OM: extending model years

The catch for the next \(n\) years before the next assessment is then added to the OM, as well as any recruitment or time varying parameter deviations. The OM is again run with no estimation where it can be used to produce sampled data for the next \(n\) years. These new data values are appended to the original dataset.

-
+

1.3.4 Subsequent runs of the management strategy

The appended data set is then used in the managment strategy again, and new catch by fleet is produced that can then be fed back to the OM.

@@ -345,7 +363,6 @@

1.3.4 Subsequent runs of the mana

- @@ -364,6 +381,7 @@

1.3.4 Subsequent runs of the mana "weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -384,6 +402,10 @@

1.3.4 Subsequent runs of the mana "text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/libs/anchor-sections-1.0.1/anchor-sections.css b/docs/manual/libs/anchor-sections-1.0.1/anchor-sections.css new file mode 100644 index 00000000..4bf387c1 --- /dev/null +++ b/docs/manual/libs/anchor-sections-1.0.1/anchor-sections.css @@ -0,0 +1,5 @@ +/* Styles for section anchors */ +a.anchor-section {margin-left: 10px; visibility: hidden; color: inherit;} +a.anchor-section::before {content: '#';} +.hasAnchor:hover a.anchor-section {visibility: visible;} +ul > li > .anchor-section {display: none;} diff --git a/docs/manual/libs/anchor-sections-1.0.1/anchor-sections.js b/docs/manual/libs/anchor-sections-1.0.1/anchor-sections.js new file mode 100644 index 00000000..fed21918 --- /dev/null +++ b/docs/manual/libs/anchor-sections-1.0.1/anchor-sections.js @@ -0,0 +1,33 @@ +// Anchor sections v1.0 written by Atsushi Yasumoto on Oct 3rd, 2020. +document.addEventListener('DOMContentLoaded', function() { + // Do nothing if AnchorJS is used + if (typeof window.anchors === 'object' && anchors.hasOwnProperty('hasAnchorJSLink')) { + return; + } + + const h = document.querySelectorAll('h1, h2, h3, h4, h5, h6'); + + // Do nothing if sections are already anchored + if (Array.from(h).some(x => x.classList.contains('hasAnchor'))) { + return null; + } + + // Use section id when pandoc runs with --section-divs + const section_id = function(x) { + return ((x.classList.contains('section') || (x.tagName === 'SECTION')) + ? x.id : ''); + }; + + // Add anchors + h.forEach(function(x) { + const id = x.id || section_id(x.parentElement); + if (id === '' || x.matches(':empty')) { + return null; + } + let anchor = document.createElement('a'); + anchor.href = '#' + id; + anchor.classList = ['anchor-section']; + x.classList.add('hasAnchor'); + x.appendChild(anchor); + }); +}); diff --git a/docs/manual/libs/gitbook-2.6.7/css/plugin-bookdown.css b/docs/manual/libs/gitbook-2.6.7/css/plugin-bookdown.css index 0ae0f665..ab7c20eb 100644 --- a/docs/manual/libs/gitbook-2.6.7/css/plugin-bookdown.css +++ b/docs/manual/libs/gitbook-2.6.7/css/plugin-bookdown.css @@ -97,3 +97,9 @@ div.proof>*:last-child:after { .header-section-number { padding-right: .5em; } +#header .multi-author { + margin: 0.5em 0 -0.5em 0; +} +#header .date { + margin-top: 1.5em; +} diff --git a/docs/manual/libs/gitbook-2.6.7/css/plugin-fontsettings.css b/docs/manual/libs/gitbook-2.6.7/css/plugin-fontsettings.css index 87236b4c..3fa6f35b 100644 --- a/docs/manual/libs/gitbook-2.6.7/css/plugin-fontsettings.css +++ b/docs/manual/libs/gitbook-2.6.7/css/plugin-fontsettings.css @@ -44,6 +44,11 @@ line-height: 30px; font-size: 1em; } + +/* sidebar transition background */ +div.book.color-theme-1 { + background: #f3eacb; +} .book.color-theme-1 .book-body { color: #704214; background: #f3eacb; @@ -51,6 +56,12 @@ .book.color-theme-1 .book-body .page-wrapper .page-inner section { background: #f3eacb; } + +/* sidebar transition background */ +div.book.color-theme-2 { + background: #1c1f2b; +} + .book.color-theme-2 .book-body { color: #bdcadb; background: #1c1f2b; diff --git a/docs/manual/libs/gitbook-2.6.7/css/style.css b/docs/manual/libs/gitbook-2.6.7/css/style.css index b8968920..1b0c622a 100644 --- a/docs/manual/libs/gitbook-2.6.7/css/style.css +++ b/docs/manual/libs/gitbook-2.6.7/css/style.css @@ -4,7 +4,12 @@ * Open sourced under MIT license by @mdo. * Some variables and mixins from Bootstrap (Apache 2 license). */.link-inherit,.link-inherit:focus,.link-inherit:hover{color:inherit}.fa,.fa-stack{display:inline-block}/*! - * Font Awesome 4.1.0 by @davegandy - http://fontawesome.io - @fontawesome + * Font Awesome 4.7.0 by @davegandy - http://fontawesome.io - @fontawesome * License - http://fontawesome.io/license (Font: SIL OFL 1.1, CSS: MIT License) - */@font-face{font-family:FontAwesome;src:url(./fontawesome/fontawesome-webfont.ttf?v=4.1.0) format('truetype');font-weight:400;font-style:normal}.fa{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1;-moz-osx-font-smoothing:grayscale}.book .book-header,.book .book-summary{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:.08em solid #eee;border-radius:.1em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:spin 2s infinite linear;-moz-animation:spin 2s infinite linear;-o-animation:spin 2s infinite linear;animation:spin 2s infinite linear}@-moz-keyframes spin{0%{-moz-transform:rotate(0)}100%{-moz-transform:rotate(359deg)}}@-webkit-keyframes spin{0%{-webkit-transform:rotate(0)}100%{-webkit-transform:rotate(359deg)}}@-o-keyframes spin{0%{-o-transform:rotate(0)}100%{-o-transform:rotate(359deg)}}@keyframes spin{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2);-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=3);-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1);-webkit-transform:scale(-1,1);-moz-transform:scale(-1,1);-ms-transform:scale(-1,1);-o-transform:scale(-1,1);transform:scale(-1,1)}.fa-flip-vertical{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1);-webkit-transform:scale(1,-1);-moz-transform:scale(1,-1);-ms-transform:scale(1,-1);-o-transform:scale(1,-1);transform:scale(1,-1)}.fa-stack{position:relative;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-cog:before,.fa-gear:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-repeat:before,.fa-rotate-right:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-image:before,.fa-photo:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-exclamation-triangle:before,.fa-warning:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-cogs:before,.fa-gears:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-floppy-o:before,.fa-save:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-bars:before,.fa-navicon:before,.fa-reorder:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-sort:before,.fa-unsorted:before{content:"\f0dc"}.fa-sort-desc:before,.fa-sort-down:before{content:"\f0dd"}.fa-sort-asc:before,.fa-sort-up:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-gavel:before,.fa-legal:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-bolt:before,.fa-flash:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-clipboard:before,.fa-paste:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-chain-broken:before,.fa-unlink:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-caret-square-o-down:before,.fa-toggle-down:before{content:"\f150"}.fa-caret-square-o-up:before,.fa-toggle-up:before{content:"\f151"}.fa-caret-square-o-right:before,.fa-toggle-right:before{content:"\f152"}.fa-eur:before,.fa-euro:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-inr:before,.fa-rupee:before{content:"\f156"}.fa-cny:before,.fa-jpy:before,.fa-rmb:before,.fa-yen:before{content:"\f157"}.fa-rouble:before,.fa-rub:before,.fa-ruble:before{content:"\f158"}.fa-krw:before,.fa-won:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-caret-square-o-left:before,.fa-toggle-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-try:before,.fa-turkish-lira:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-bank:before,.fa-institution:before,.fa-university:before{content:"\f19c"}.fa-graduation-cap:before,.fa-mortar-board:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-square:before,.fa-pied-piper:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-image-o:before,.fa-file-photo-o:before,.fa-file-picture-o:before{content:"\f1c5"}.fa-file-archive-o:before,.fa-file-zip-o:before{content:"\f1c6"}.fa-file-audio-o:before,.fa-file-sound-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-ring:before,.fa-life-saver:before,.fa-support:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-rebel:before{content:"\f1d0"}.fa-empire:before,.fa-ge:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-paper-plane:before,.fa-send:before{content:"\f1d8"}.fa-paper-plane-o:before,.fa-send-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.book-langs-index{width:100%;height:100%;padding:40px 0;margin:0;overflow:auto}@media (max-width:600px){.book-langs-index{padding:0}}.book-langs-index .inner{max-width:600px;width:100%;margin:0 auto;padding:30px;background:#fff;border-radius:3px}.book-langs-index .inner h3{margin:0}.book-langs-index .inner .languages{list-style:none;padding:20px 30px;margin-top:20px;border-top:1px solid #eee}.book-langs-index .inner .languages:after,.book-langs-index .inner .languages:before{content:" ";display:table;line-height:0}.book-langs-index .inner .languages li{width:50%;float:left;padding:10px 5px;font-size:16px}@media (max-width:600px){.book-langs-index .inner .languages li{width:100%;max-width:100%}}.book .book-header{overflow:visible;height:50px;padding:0 8px;z-index:2;font-size:.85em;color:#7e888b;background:0 0}.book .book-header .btn{display:block;height:50px;padding:0 15px;border-bottom:none;color:#ccc;text-transform:uppercase;line-height:50px;-webkit-box-shadow:none!important;box-shadow:none!important;position:relative;font-size:14px}.book .book-header .btn:hover{position:relative;text-decoration:none;color:#444;background:0 0}.book .book-header h1{margin:0;font-size:20px;font-weight:200;text-align:center;line-height:50px;opacity:0;padding-left:200px;padding-right:200px;-webkit-transition:opacity .2s ease;-moz-transition:opacity .2s ease;-o-transition:opacity .2s ease;transition:opacity .2s ease;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.book .book-header h1 a,.book .book-header h1 a:hover{color:inherit;text-decoration:none}@media (max-width:1000px){.book .book-header h1{display:none}}.book .book-header h1 i{display:none}.book .book-header:hover h1{opacity:1}.book.is-loading .book-header h1 i{display:inline-block}.book.is-loading .book-header h1 a{display:none}.dropdown{position:relative}.dropdown-menu{position:absolute;top:100%;left:0;z-index:100;display:none;float:left;min-width:160px;padding:0;margin:2px 0 0;list-style:none;font-size:14px;background-color:#fafafa;border:1px solid rgba(0,0,0,.07);border-radius:1px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175);background-clip:padding-box}.dropdown-menu.open{display:block}.dropdown-menu.dropdown-left{left:auto;right:4%}.dropdown-menu.dropdown-left .dropdown-caret{right:14px;left:auto}.dropdown-menu .dropdown-caret{position:absolute;top:-8px;left:14px;width:18px;height:10px;float:left;overflow:hidden}.dropdown-menu .dropdown-caret .caret-inner,.dropdown-menu .dropdown-caret .caret-outer{display:inline-block;top:0;border-left:9px solid transparent;border-right:9px solid transparent;position:absolute}.dropdown-menu .dropdown-caret .caret-outer{border-bottom:9px solid rgba(0,0,0,.1);height:auto;left:0;width:auto;margin-left:-1px}.dropdown-menu .dropdown-caret .caret-inner{margin-top:-1px;top:1px;border-bottom:9px solid #fafafa}.dropdown-menu .buttons{border-bottom:1px solid rgba(0,0,0,.07)}.dropdown-menu .buttons:after,.dropdown-menu .buttons:before{content:" ";display:table;line-height:0}.dropdown-menu .buttons:last-child{border-bottom:none}.dropdown-menu .buttons .button{border:0;background-color:transparent;color:#a6a6a6;width:100%;text-align:center;float:left;line-height:1.42857143;padding:8px 4px}.alert,.dropdown-menu .buttons .button:hover{color:#444}.dropdown-menu .buttons .button:focus,.dropdown-menu .buttons .button:hover{outline:0}.dropdown-menu .buttons .button.size-2{width:50%}.dropdown-menu .buttons .button.size-3{width:33%}.alert{padding:15px;margin-bottom:20px;background:#eee;border-bottom:5px solid #ddd}.alert-success{background:#dff0d8;border-color:#d6e9c6;color:#3c763d}.alert-info{background:#d9edf7;border-color:#bce8f1;color:#31708f}.alert-danger{background:#f2dede;border-color:#ebccd1;color:#a94442}.alert-warning{background:#fcf8e3;border-color:#faebcc;color:#8a6d3b}.book .book-summary{position:absolute;top:0;left:-300px;bottom:0;z-index:1;width:300px;color:#364149;background:#fafafa;border-right:1px solid rgba(0,0,0,.07);-webkit-transition:left 250ms ease;-moz-transition:left 250ms ease;-o-transition:left 250ms ease;transition:left 250ms ease}.book .book-summary ul.summary{position:absolute;top:0;left:0;right:0;bottom:0;overflow-y:auto;list-style:none;margin:0;padding:0;-webkit-transition:top .5s ease;-moz-transition:top .5s ease;-o-transition:top .5s ease;transition:top .5s ease}.book .book-summary ul.summary li{list-style:none}.book .book-summary ul.summary li.divider{height:1px;margin:7px 0;overflow:hidden;background:rgba(0,0,0,.07)}.book .book-summary ul.summary li i.fa-check{display:none;position:absolute;right:9px;top:16px;font-size:9px;color:#3c3}.book .book-summary ul.summary li.done>a{color:#364149;font-weight:400}.book .book-summary ul.summary li.done>a i{display:inline}.book .book-summary ul.summary li a,.book .book-summary ul.summary li span{display:block;padding:10px 15px;border-bottom:none;color:#364149;background:0 0;text-overflow:ellipsis;overflow:hidden;white-space:nowrap;position:relative}.book .book-summary ul.summary li span{cursor:not-allowed;opacity:.3;filter:alpha(opacity=30)}.book .book-summary ul.summary li a:hover,.book .book-summary ul.summary li.active>a{color:#008cff;background:0 0;text-decoration:none}.book .book-summary ul.summary li ul{padding-left:20px}@media (max-width:600px){.book .book-summary{width:calc(100% - 60px);bottom:0;left:-100%}}.book.with-summary .book-summary{left:0}.book.without-animation .book-summary{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.book{position:relative;width:100%;height:100%}.book .book-body,.book .book-body .body-inner{position:absolute;top:0;left:0;overflow-y:auto;bottom:0;right:0}.book .book-body{color:#000;background:#fff;-webkit-transition:left 250ms ease;-moz-transition:left 250ms ease;-o-transition:left 250ms ease;transition:left 250ms ease}.book .book-body .page-wrapper{position:relative;outline:0}.book .book-body .page-wrapper .page-inner{max-width:800px;margin:0 auto;padding:20px 0 40px}.book .book-body .page-wrapper .page-inner section{margin:0;padding:5px 15px;background:#fff;border-radius:2px;line-height:1.7;font-size:1.6rem}.book .book-body .page-wrapper .page-inner .btn-group .btn{border-radius:0;background:#eee;border:0}@media (max-width:1240px){.book .book-body{-webkit-transition:-webkit-transform 250ms ease;-moz-transition:-moz-transform 250ms ease;-o-transition:-o-transform 250ms ease;transition:transform 250ms ease;padding-bottom:20px}.book .book-body .body-inner{position:static;min-height:calc(100% - 50px)}}@media (min-width:600px){.book.with-summary .book-body{left:300px}}@media (max-width:600px){.book.with-summary{overflow:hidden}.book.with-summary .book-body{-webkit-transform:translate(calc(100% - 60px),0);-moz-transform:translate(calc(100% - 60px),0);-ms-transform:translate(calc(100% - 60px),0);-o-transform:translate(calc(100% - 60px),0);transform:translate(calc(100% - 60px),0)}}.book.without-animation .book-body{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.buttons:after,.buttons:before{content:" ";display:table;line-height:0}.button{border:0;background:#eee;color:#666;width:100%;text-align:center;float:left;line-height:1.42857143;padding:8px 4px}.button:hover{color:#444}.button:focus,.button:hover{outline:0}.button.size-2{width:50%}.button.size-3{width:33%}.book .book-body .page-wrapper .page-inner section{display:none}.book .book-body .page-wrapper .page-inner section.normal{display:block;word-wrap:break-word;overflow:hidden;color:#333;line-height:1.7;text-size-adjust:100%;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%}.book .book-body .page-wrapper .page-inner section.normal *{box-sizing:border-box;-webkit-box-sizing:border-box;}.book .book-body .page-wrapper .page-inner section.normal>:first-child{margin-top:0!important}.book .book-body .page-wrapper .page-inner section.normal>:last-child{margin-bottom:0!important}.book .book-body .page-wrapper .page-inner section.normal blockquote,.book .book-body .page-wrapper .page-inner section.normal code,.book .book-body .page-wrapper .page-inner section.normal figure,.book .book-body .page-wrapper .page-inner section.normal img,.book .book-body .page-wrapper .page-inner section.normal pre,.book .book-body .page-wrapper .page-inner section.normal table,.book .book-body .page-wrapper .page-inner section.normal tr{page-break-inside:avoid}.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5,.book .book-body .page-wrapper .page-inner section.normal p{orphans:3;widows:3}.book .book-body .page-wrapper .page-inner section.normal h1,.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5{page-break-after:avoid}.book .book-body .page-wrapper .page-inner section.normal b,.book .book-body .page-wrapper .page-inner section.normal strong{font-weight:700}.book .book-body .page-wrapper .page-inner section.normal em{font-style:italic}.book .book-body .page-wrapper .page-inner section.normal blockquote,.book .book-body .page-wrapper .page-inner section.normal dl,.book .book-body .page-wrapper .page-inner section.normal ol,.book .book-body .page-wrapper .page-inner section.normal p,.book .book-body .page-wrapper .page-inner section.normal table,.book .book-body .page-wrapper .page-inner section.normal ul{margin-top:0;margin-bottom:.85em}.book .book-body .page-wrapper .page-inner section.normal a{color:#4183c4;text-decoration:none;background:0 0}.book .book-body .page-wrapper .page-inner section.normal a:active,.book .book-body .page-wrapper .page-inner section.normal a:focus,.book .book-body .page-wrapper .page-inner section.normal a:hover{outline:0;text-decoration:underline}.book .book-body .page-wrapper .page-inner section.normal img{border:0;max-width:100%}.book .book-body .page-wrapper .page-inner section.normal hr{height:4px;padding:0;margin:1.7em 0;overflow:hidden;background-color:#e7e7e7;border:none}.book .book-body .page-wrapper .page-inner section.normal hr:after,.book .book-body .page-wrapper .page-inner section.normal hr:before{display:table;content:" "}.book .book-body .page-wrapper .page-inner section.normal h1,.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5,.book .book-body .page-wrapper .page-inner section.normal h6{margin-top:1.275em;margin-bottom:.85em;}.book .book-body .page-wrapper .page-inner section.normal h1{font-size:2em}.book .book-body .page-wrapper .page-inner section.normal h2{font-size:1.75em}.book .book-body .page-wrapper .page-inner section.normal h3{font-size:1.5em}.book .book-body .page-wrapper .page-inner section.normal h4{font-size:1.25em}.book .book-body .page-wrapper .page-inner section.normal h5{font-size:1em}.book .book-body .page-wrapper .page-inner section.normal h6{font-size:1em;color:#777}.book .book-body .page-wrapper .page-inner section.normal code,.book .book-body .page-wrapper .page-inner section.normal pre{font-family:Consolas,"Liberation Mono",Menlo,Courier,monospace;direction:ltr;border:none;color:inherit}.book .book-body .page-wrapper .page-inner section.normal pre{overflow:auto;word-wrap:normal;margin:0 0 1.275em;padding:.85em 1em;background:#f7f7f7}.book .book-body .page-wrapper .page-inner section.normal pre>code{display:inline;max-width:initial;padding:0;margin:0;overflow:initial;line-height:inherit;font-size:.85em;white-space:pre;background:0 0}.book .book-body .page-wrapper .page-inner section.normal pre>code:after,.book .book-body .page-wrapper .page-inner section.normal pre>code:before{content:normal}.book .book-body .page-wrapper .page-inner section.normal code{padding:.2em;margin:0;font-size:.85em;background-color:#f7f7f7}.book .book-body .page-wrapper .page-inner section.normal code:after,.book .book-body .page-wrapper .page-inner section.normal code:before{letter-spacing:-.2em;content:"\00a0"}.book .book-body .page-wrapper .page-inner section.normal ol,.book .book-body .page-wrapper .page-inner section.normal ul{padding:0 0 0 2em;margin:0 0 .85em}.book .book-body .page-wrapper .page-inner section.normal ol ol,.book .book-body .page-wrapper .page-inner section.normal ol ul,.book .book-body .page-wrapper .page-inner section.normal ul ol,.book .book-body .page-wrapper .page-inner section.normal ul ul{margin-top:0;margin-bottom:0}.book .book-body .page-wrapper .page-inner section.normal ol ol{list-style-type:lower-roman}.book .book-body .page-wrapper .page-inner section.normal blockquote{margin:0 0 .85em;padding:0 15px;opacity:0.75;border-left:4px solid #dcdcdc}.book .book-body .page-wrapper .page-inner section.normal blockquote:first-child{margin-top:0}.book .book-body .page-wrapper .page-inner section.normal blockquote:last-child{margin-bottom:0}.book .book-body .page-wrapper .page-inner section.normal dl{padding:0}.book .book-body .page-wrapper .page-inner section.normal dl dt{padding:0;margin-top:.85em;font-style:italic;font-weight:700}.book .book-body .page-wrapper .page-inner section.normal dl dd{padding:0 .85em;margin-bottom:.85em}.book .book-body .page-wrapper .page-inner section.normal dd{margin-left:0}.book .book-body .page-wrapper .page-inner section.normal .glossary-term{cursor:help;text-decoration:underline}.book .book-body .navigation{position:absolute;top:50px;bottom:0;margin:0;max-width:150px;min-width:90px;display:flex;justify-content:center;align-content:center;flex-direction:column;font-size:40px;color:#ccc;text-align:center;-webkit-transition:all 350ms ease;-moz-transition:all 350ms ease;-o-transition:all 350ms ease;transition:all 350ms ease}.book .book-body .navigation:hover{text-decoration:none;color:#444}.book .book-body .navigation.navigation-next{right:0}.book .book-body .navigation.navigation-prev{left:0}@media (max-width:1240px){.book .book-body .navigation{position:static;top:auto;max-width:50%;width:50%;display:inline-block;float:left}.book .book-body .navigation.navigation-unique{max-width:100%;width:100%}}.book .book-body .page-wrapper .page-inner section.glossary{margin-bottom:40px}.book .book-body .page-wrapper .page-inner section.glossary h2 a,.book .book-body .page-wrapper .page-inner section.glossary h2 a:hover{color:inherit;text-decoration:none}.book .book-body .page-wrapper .page-inner section.glossary .glossary-index{list-style:none;margin:0;padding:0}.book .book-body .page-wrapper .page-inner section.glossary .glossary-index li{display:inline;margin:0 8px;white-space:nowrap}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-overflow-scrolling:touch;-webkit-tap-highlight-color:transparent;-webkit-text-size-adjust:none;-webkit-touch-callout:none}a{text-decoration:none}body,html{height:100%}html{font-size:62.5%}body{text-rendering:optimizeLegibility;font-smoothing:antialiased;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;letter-spacing:.2px;text-size-adjust:100%} + */@font-face{font-family:FontAwesome;src:url(./fontawesome/fontawesome-webfont.ttf?v=4.7.0) format('truetype');font-weight:400;font-style:normal}.fa{font-family:FontAwesome;font-style:normal;font-weight:400;line-height:1;-moz-osx-font-smoothing:grayscale}.book .book-header,.book .book-summary{font-family:"Helvetica Neue",Helvetica,Arial,sans-serif}.fa-lg{font-size:1.33333333em;line-height:.75em;vertical-align:-15%}.fa-2x{font-size:2em}.fa-3x{font-size:3em}.fa-4x{font-size:4em}.fa-5x{font-size:5em}.fa-fw{width:1.28571429em;text-align:center}.fa-ul{padding-left:0;margin-left:2.14285714em;list-style-type:none}.fa-li{position:absolute;left:-2.14285714em;width:2.14285714em;top:.14285714em;text-align:center}.fa-li.fa-lg{left:-1.85714286em}.fa-border{padding:.2em .25em .15em;border:.08em solid #eee;border-radius:.1em}.pull-right{float:right}.pull-left{float:left}.fa.pull-left{margin-right:.3em}.fa.pull-right{margin-left:.3em}.fa-spin{-webkit-animation:spin 2s infinite linear;-moz-animation:spin 2s infinite linear;-o-animation:spin 2s infinite linear;animation:spin 2s infinite linear}@-moz-keyframes spin{0%{-moz-transform:rotate(0)}100%{-moz-transform:rotate(359deg)}}@-webkit-keyframes spin{0%{-webkit-transform:rotate(0)}100%{-webkit-transform:rotate(359deg)}}@-o-keyframes spin{0%{-o-transform:rotate(0)}100%{-o-transform:rotate(359deg)}}@keyframes spin{0%{-webkit-transform:rotate(0);transform:rotate(0)}100%{-webkit-transform:rotate(359deg);transform:rotate(359deg)}}.fa-rotate-90{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);-webkit-transform:rotate(90deg);-moz-transform:rotate(90deg);-ms-transform:rotate(90deg);-o-transform:rotate(90deg);transform:rotate(90deg)}.fa-rotate-180{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2);-webkit-transform:rotate(180deg);-moz-transform:rotate(180deg);-ms-transform:rotate(180deg);-o-transform:rotate(180deg);transform:rotate(180deg)}.fa-rotate-270{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=3);-webkit-transform:rotate(270deg);-moz-transform:rotate(270deg);-ms-transform:rotate(270deg);-o-transform:rotate(270deg);transform:rotate(270deg)}.fa-flip-horizontal{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=0, mirror=1);-webkit-transform:scale(-1,1);-moz-transform:scale(-1,1);-ms-transform:scale(-1,1);-o-transform:scale(-1,1);transform:scale(-1,1)}.fa-flip-vertical{filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=2, mirror=1);-webkit-transform:scale(1,-1);-moz-transform:scale(1,-1);-ms-transform:scale(1,-1);-o-transform:scale(1,-1);transform:scale(1,-1)}.fa-stack{position:relative;width:2em;height:2em;line-height:2em;vertical-align:middle}.fa-stack-1x,.fa-stack-2x{position:absolute;left:0;width:100%;text-align:center}.fa-stack-1x{line-height:inherit}.fa-stack-2x{font-size:2em}.fa-inverse{color:#fff}.fa-glass:before{content:"\f000"}.fa-music:before{content:"\f001"}.fa-search:before{content:"\f002"}.fa-envelope-o:before{content:"\f003"}.fa-heart:before{content:"\f004"}.fa-star:before{content:"\f005"}.fa-star-o:before{content:"\f006"}.fa-user:before{content:"\f007"}.fa-film:before{content:"\f008"}.fa-th-large:before{content:"\f009"}.fa-th:before{content:"\f00a"}.fa-th-list:before{content:"\f00b"}.fa-check:before{content:"\f00c"}.fa-times:before{content:"\f00d"}.fa-search-plus:before{content:"\f00e"}.fa-search-minus:before{content:"\f010"}.fa-power-off:before{content:"\f011"}.fa-signal:before{content:"\f012"}.fa-cog:before,.fa-gear:before{content:"\f013"}.fa-trash-o:before{content:"\f014"}.fa-home:before{content:"\f015"}.fa-file-o:before{content:"\f016"}.fa-clock-o:before{content:"\f017"}.fa-road:before{content:"\f018"}.fa-download:before{content:"\f019"}.fa-arrow-circle-o-down:before{content:"\f01a"}.fa-arrow-circle-o-up:before{content:"\f01b"}.fa-inbox:before{content:"\f01c"}.fa-play-circle-o:before{content:"\f01d"}.fa-repeat:before,.fa-rotate-right:before{content:"\f01e"}.fa-refresh:before{content:"\f021"}.fa-list-alt:before{content:"\f022"}.fa-lock:before{content:"\f023"}.fa-flag:before{content:"\f024"}.fa-headphones:before{content:"\f025"}.fa-volume-off:before{content:"\f026"}.fa-volume-down:before{content:"\f027"}.fa-volume-up:before{content:"\f028"}.fa-qrcode:before{content:"\f029"}.fa-barcode:before{content:"\f02a"}.fa-tag:before{content:"\f02b"}.fa-tags:before{content:"\f02c"}.fa-book:before{content:"\f02d"}.fa-bookmark:before{content:"\f02e"}.fa-print:before{content:"\f02f"}.fa-camera:before{content:"\f030"}.fa-font:before{content:"\f031"}.fa-bold:before{content:"\f032"}.fa-italic:before{content:"\f033"}.fa-text-height:before{content:"\f034"}.fa-text-width:before{content:"\f035"}.fa-align-left:before{content:"\f036"}.fa-align-center:before{content:"\f037"}.fa-align-right:before{content:"\f038"}.fa-align-justify:before{content:"\f039"}.fa-list:before{content:"\f03a"}.fa-dedent:before,.fa-outdent:before{content:"\f03b"}.fa-indent:before{content:"\f03c"}.fa-video-camera:before{content:"\f03d"}.fa-image:before,.fa-photo:before,.fa-picture-o:before{content:"\f03e"}.fa-pencil:before{content:"\f040"}.fa-map-marker:before{content:"\f041"}.fa-adjust:before{content:"\f042"}.fa-tint:before{content:"\f043"}.fa-edit:before,.fa-pencil-square-o:before{content:"\f044"}.fa-share-square-o:before{content:"\f045"}.fa-check-square-o:before{content:"\f046"}.fa-arrows:before{content:"\f047"}.fa-step-backward:before{content:"\f048"}.fa-fast-backward:before{content:"\f049"}.fa-backward:before{content:"\f04a"}.fa-play:before{content:"\f04b"}.fa-pause:before{content:"\f04c"}.fa-stop:before{content:"\f04d"}.fa-forward:before{content:"\f04e"}.fa-fast-forward:before{content:"\f050"}.fa-step-forward:before{content:"\f051"}.fa-eject:before{content:"\f052"}.fa-chevron-left:before{content:"\f053"}.fa-chevron-right:before{content:"\f054"}.fa-plus-circle:before{content:"\f055"}.fa-minus-circle:before{content:"\f056"}.fa-times-circle:before{content:"\f057"}.fa-check-circle:before{content:"\f058"}.fa-question-circle:before{content:"\f059"}.fa-info-circle:before{content:"\f05a"}.fa-crosshairs:before{content:"\f05b"}.fa-times-circle-o:before{content:"\f05c"}.fa-check-circle-o:before{content:"\f05d"}.fa-ban:before{content:"\f05e"}.fa-arrow-left:before{content:"\f060"}.fa-arrow-right:before{content:"\f061"}.fa-arrow-up:before{content:"\f062"}.fa-arrow-down:before{content:"\f063"}.fa-mail-forward:before,.fa-share:before{content:"\f064"}.fa-expand:before{content:"\f065"}.fa-compress:before{content:"\f066"}.fa-plus:before{content:"\f067"}.fa-minus:before{content:"\f068"}.fa-asterisk:before{content:"\f069"}.fa-exclamation-circle:before{content:"\f06a"}.fa-gift:before{content:"\f06b"}.fa-leaf:before{content:"\f06c"}.fa-fire:before{content:"\f06d"}.fa-eye:before{content:"\f06e"}.fa-eye-slash:before{content:"\f070"}.fa-exclamation-triangle:before,.fa-warning:before{content:"\f071"}.fa-plane:before{content:"\f072"}.fa-calendar:before{content:"\f073"}.fa-random:before{content:"\f074"}.fa-comment:before{content:"\f075"}.fa-magnet:before{content:"\f076"}.fa-chevron-up:before{content:"\f077"}.fa-chevron-down:before{content:"\f078"}.fa-retweet:before{content:"\f079"}.fa-shopping-cart:before{content:"\f07a"}.fa-folder:before{content:"\f07b"}.fa-folder-open:before{content:"\f07c"}.fa-arrows-v:before{content:"\f07d"}.fa-arrows-h:before{content:"\f07e"}.fa-bar-chart-o:before{content:"\f080"}.fa-twitter-square:before{content:"\f081"}.fa-facebook-square:before{content:"\f082"}.fa-camera-retro:before{content:"\f083"}.fa-key:before{content:"\f084"}.fa-cogs:before,.fa-gears:before{content:"\f085"}.fa-comments:before{content:"\f086"}.fa-thumbs-o-up:before{content:"\f087"}.fa-thumbs-o-down:before{content:"\f088"}.fa-star-half:before{content:"\f089"}.fa-heart-o:before{content:"\f08a"}.fa-sign-out:before{content:"\f08b"}.fa-linkedin-square:before{content:"\f08c"}.fa-thumb-tack:before{content:"\f08d"}.fa-external-link:before{content:"\f08e"}.fa-sign-in:before{content:"\f090"}.fa-trophy:before{content:"\f091"}.fa-github-square:before{content:"\f092"}.fa-upload:before{content:"\f093"}.fa-lemon-o:before{content:"\f094"}.fa-phone:before{content:"\f095"}.fa-square-o:before{content:"\f096"}.fa-bookmark-o:before{content:"\f097"}.fa-phone-square:before{content:"\f098"}.fa-twitter:before{content:"\f099"}.fa-facebook:before{content:"\f09a"}.fa-github:before{content:"\f09b"}.fa-unlock:before{content:"\f09c"}.fa-credit-card:before{content:"\f09d"}.fa-rss:before{content:"\f09e"}.fa-hdd-o:before{content:"\f0a0"}.fa-bullhorn:before{content:"\f0a1"}.fa-bell:before{content:"\f0f3"}.fa-certificate:before{content:"\f0a3"}.fa-hand-o-right:before{content:"\f0a4"}.fa-hand-o-left:before{content:"\f0a5"}.fa-hand-o-up:before{content:"\f0a6"}.fa-hand-o-down:before{content:"\f0a7"}.fa-arrow-circle-left:before{content:"\f0a8"}.fa-arrow-circle-right:before{content:"\f0a9"}.fa-arrow-circle-up:before{content:"\f0aa"}.fa-arrow-circle-down:before{content:"\f0ab"}.fa-globe:before{content:"\f0ac"}.fa-wrench:before{content:"\f0ad"}.fa-tasks:before{content:"\f0ae"}.fa-filter:before{content:"\f0b0"}.fa-briefcase:before{content:"\f0b1"}.fa-arrows-alt:before{content:"\f0b2"}.fa-group:before,.fa-users:before{content:"\f0c0"}.fa-chain:before,.fa-link:before{content:"\f0c1"}.fa-cloud:before{content:"\f0c2"}.fa-flask:before{content:"\f0c3"}.fa-cut:before,.fa-scissors:before{content:"\f0c4"}.fa-copy:before,.fa-files-o:before{content:"\f0c5"}.fa-paperclip:before{content:"\f0c6"}.fa-floppy-o:before,.fa-save:before{content:"\f0c7"}.fa-square:before{content:"\f0c8"}.fa-bars:before,.fa-navicon:before,.fa-reorder:before{content:"\f0c9"}.fa-list-ul:before{content:"\f0ca"}.fa-list-ol:before{content:"\f0cb"}.fa-strikethrough:before{content:"\f0cc"}.fa-underline:before{content:"\f0cd"}.fa-table:before{content:"\f0ce"}.fa-magic:before{content:"\f0d0"}.fa-truck:before{content:"\f0d1"}.fa-pinterest:before{content:"\f0d2"}.fa-pinterest-square:before{content:"\f0d3"}.fa-google-plus-square:before{content:"\f0d4"}.fa-google-plus:before{content:"\f0d5"}.fa-money:before{content:"\f0d6"}.fa-caret-down:before{content:"\f0d7"}.fa-caret-up:before{content:"\f0d8"}.fa-caret-left:before{content:"\f0d9"}.fa-caret-right:before{content:"\f0da"}.fa-columns:before{content:"\f0db"}.fa-sort:before,.fa-unsorted:before{content:"\f0dc"}.fa-sort-desc:before,.fa-sort-down:before{content:"\f0dd"}.fa-sort-asc:before,.fa-sort-up:before{content:"\f0de"}.fa-envelope:before{content:"\f0e0"}.fa-linkedin:before{content:"\f0e1"}.fa-rotate-left:before,.fa-undo:before{content:"\f0e2"}.fa-gavel:before,.fa-legal:before{content:"\f0e3"}.fa-dashboard:before,.fa-tachometer:before{content:"\f0e4"}.fa-comment-o:before{content:"\f0e5"}.fa-comments-o:before{content:"\f0e6"}.fa-bolt:before,.fa-flash:before{content:"\f0e7"}.fa-sitemap:before{content:"\f0e8"}.fa-umbrella:before{content:"\f0e9"}.fa-clipboard:before,.fa-paste:before{content:"\f0ea"}.fa-lightbulb-o:before{content:"\f0eb"}.fa-exchange:before{content:"\f0ec"}.fa-cloud-download:before{content:"\f0ed"}.fa-cloud-upload:before{content:"\f0ee"}.fa-user-md:before{content:"\f0f0"}.fa-stethoscope:before{content:"\f0f1"}.fa-suitcase:before{content:"\f0f2"}.fa-bell-o:before{content:"\f0a2"}.fa-coffee:before{content:"\f0f4"}.fa-cutlery:before{content:"\f0f5"}.fa-file-text-o:before{content:"\f0f6"}.fa-building-o:before{content:"\f0f7"}.fa-hospital-o:before{content:"\f0f8"}.fa-ambulance:before{content:"\f0f9"}.fa-medkit:before{content:"\f0fa"}.fa-fighter-jet:before{content:"\f0fb"}.fa-beer:before{content:"\f0fc"}.fa-h-square:before{content:"\f0fd"}.fa-plus-square:before{content:"\f0fe"}.fa-angle-double-left:before{content:"\f100"}.fa-angle-double-right:before{content:"\f101"}.fa-angle-double-up:before{content:"\f102"}.fa-angle-double-down:before{content:"\f103"}.fa-angle-left:before{content:"\f104"}.fa-angle-right:before{content:"\f105"}.fa-angle-up:before{content:"\f106"}.fa-angle-down:before{content:"\f107"}.fa-desktop:before{content:"\f108"}.fa-laptop:before{content:"\f109"}.fa-tablet:before{content:"\f10a"}.fa-mobile-phone:before,.fa-mobile:before{content:"\f10b"}.fa-circle-o:before{content:"\f10c"}.fa-quote-left:before{content:"\f10d"}.fa-quote-right:before{content:"\f10e"}.fa-spinner:before{content:"\f110"}.fa-circle:before{content:"\f111"}.fa-mail-reply:before,.fa-reply:before{content:"\f112"}.fa-github-alt:before{content:"\f113"}.fa-folder-o:before{content:"\f114"}.fa-folder-open-o:before{content:"\f115"}.fa-smile-o:before{content:"\f118"}.fa-frown-o:before{content:"\f119"}.fa-meh-o:before{content:"\f11a"}.fa-gamepad:before{content:"\f11b"}.fa-keyboard-o:before{content:"\f11c"}.fa-flag-o:before{content:"\f11d"}.fa-flag-checkered:before{content:"\f11e"}.fa-terminal:before{content:"\f120"}.fa-code:before{content:"\f121"}.fa-mail-reply-all:before,.fa-reply-all:before{content:"\f122"}.fa-star-half-empty:before,.fa-star-half-full:before,.fa-star-half-o:before{content:"\f123"}.fa-location-arrow:before{content:"\f124"}.fa-crop:before{content:"\f125"}.fa-code-fork:before{content:"\f126"}.fa-chain-broken:before,.fa-unlink:before{content:"\f127"}.fa-question:before{content:"\f128"}.fa-info:before{content:"\f129"}.fa-exclamation:before{content:"\f12a"}.fa-superscript:before{content:"\f12b"}.fa-subscript:before{content:"\f12c"}.fa-eraser:before{content:"\f12d"}.fa-puzzle-piece:before{content:"\f12e"}.fa-microphone:before{content:"\f130"}.fa-microphone-slash:before{content:"\f131"}.fa-shield:before{content:"\f132"}.fa-calendar-o:before{content:"\f133"}.fa-fire-extinguisher:before{content:"\f134"}.fa-rocket:before{content:"\f135"}.fa-maxcdn:before{content:"\f136"}.fa-chevron-circle-left:before{content:"\f137"}.fa-chevron-circle-right:before{content:"\f138"}.fa-chevron-circle-up:before{content:"\f139"}.fa-chevron-circle-down:before{content:"\f13a"}.fa-html5:before{content:"\f13b"}.fa-css3:before{content:"\f13c"}.fa-anchor:before{content:"\f13d"}.fa-unlock-alt:before{content:"\f13e"}.fa-bullseye:before{content:"\f140"}.fa-ellipsis-h:before{content:"\f141"}.fa-ellipsis-v:before{content:"\f142"}.fa-rss-square:before{content:"\f143"}.fa-play-circle:before{content:"\f144"}.fa-ticket:before{content:"\f145"}.fa-minus-square:before{content:"\f146"}.fa-minus-square-o:before{content:"\f147"}.fa-level-up:before{content:"\f148"}.fa-level-down:before{content:"\f149"}.fa-check-square:before{content:"\f14a"}.fa-pencil-square:before{content:"\f14b"}.fa-external-link-square:before{content:"\f14c"}.fa-share-square:before{content:"\f14d"}.fa-compass:before{content:"\f14e"}.fa-caret-square-o-down:before,.fa-toggle-down:before{content:"\f150"}.fa-caret-square-o-up:before,.fa-toggle-up:before{content:"\f151"}.fa-caret-square-o-right:before,.fa-toggle-right:before{content:"\f152"}.fa-eur:before,.fa-euro:before{content:"\f153"}.fa-gbp:before{content:"\f154"}.fa-dollar:before,.fa-usd:before{content:"\f155"}.fa-inr:before,.fa-rupee:before{content:"\f156"}.fa-cny:before,.fa-jpy:before,.fa-rmb:before,.fa-yen:before{content:"\f157"}.fa-rouble:before,.fa-rub:before,.fa-ruble:before{content:"\f158"}.fa-krw:before,.fa-won:before{content:"\f159"}.fa-bitcoin:before,.fa-btc:before{content:"\f15a"}.fa-file:before{content:"\f15b"}.fa-file-text:before{content:"\f15c"}.fa-sort-alpha-asc:before{content:"\f15d"}.fa-sort-alpha-desc:before{content:"\f15e"}.fa-sort-amount-asc:before{content:"\f160"}.fa-sort-amount-desc:before{content:"\f161"}.fa-sort-numeric-asc:before{content:"\f162"}.fa-sort-numeric-desc:before{content:"\f163"}.fa-thumbs-up:before{content:"\f164"}.fa-thumbs-down:before{content:"\f165"}.fa-youtube-square:before{content:"\f166"}.fa-youtube:before{content:"\f167"}.fa-xing:before{content:"\f168"}.fa-xing-square:before{content:"\f169"}.fa-youtube-play:before{content:"\f16a"}.fa-dropbox:before{content:"\f16b"}.fa-stack-overflow:before{content:"\f16c"}.fa-instagram:before{content:"\f16d"}.fa-flickr:before{content:"\f16e"}.fa-adn:before{content:"\f170"}.fa-bitbucket:before{content:"\f171"}.fa-bitbucket-square:before{content:"\f172"}.fa-tumblr:before{content:"\f173"}.fa-tumblr-square:before{content:"\f174"}.fa-long-arrow-down:before{content:"\f175"}.fa-long-arrow-up:before{content:"\f176"}.fa-long-arrow-left:before{content:"\f177"}.fa-long-arrow-right:before{content:"\f178"}.fa-apple:before{content:"\f179"}.fa-windows:before{content:"\f17a"}.fa-android:before{content:"\f17b"}.fa-linux:before{content:"\f17c"}.fa-dribbble:before{content:"\f17d"}.fa-skype:before{content:"\f17e"}.fa-foursquare:before{content:"\f180"}.fa-trello:before{content:"\f181"}.fa-female:before{content:"\f182"}.fa-male:before{content:"\f183"}.fa-gittip:before{content:"\f184"}.fa-sun-o:before{content:"\f185"}.fa-moon-o:before{content:"\f186"}.fa-archive:before{content:"\f187"}.fa-bug:before{content:"\f188"}.fa-vk:before{content:"\f189"}.fa-weibo:before{content:"\f18a"}.fa-renren:before{content:"\f18b"}.fa-pagelines:before{content:"\f18c"}.fa-stack-exchange:before{content:"\f18d"}.fa-arrow-circle-o-right:before{content:"\f18e"}.fa-arrow-circle-o-left:before{content:"\f190"}.fa-caret-square-o-left:before,.fa-toggle-left:before{content:"\f191"}.fa-dot-circle-o:before{content:"\f192"}.fa-wheelchair:before{content:"\f193"}.fa-vimeo-square:before{content:"\f194"}.fa-try:before,.fa-turkish-lira:before{content:"\f195"}.fa-plus-square-o:before{content:"\f196"}.fa-space-shuttle:before{content:"\f197"}.fa-slack:before{content:"\f198"}.fa-envelope-square:before{content:"\f199"}.fa-wordpress:before{content:"\f19a"}.fa-openid:before{content:"\f19b"}.fa-bank:before,.fa-institution:before,.fa-university:before{content:"\f19c"}.fa-graduation-cap:before,.fa-mortar-board:before{content:"\f19d"}.fa-yahoo:before{content:"\f19e"}.fa-google:before{content:"\f1a0"}.fa-reddit:before{content:"\f1a1"}.fa-reddit-square:before{content:"\f1a2"}.fa-stumbleupon-circle:before{content:"\f1a3"}.fa-stumbleupon:before{content:"\f1a4"}.fa-delicious:before{content:"\f1a5"}.fa-digg:before{content:"\f1a6"}.fa-pied-piper-square:before,.fa-pied-piper:before{content:"\f1a7"}.fa-pied-piper-alt:before{content:"\f1a8"}.fa-drupal:before{content:"\f1a9"}.fa-joomla:before{content:"\f1aa"}.fa-language:before{content:"\f1ab"}.fa-fax:before{content:"\f1ac"}.fa-building:before{content:"\f1ad"}.fa-child:before{content:"\f1ae"}.fa-paw:before{content:"\f1b0"}.fa-spoon:before{content:"\f1b1"}.fa-cube:before{content:"\f1b2"}.fa-cubes:before{content:"\f1b3"}.fa-behance:before{content:"\f1b4"}.fa-behance-square:before{content:"\f1b5"}.fa-steam:before{content:"\f1b6"}.fa-steam-square:before{content:"\f1b7"}.fa-recycle:before{content:"\f1b8"}.fa-automobile:before,.fa-car:before{content:"\f1b9"}.fa-cab:before,.fa-taxi:before{content:"\f1ba"}.fa-tree:before{content:"\f1bb"}.fa-spotify:before{content:"\f1bc"}.fa-deviantart:before{content:"\f1bd"}.fa-soundcloud:before{content:"\f1be"}.fa-database:before{content:"\f1c0"}.fa-file-pdf-o:before{content:"\f1c1"}.fa-file-word-o:before{content:"\f1c2"}.fa-file-excel-o:before{content:"\f1c3"}.fa-file-powerpoint-o:before{content:"\f1c4"}.fa-file-image-o:before,.fa-file-photo-o:before,.fa-file-picture-o:before{content:"\f1c5"}.fa-file-archive-o:before,.fa-file-zip-o:before{content:"\f1c6"}.fa-file-audio-o:before,.fa-file-sound-o:before{content:"\f1c7"}.fa-file-movie-o:before,.fa-file-video-o:before{content:"\f1c8"}.fa-file-code-o:before{content:"\f1c9"}.fa-vine:before{content:"\f1ca"}.fa-codepen:before{content:"\f1cb"}.fa-jsfiddle:before{content:"\f1cc"}.fa-life-bouy:before,.fa-life-ring:before,.fa-life-saver:before,.fa-support:before{content:"\f1cd"}.fa-circle-o-notch:before{content:"\f1ce"}.fa-ra:before,.fa-rebel:before{content:"\f1d0"}.fa-empire:before,.fa-ge:before{content:"\f1d1"}.fa-git-square:before{content:"\f1d2"}.fa-git:before{content:"\f1d3"}.fa-hacker-news:before{content:"\f1d4"}.fa-tencent-weibo:before{content:"\f1d5"}.fa-qq:before{content:"\f1d6"}.fa-wechat:before,.fa-weixin:before{content:"\f1d7"}.fa-paper-plane:before,.fa-send:before{content:"\f1d8"}.fa-paper-plane-o:before,.fa-send-o:before{content:"\f1d9"}.fa-history:before{content:"\f1da"}.fa-circle-thin:before{content:"\f1db"}.fa-header:before{content:"\f1dc"}.fa-paragraph:before{content:"\f1dd"}.fa-sliders:before{content:"\f1de"}.fa-share-alt:before{content:"\f1e0"}.fa-share-alt-square:before{content:"\f1e1"}.fa-bomb:before{content:"\f1e2"}.book-langs-index{width:100%;height:100%;padding:40px 0;margin:0;overflow:auto}@media (max-width:600px){.book-langs-index{padding:0}}.book-langs-index .inner{max-width:600px;width:100%;margin:0 auto;padding:30px;background:#fff;border-radius:3px}.book-langs-index .inner h3{margin:0}.book-langs-index .inner .languages{list-style:none;padding:20px 30px;margin-top:20px;border-top:1px solid #eee}.book-langs-index .inner .languages:after,.book-langs-index .inner .languages:before{content:" ";display:table;line-height:0}.book-langs-index .inner .languages li{width:50%;float:left;padding:10px 5px;font-size:16px}@media (max-width:600px){.book-langs-index .inner .languages li{width:100%;max-width:100%}}.book .book-header{overflow:visible;height:50px;padding:0 8px;z-index:2;font-size:.85em;color:#7e888b;background:0 0}.book .book-header .btn{display:block;height:50px;padding:0 15px;border-bottom:none;color:#ccc;text-transform:uppercase;line-height:50px;-webkit-box-shadow:none!important;box-shadow:none!important;position:relative;font-size:14px}.book .book-header .btn:hover{position:relative;text-decoration:none;color:#444;background:0 0}.book .book-header h1{margin:0;font-size:20px;font-weight:200;text-align:center;line-height:50px;opacity:0;padding-left:200px;padding-right:200px;-webkit-transition:opacity .2s ease;-moz-transition:opacity .2s ease;-o-transition:opacity .2s ease;transition:opacity .2s ease;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.book .book-header h1 a,.book .book-header h1 a:hover{color:inherit;text-decoration:none}@media (max-width:1000px){.book .book-header h1{display:none}}.book .book-header h1 i{display:none}.book .book-header:hover h1{opacity:1}.book.is-loading .book-header h1 i{display:inline-block}.book.is-loading .book-header h1 a{display:none}.dropdown{position:relative}.dropdown-menu{position:absolute;top:100%;left:0;z-index:100;display:none;float:left;min-width:160px;padding:0;margin:2px 0 0;list-style:none;font-size:14px;background-color:#fafafa;border:1px solid rgba(0,0,0,.07);border-radius:1px;-webkit-box-shadow:0 6px 12px rgba(0,0,0,.175);box-shadow:0 6px 12px rgba(0,0,0,.175);background-clip:padding-box}.dropdown-menu.open{display:block}.dropdown-menu.dropdown-left{left:auto;right:4%}.dropdown-menu.dropdown-left .dropdown-caret{right:14px;left:auto}.dropdown-menu .dropdown-caret{position:absolute;top:-8px;left:14px;width:18px;height:10px;float:left;overflow:hidden}.dropdown-menu .dropdown-caret .caret-inner,.dropdown-menu .dropdown-caret .caret-outer{display:inline-block;top:0;border-left:9px solid transparent;border-right:9px solid transparent;position:absolute}.dropdown-menu .dropdown-caret .caret-outer{border-bottom:9px solid rgba(0,0,0,.1);height:auto;left:0;width:auto;margin-left:-1px}.dropdown-menu .dropdown-caret .caret-inner{margin-top:-1px;top:1px;border-bottom:9px solid #fafafa}.dropdown-menu .buttons{border-bottom:1px solid rgba(0,0,0,.07)}.dropdown-menu .buttons:after,.dropdown-menu .buttons:before{content:" ";display:table;line-height:0}.dropdown-menu .buttons:last-child{border-bottom:none}.dropdown-menu .buttons .button{border:0;background-color:transparent;color:#a6a6a6;width:100%;text-align:center;float:left;line-height:1.42857143;padding:8px 4px}.alert,.dropdown-menu .buttons .button:hover{color:#444}.dropdown-menu .buttons .button:focus,.dropdown-menu .buttons .button:hover{outline:0}.dropdown-menu .buttons .button.size-2{width:50%}.dropdown-menu .buttons .button.size-3{width:33%}.alert{padding:15px;margin-bottom:20px;background:#eee;border-bottom:5px solid #ddd}.alert-success{background:#dff0d8;border-color:#d6e9c6;color:#3c763d}.alert-info{background:#d9edf7;border-color:#bce8f1;color:#31708f}.alert-danger{background:#f2dede;border-color:#ebccd1;color:#a94442}.alert-warning{background:#fcf8e3;border-color:#faebcc;color:#8a6d3b}.book .book-summary{position:absolute;top:0;left:-300px;bottom:0;z-index:1;width:300px;color:#364149;background:#fafafa;border-right:1px solid rgba(0,0,0,.07);-webkit-transition:left 250ms ease;-moz-transition:left 250ms ease;-o-transition:left 250ms ease;transition:left 250ms ease}.book .book-summary ul.summary{position:absolute;top:0;left:0;right:0;bottom:0;overflow-y:auto;list-style:none;margin:0;padding:0;-webkit-transition:top .5s ease;-moz-transition:top .5s ease;-o-transition:top .5s ease;transition:top .5s ease}.book .book-summary ul.summary li{list-style:none}.book .book-summary ul.summary li.divider{height:1px;margin:7px 0;overflow:hidden;background:rgba(0,0,0,.07)}.book .book-summary ul.summary li i.fa-check{display:none;position:absolute;right:9px;top:16px;font-size:9px;color:#3c3}.book .book-summary ul.summary li.done>a{color:#364149;font-weight:400}.book .book-summary ul.summary li.done>a i{display:inline}.book .book-summary ul.summary li a,.book .book-summary ul.summary li span{display:block;padding:10px 15px;border-bottom:none;color:#364149;background:0 0;text-overflow:ellipsis;overflow:hidden;white-space:nowrap;position:relative}.book .book-summary ul.summary li span{cursor:not-allowed;opacity:.3;filter:alpha(opacity=30)}.book .book-summary ul.summary li a:hover,.book .book-summary ul.summary li.active>a{color:#008cff;background:0 0;text-decoration:none}.book .book-summary ul.summary li ul{padding-left:20px}@media (max-width:600px){.book .book-summary{width:calc(100% - 60px);bottom:0;left:-100%}}.book.with-summary .book-summary{left:0}.book.without-animation .book-summary{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.book{position:relative;width:100%;height:100%}.book .book-body,.book .book-body .body-inner{position:absolute;top:0;left:0;overflow-y:auto;bottom:0;right:0}.book .book-body{color:#000;background:#fff;-webkit-transition:left 250ms ease;-moz-transition:left 250ms ease;-o-transition:left 250ms ease;transition:left 250ms ease}.book .book-body .page-wrapper{position:relative;outline:0}.book .book-body .page-wrapper .page-inner{max-width:800px;margin:0 auto;padding:20px 0 40px}.book .book-body .page-wrapper .page-inner section{margin:0;padding:5px 15px;background:#fff;border-radius:2px;line-height:1.7;font-size:1.6rem}.book .book-body .page-wrapper .page-inner .btn-group .btn{border-radius:0;background:#eee;border:0}@media (max-width:1240px){.book .book-body{-webkit-transition:-webkit-transform 250ms ease;-moz-transition:-moz-transform 250ms ease;-o-transition:-o-transform 250ms ease;transition:transform 250ms ease;padding-bottom:20px}.book .book-body .body-inner{position:static;min-height:calc(100% - 50px)}}@media (min-width:600px){.book.with-summary .book-body{left:300px}}@media (max-width:600px){.book.with-summary{overflow:hidden}.book.with-summary .book-body{-webkit-transform:translate(calc(100% - 60px),0);-moz-transform:translate(calc(100% - 60px),0);-ms-transform:translate(calc(100% - 60px),0);-o-transform:translate(calc(100% - 60px),0);transform:translate(calc(100% - 60px),0)}}.book.without-animation .book-body{-webkit-transition:none!important;-moz-transition:none!important;-o-transition:none!important;transition:none!important}.buttons:after,.buttons:before{content:" ";display:table;line-height:0}.button{border:0;background:#eee;color:#666;width:100%;text-align:center;float:left;line-height:1.42857143;padding:8px 4px}.button:hover{color:#444}.button:focus,.button:hover{outline:0}.button.size-2{width:50%}.button.size-3{width:33%}.book .book-body .page-wrapper .page-inner section{display:none}.book .book-body .page-wrapper .page-inner section.normal{display:block;word-wrap:break-word;overflow:hidden;color:#333;line-height:1.7;text-size-adjust:100%;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%;-moz-text-size-adjust:100%}.book .book-body .page-wrapper .page-inner section.normal *{box-sizing:border-box;-webkit-box-sizing:border-box;}.book .book-body .page-wrapper .page-inner section.normal>:first-child{margin-top:0!important}.book .book-body .page-wrapper .page-inner section.normal>:last-child{margin-bottom:0!important}.book .book-body .page-wrapper .page-inner section.normal blockquote,.book .book-body .page-wrapper .page-inner section.normal code,.book .book-body .page-wrapper .page-inner section.normal figure,.book .book-body .page-wrapper .page-inner section.normal img,.book .book-body .page-wrapper .page-inner section.normal pre,.book .book-body .page-wrapper .page-inner section.normal table,.book .book-body .page-wrapper .page-inner section.normal tr{page-break-inside:avoid}.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5,.book .book-body .page-wrapper .page-inner section.normal p{orphans:3;widows:3}.book .book-body .page-wrapper .page-inner section.normal h1,.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5{page-break-after:avoid}.book .book-body .page-wrapper .page-inner section.normal b,.book .book-body .page-wrapper .page-inner section.normal strong{font-weight:700}.book .book-body .page-wrapper .page-inner section.normal em{font-style:italic}.book .book-body .page-wrapper .page-inner section.normal blockquote,.book .book-body .page-wrapper .page-inner section.normal dl,.book .book-body .page-wrapper .page-inner section.normal ol,.book .book-body .page-wrapper .page-inner section.normal p,.book .book-body .page-wrapper .page-inner section.normal table,.book .book-body .page-wrapper .page-inner section.normal ul{margin-top:0;margin-bottom:.85em}.book .book-body .page-wrapper .page-inner section.normal a{color:#4183c4;text-decoration:none;background:0 0}.book .book-body .page-wrapper .page-inner section.normal a:active,.book .book-body .page-wrapper .page-inner section.normal a:focus,.book .book-body .page-wrapper .page-inner section.normal a:hover{outline:0;text-decoration:underline}.book .book-body .page-wrapper .page-inner section.normal img{border:0;max-width:100%}.book .book-body .page-wrapper .page-inner section.normal hr{height:4px;padding:0;margin:1.7em 0;overflow:hidden;background-color:#e7e7e7;border:none}.book .book-body .page-wrapper .page-inner section.normal hr:after,.book .book-body .page-wrapper .page-inner section.normal hr:before{display:table;content:" "}.book .book-body .page-wrapper .page-inner section.normal h1,.book .book-body .page-wrapper .page-inner section.normal h2,.book .book-body .page-wrapper .page-inner section.normal h3,.book .book-body .page-wrapper .page-inner section.normal h4,.book .book-body .page-wrapper .page-inner section.normal h5,.book .book-body .page-wrapper .page-inner section.normal h6{margin-top:1.275em;margin-bottom:.85em;}.book .book-body .page-wrapper .page-inner section.normal h1{font-size:2em}.book .book-body .page-wrapper .page-inner section.normal h2{font-size:1.75em}.book .book-body .page-wrapper .page-inner section.normal h3{font-size:1.5em}.book .book-body .page-wrapper .page-inner section.normal h4{font-size:1.25em}.book .book-body .page-wrapper .page-inner section.normal h5{font-size:1em}.book .book-body .page-wrapper .page-inner section.normal h6{font-size:1em;color:#777}.book .book-body .page-wrapper .page-inner section.normal code,.book .book-body .page-wrapper .page-inner section.normal pre{font-family:Consolas,"Liberation Mono",Menlo,Courier,monospace;direction:ltr;border:none;color:inherit}.book .book-body .page-wrapper .page-inner section.normal pre{overflow:auto;word-wrap:normal;margin:0 0 1.275em;padding:.85em 1em;background:#f7f7f7}.book .book-body .page-wrapper .page-inner section.normal pre>code{display:inline;max-width:initial;padding:0;margin:0;overflow:initial;line-height:inherit;font-size:.85em;white-space:pre;background:0 0}.book .book-body .page-wrapper .page-inner section.normal pre>code:after,.book .book-body .page-wrapper .page-inner section.normal pre>code:before{content:normal}.book .book-body .page-wrapper .page-inner section.normal code{padding:.2em;margin:0;font-size:.85em;background-color:#f7f7f7}.book .book-body .page-wrapper .page-inner section.normal code:after,.book .book-body .page-wrapper .page-inner section.normal code:before{letter-spacing:-.2em;content:"\00a0"}.book .book-body .page-wrapper .page-inner section.normal ol,.book .book-body .page-wrapper .page-inner section.normal ul{padding:0 0 0 2em;margin:0 0 .85em}.book .book-body .page-wrapper .page-inner section.normal ol ol,.book .book-body .page-wrapper .page-inner section.normal ol ul,.book .book-body .page-wrapper .page-inner section.normal ul ol,.book .book-body .page-wrapper .page-inner section.normal ul ul{margin-top:0;margin-bottom:0}.book .book-body .page-wrapper .page-inner section.normal ol ol{list-style-type:lower-roman}.book .book-body .page-wrapper .page-inner section.normal blockquote{margin:0 0 .85em;padding:0 15px;opacity:0.75;border-left:4px solid #dcdcdc}.book .book-body .page-wrapper .page-inner section.normal blockquote:first-child{margin-top:0}.book .book-body .page-wrapper .page-inner section.normal blockquote:last-child{margin-bottom:0}.book .book-body .page-wrapper .page-inner section.normal dl{padding:0}.book .book-body .page-wrapper .page-inner section.normal dl dt{padding:0;margin-top:.85em;font-style:italic;font-weight:700}.book .book-body .page-wrapper .page-inner section.normal dl dd{padding:0 .85em;margin-bottom:.85em}.book .book-body .page-wrapper .page-inner section.normal dd{margin-left:0}.book .book-body .page-wrapper .page-inner section.normal .glossary-term{cursor:help;text-decoration:underline}.book .book-body .navigation{position:absolute;top:50px;bottom:0;margin:0;max-width:150px;min-width:90px;display:flex;justify-content:center;align-content:center;flex-direction:column;font-size:40px;color:#ccc;text-align:center;-webkit-transition:all 350ms ease;-moz-transition:all 350ms ease;-o-transition:all 350ms ease;transition:all 350ms ease}.book .book-body .navigation:hover{text-decoration:none;color:#444}.book .book-body .navigation.navigation-next{right:0}.book .book-body .navigation.navigation-prev{left:0}@media (max-width:1240px){.book .book-body .navigation{position:static;top:auto;max-width:50%;width:50%;display:inline-block;float:left}.book .book-body .navigation.navigation-unique{max-width:100%;width:100%}}.book .book-body .page-wrapper .page-inner section.glossary{margin-bottom:40px}.book .book-body .page-wrapper .page-inner section.glossary h2 a,.book .book-body .page-wrapper .page-inner section.glossary h2 a:hover{color:inherit;text-decoration:none}.book .book-body .page-wrapper .page-inner section.glossary .glossary-index{list-style:none;margin:0;padding:0}.book .book-body .page-wrapper .page-inner section.glossary .glossary-index li{display:inline;margin:0 8px;white-space:nowrap}*{-webkit-box-sizing:border-box;-moz-box-sizing:border-box;box-sizing:border-box;-webkit-overflow-scrolling:touch;-webkit-tap-highlight-color:transparent;-webkit-text-size-adjust:none;-webkit-touch-callout:none}a{text-decoration:none}body,html{height:100%}html{font-size:62.5%}body{text-rendering:optimizeLegibility;font-smoothing:antialiased;font-family:"Helvetica Neue",Helvetica,Arial,sans-serif;font-size:14px;letter-spacing:.2px;text-size-adjust:100%} .book .book-summary ul.summary li a span {display:inline;padding:initial;overflow:visible;cursor:auto;opacity:1;} +/* show arrow before summary tag as in bootstrap */ +details > summary {display:list-item;cursor:pointer;} +/*add whatsapp icon from FA 5.1.1 +TODO: remove when updating fontawesome*/ +.fa-whatsapp:before{content:"\f232"} diff --git a/docs/manual/libs/gitbook-2.6.7/js/plugin-search.js b/docs/manual/libs/gitbook-2.6.7/js/plugin-search.js index 31b57867..747fcceb 100644 --- a/docs/manual/libs/gitbook-2.6.7/js/plugin-search.js +++ b/docs/manual/libs/gitbook-2.6.7/js/plugin-search.js @@ -1,9 +1,24 @@ gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { var index = null; + var fuse = null; + var _search = {engine: 'lunr', opts: {}}; var $searchInput, $searchLabel, $searchForm; var $highlighted = [], hi, hiOpts = { className: 'search-highlight' }; var collapse = false, toc_visible = []; + function init(config) { + // Instantiate search settings + _search = gitbook.storage.get("search", { + engine: config.search.engine || 'lunr', + opts: config.search.options || {}, + }); + }; + + // Save current search settings + function saveSearchSettings() { + gitbook.storage.set("search", _search); + } + // Use a specific index function loadIndex(data) { // [Yihui] In bookdown, I use a character matrix to store the chapter @@ -14,18 +29,36 @@ gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { // lunr cannot handle non-English text very well, e.g. the default // tokenizer cannot deal with Chinese text, so we may want to replace // lunr with a dumb simple text matching approach. - index = lunr(function () { - this.ref('url'); - this.field('title', { boost: 10 }); - this.field('body'); - }); - data.map(function(item) { - index.add({ - url: item[0], - title: item[1], - body: item[2] + if (_search.engine === 'lunr') { + index = lunr(function () { + this.ref('url'); + this.field('title', { boost: 10 }); + this.field('body'); }); - }); + data.map(function(item) { + index.add({ + url: item[0], + title: item[1], + body: item[2] + }); + }); + return; + } + fuse = new Fuse(data.map((_data => { + return { + url: _data[0], + title: _data[1], + body: _data[2] + }; + })), Object.assign( + { + includeScore: true, + threshold: 0.1, + ignoreLocation: true, + keys: ["title", "body"] + }, + _search.opts + )); } // Fetch the search index @@ -36,20 +69,33 @@ gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { // Search for a term and return results function search(q) { - if (!index) return; - - var results = _.chain(index.search(q)) - .map(function(result) { - var parts = result.ref.split("#"); - return { - path: parts[0], - hash: parts[1] - }; - }) - .value(); + let results = []; + switch (_search.engine) { + case 'fuse': + if (!fuse) return; + results = fuse.search(q).map(function(result) { + var parts = result.item.url.split('#'); + return { + path: parts[0], + hash: parts[1] + }; + }); + break; + case 'lunr': + default: + if (!index) return; + results = _.chain(index.search(q)).map(function(result) { + var parts = result.ref.split("#"); + return { + path: parts[0], + hash: parts[1] + }; + }) + .value(); + } // [Yihui] Highlight the search keyword on current page - $highlighted = results.length === 0 ? [] : $('.page-inner') + $highlighted = $('.page-inner') .unhighlight(hiOpts).highlight(q, hiOpts).find('span.search-highlight'); scrollToHighlighted(0); @@ -172,6 +218,7 @@ gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { gitbook.events.bind("start", function(e, config) { // [Yihui] disable search if (config.search === false) return; + init(config); collapse = !config.toc || config.toc.collapse === 'section' || config.toc.collapse === 'subsection'; diff --git a/docs/manual/libs/gitbook-2.6.7/js/plugin-sharing.js b/docs/manual/libs/gitbook-2.6.7/js/plugin-sharing.js index 1994e6c7..9a01b0f8 100644 --- a/docs/manual/libs/gitbook-2.6.7/js/plugin-sharing.js +++ b/docs/manual/libs/gitbook-2.6.7/js/plugin-sharing.js @@ -57,10 +57,21 @@ gitbook.require(["gitbook", "lodash", "jQuery"], function(gitbook, _, $) { e.preventDefault(); window.open("http://vkontakte.ru/share.php?url="+encodeURIComponent(location.href)); } - } + }, + 'whatsapp': { + 'label': 'Whatsapp', + 'icon': 'fa fa-whatsapp', + 'onClick': function(e) { + e.preventDefault(); + var url = encodeURIComponent(location.href); + window.open((isMobile() ? "whatsapp://send" : "https://web.whatsapp.com/send") + "?text=" + url); + } + }, }; - + function isMobile() { + return !!navigator.maxTouchPoints; + } gitbook.events.bind("start", function(e, config) { var opts = config.sharing; diff --git a/docs/manual/libs/header-attrs-2.11/header-attrs.js b/docs/manual/libs/header-attrs-2.11/header-attrs.js new file mode 100644 index 00000000..dd57d92e --- /dev/null +++ b/docs/manual/libs/header-attrs-2.11/header-attrs.js @@ -0,0 +1,12 @@ +// Pandoc 2.9 adds attributes on both header and div. We remove the former (to +// be compatible with the behavior of Pandoc < 2.8). +document.addEventListener('DOMContentLoaded', function(e) { + var hs = document.querySelectorAll("div.section[class*='level'] > :first-child"); + var i, h, a; + for (i = 0; i < hs.length; i++) { + h = hs[i]; + if (!/^h[1-6]$/i.test(h.tagName)) continue; // it should be a header h1-h6 + a = h.attributes; + while (a.length > 0) h.removeAttribute(a[0].name); + } +}); diff --git a/docs/manual/libs/jquery-3.6.0/jquery-3.6.0.min.js b/docs/manual/libs/jquery-3.6.0/jquery-3.6.0.min.js new file mode 100644 index 00000000..c4c6022f --- /dev/null +++ b/docs/manual/libs/jquery-3.6.0/jquery-3.6.0.min.js @@ -0,0 +1,2 @@ +/*! jQuery v3.6.0 | (c) OpenJS Foundation and other contributors | jquery.org/license */ +!function(e,t){"use strict";"object"==typeof module&&"object"==typeof module.exports?module.exports=e.document?t(e,!0):function(e){if(!e.document)throw new Error("jQuery requires a window with a document");return t(e)}:t(e)}("undefined"!=typeof window?window:this,function(C,e){"use strict";var t=[],r=Object.getPrototypeOf,s=t.slice,g=t.flat?function(e){return t.flat.call(e)}:function(e){return t.concat.apply([],e)},u=t.push,i=t.indexOf,n={},o=n.toString,v=n.hasOwnProperty,a=v.toString,l=a.call(Object),y={},m=function(e){return"function"==typeof e&&"number"!=typeof e.nodeType&&"function"!=typeof e.item},x=function(e){return null!=e&&e===e.window},E=C.document,c={type:!0,src:!0,nonce:!0,noModule:!0};function b(e,t,n){var r,i,o=(n=n||E).createElement("script");if(o.text=e,t)for(r in c)(i=t[r]||t.getAttribute&&t.getAttribute(r))&&o.setAttribute(r,i);n.head.appendChild(o).parentNode.removeChild(o)}function w(e){return null==e?e+"":"object"==typeof e||"function"==typeof e?n[o.call(e)]||"object":typeof e}var f="3.6.0",S=function(e,t){return new S.fn.init(e,t)};function p(e){var t=!!e&&"length"in e&&e.length,n=w(e);return!m(e)&&!x(e)&&("array"===n||0===t||"number"==typeof t&&0+~]|"+M+")"+M+"*"),U=new RegExp(M+"|>"),X=new RegExp(F),V=new RegExp("^"+I+"$"),G={ID:new RegExp("^#("+I+")"),CLASS:new RegExp("^\\.("+I+")"),TAG:new RegExp("^("+I+"|[*])"),ATTR:new RegExp("^"+W),PSEUDO:new RegExp("^"+F),CHILD:new RegExp("^:(only|first|last|nth|nth-last)-(child|of-type)(?:\\("+M+"*(even|odd|(([+-]|)(\\d*)n|)"+M+"*(?:([+-]|)"+M+"*(\\d+)|))"+M+"*\\)|)","i"),bool:new RegExp("^(?:"+R+")$","i"),needsContext:new RegExp("^"+M+"*[>+~]|:(even|odd|eq|gt|lt|nth|first|last)(?:\\("+M+"*((?:-\\d)?\\d*)"+M+"*\\)|)(?=[^-]|$)","i")},Y=/HTML$/i,Q=/^(?:input|select|textarea|button)$/i,J=/^h\d$/i,K=/^[^{]+\{\s*\[native \w/,Z=/^(?:#([\w-]+)|(\w+)|\.([\w-]+))$/,ee=/[+~]/,te=new RegExp("\\\\[\\da-fA-F]{1,6}"+M+"?|\\\\([^\\r\\n\\f])","g"),ne=function(e,t){var n="0x"+e.slice(1)-65536;return t||(n<0?String.fromCharCode(n+65536):String.fromCharCode(n>>10|55296,1023&n|56320))},re=/([\0-\x1f\x7f]|^-?\d)|^-$|[^\0-\x1f\x7f-\uFFFF\w-]/g,ie=function(e,t){return t?"\0"===e?"\ufffd":e.slice(0,-1)+"\\"+e.charCodeAt(e.length-1).toString(16)+" ":"\\"+e},oe=function(){T()},ae=be(function(e){return!0===e.disabled&&"fieldset"===e.nodeName.toLowerCase()},{dir:"parentNode",next:"legend"});try{H.apply(t=O.call(p.childNodes),p.childNodes),t[p.childNodes.length].nodeType}catch(e){H={apply:t.length?function(e,t){L.apply(e,O.call(t))}:function(e,t){var n=e.length,r=0;while(e[n++]=t[r++]);e.length=n-1}}}function se(t,e,n,r){var i,o,a,s,u,l,c,f=e&&e.ownerDocument,p=e?e.nodeType:9;if(n=n||[],"string"!=typeof t||!t||1!==p&&9!==p&&11!==p)return n;if(!r&&(T(e),e=e||C,E)){if(11!==p&&(u=Z.exec(t)))if(i=u[1]){if(9===p){if(!(a=e.getElementById(i)))return n;if(a.id===i)return n.push(a),n}else if(f&&(a=f.getElementById(i))&&y(e,a)&&a.id===i)return n.push(a),n}else{if(u[2])return H.apply(n,e.getElementsByTagName(t)),n;if((i=u[3])&&d.getElementsByClassName&&e.getElementsByClassName)return H.apply(n,e.getElementsByClassName(i)),n}if(d.qsa&&!N[t+" "]&&(!v||!v.test(t))&&(1!==p||"object"!==e.nodeName.toLowerCase())){if(c=t,f=e,1===p&&(U.test(t)||z.test(t))){(f=ee.test(t)&&ye(e.parentNode)||e)===e&&d.scope||((s=e.getAttribute("id"))?s=s.replace(re,ie):e.setAttribute("id",s=S)),o=(l=h(t)).length;while(o--)l[o]=(s?"#"+s:":scope")+" "+xe(l[o]);c=l.join(",")}try{return H.apply(n,f.querySelectorAll(c)),n}catch(e){N(t,!0)}finally{s===S&&e.removeAttribute("id")}}}return g(t.replace($,"$1"),e,n,r)}function ue(){var r=[];return function e(t,n){return r.push(t+" ")>b.cacheLength&&delete e[r.shift()],e[t+" "]=n}}function le(e){return e[S]=!0,e}function ce(e){var t=C.createElement("fieldset");try{return!!e(t)}catch(e){return!1}finally{t.parentNode&&t.parentNode.removeChild(t),t=null}}function fe(e,t){var n=e.split("|"),r=n.length;while(r--)b.attrHandle[n[r]]=t}function pe(e,t){var n=t&&e,r=n&&1===e.nodeType&&1===t.nodeType&&e.sourceIndex-t.sourceIndex;if(r)return r;if(n)while(n=n.nextSibling)if(n===t)return-1;return e?1:-1}function de(t){return function(e){return"input"===e.nodeName.toLowerCase()&&e.type===t}}function he(n){return function(e){var t=e.nodeName.toLowerCase();return("input"===t||"button"===t)&&e.type===n}}function ge(t){return function(e){return"form"in e?e.parentNode&&!1===e.disabled?"label"in e?"label"in e.parentNode?e.parentNode.disabled===t:e.disabled===t:e.isDisabled===t||e.isDisabled!==!t&&ae(e)===t:e.disabled===t:"label"in e&&e.disabled===t}}function ve(a){return le(function(o){return o=+o,le(function(e,t){var n,r=a([],e.length,o),i=r.length;while(i--)e[n=r[i]]&&(e[n]=!(t[n]=e[n]))})})}function ye(e){return e&&"undefined"!=typeof e.getElementsByTagName&&e}for(e in d=se.support={},i=se.isXML=function(e){var t=e&&e.namespaceURI,n=e&&(e.ownerDocument||e).documentElement;return!Y.test(t||n&&n.nodeName||"HTML")},T=se.setDocument=function(e){var t,n,r=e?e.ownerDocument||e:p;return r!=C&&9===r.nodeType&&r.documentElement&&(a=(C=r).documentElement,E=!i(C),p!=C&&(n=C.defaultView)&&n.top!==n&&(n.addEventListener?n.addEventListener("unload",oe,!1):n.attachEvent&&n.attachEvent("onunload",oe)),d.scope=ce(function(e){return a.appendChild(e).appendChild(C.createElement("div")),"undefined"!=typeof e.querySelectorAll&&!e.querySelectorAll(":scope fieldset div").length}),d.attributes=ce(function(e){return e.className="i",!e.getAttribute("className")}),d.getElementsByTagName=ce(function(e){return e.appendChild(C.createComment("")),!e.getElementsByTagName("*").length}),d.getElementsByClassName=K.test(C.getElementsByClassName),d.getById=ce(function(e){return a.appendChild(e).id=S,!C.getElementsByName||!C.getElementsByName(S).length}),d.getById?(b.filter.ID=function(e){var t=e.replace(te,ne);return function(e){return e.getAttribute("id")===t}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n=t.getElementById(e);return n?[n]:[]}}):(b.filter.ID=function(e){var n=e.replace(te,ne);return function(e){var t="undefined"!=typeof e.getAttributeNode&&e.getAttributeNode("id");return t&&t.value===n}},b.find.ID=function(e,t){if("undefined"!=typeof t.getElementById&&E){var n,r,i,o=t.getElementById(e);if(o){if((n=o.getAttributeNode("id"))&&n.value===e)return[o];i=t.getElementsByName(e),r=0;while(o=i[r++])if((n=o.getAttributeNode("id"))&&n.value===e)return[o]}return[]}}),b.find.TAG=d.getElementsByTagName?function(e,t){return"undefined"!=typeof t.getElementsByTagName?t.getElementsByTagName(e):d.qsa?t.querySelectorAll(e):void 0}:function(e,t){var n,r=[],i=0,o=t.getElementsByTagName(e);if("*"===e){while(n=o[i++])1===n.nodeType&&r.push(n);return r}return o},b.find.CLASS=d.getElementsByClassName&&function(e,t){if("undefined"!=typeof t.getElementsByClassName&&E)return t.getElementsByClassName(e)},s=[],v=[],(d.qsa=K.test(C.querySelectorAll))&&(ce(function(e){var t;a.appendChild(e).innerHTML="",e.querySelectorAll("[msallowcapture^='']").length&&v.push("[*^$]="+M+"*(?:''|\"\")"),e.querySelectorAll("[selected]").length||v.push("\\["+M+"*(?:value|"+R+")"),e.querySelectorAll("[id~="+S+"-]").length||v.push("~="),(t=C.createElement("input")).setAttribute("name",""),e.appendChild(t),e.querySelectorAll("[name='']").length||v.push("\\["+M+"*name"+M+"*="+M+"*(?:''|\"\")"),e.querySelectorAll(":checked").length||v.push(":checked"),e.querySelectorAll("a#"+S+"+*").length||v.push(".#.+[+~]"),e.querySelectorAll("\\\f"),v.push("[\\r\\n\\f]")}),ce(function(e){e.innerHTML="";var t=C.createElement("input");t.setAttribute("type","hidden"),e.appendChild(t).setAttribute("name","D"),e.querySelectorAll("[name=d]").length&&v.push("name"+M+"*[*^$|!~]?="),2!==e.querySelectorAll(":enabled").length&&v.push(":enabled",":disabled"),a.appendChild(e).disabled=!0,2!==e.querySelectorAll(":disabled").length&&v.push(":enabled",":disabled"),e.querySelectorAll("*,:x"),v.push(",.*:")})),(d.matchesSelector=K.test(c=a.matches||a.webkitMatchesSelector||a.mozMatchesSelector||a.oMatchesSelector||a.msMatchesSelector))&&ce(function(e){d.disconnectedMatch=c.call(e,"*"),c.call(e,"[s!='']:x"),s.push("!=",F)}),v=v.length&&new RegExp(v.join("|")),s=s.length&&new RegExp(s.join("|")),t=K.test(a.compareDocumentPosition),y=t||K.test(a.contains)?function(e,t){var n=9===e.nodeType?e.documentElement:e,r=t&&t.parentNode;return e===r||!(!r||1!==r.nodeType||!(n.contains?n.contains(r):e.compareDocumentPosition&&16&e.compareDocumentPosition(r)))}:function(e,t){if(t)while(t=t.parentNode)if(t===e)return!0;return!1},j=t?function(e,t){if(e===t)return l=!0,0;var n=!e.compareDocumentPosition-!t.compareDocumentPosition;return n||(1&(n=(e.ownerDocument||e)==(t.ownerDocument||t)?e.compareDocumentPosition(t):1)||!d.sortDetached&&t.compareDocumentPosition(e)===n?e==C||e.ownerDocument==p&&y(p,e)?-1:t==C||t.ownerDocument==p&&y(p,t)?1:u?P(u,e)-P(u,t):0:4&n?-1:1)}:function(e,t){if(e===t)return l=!0,0;var n,r=0,i=e.parentNode,o=t.parentNode,a=[e],s=[t];if(!i||!o)return e==C?-1:t==C?1:i?-1:o?1:u?P(u,e)-P(u,t):0;if(i===o)return pe(e,t);n=e;while(n=n.parentNode)a.unshift(n);n=t;while(n=n.parentNode)s.unshift(n);while(a[r]===s[r])r++;return r?pe(a[r],s[r]):a[r]==p?-1:s[r]==p?1:0}),C},se.matches=function(e,t){return se(e,null,null,t)},se.matchesSelector=function(e,t){if(T(e),d.matchesSelector&&E&&!N[t+" "]&&(!s||!s.test(t))&&(!v||!v.test(t)))try{var n=c.call(e,t);if(n||d.disconnectedMatch||e.document&&11!==e.document.nodeType)return n}catch(e){N(t,!0)}return 0":{dir:"parentNode",first:!0}," ":{dir:"parentNode"},"+":{dir:"previousSibling",first:!0},"~":{dir:"previousSibling"}},preFilter:{ATTR:function(e){return e[1]=e[1].replace(te,ne),e[3]=(e[3]||e[4]||e[5]||"").replace(te,ne),"~="===e[2]&&(e[3]=" "+e[3]+" "),e.slice(0,4)},CHILD:function(e){return e[1]=e[1].toLowerCase(),"nth"===e[1].slice(0,3)?(e[3]||se.error(e[0]),e[4]=+(e[4]?e[5]+(e[6]||1):2*("even"===e[3]||"odd"===e[3])),e[5]=+(e[7]+e[8]||"odd"===e[3])):e[3]&&se.error(e[0]),e},PSEUDO:function(e){var t,n=!e[6]&&e[2];return G.CHILD.test(e[0])?null:(e[3]?e[2]=e[4]||e[5]||"":n&&X.test(n)&&(t=h(n,!0))&&(t=n.indexOf(")",n.length-t)-n.length)&&(e[0]=e[0].slice(0,t),e[2]=n.slice(0,t)),e.slice(0,3))}},filter:{TAG:function(e){var t=e.replace(te,ne).toLowerCase();return"*"===e?function(){return!0}:function(e){return e.nodeName&&e.nodeName.toLowerCase()===t}},CLASS:function(e){var t=m[e+" "];return t||(t=new RegExp("(^|"+M+")"+e+"("+M+"|$)"))&&m(e,function(e){return t.test("string"==typeof e.className&&e.className||"undefined"!=typeof e.getAttribute&&e.getAttribute("class")||"")})},ATTR:function(n,r,i){return function(e){var t=se.attr(e,n);return null==t?"!="===r:!r||(t+="","="===r?t===i:"!="===r?t!==i:"^="===r?i&&0===t.indexOf(i):"*="===r?i&&-1:\x20\t\r\n\f]*)[\x20\t\r\n\f]*\/?>(?:<\/\1>|)$/i;function j(e,n,r){return m(n)?S.grep(e,function(e,t){return!!n.call(e,t,e)!==r}):n.nodeType?S.grep(e,function(e){return e===n!==r}):"string"!=typeof n?S.grep(e,function(e){return-1)[^>]*|#([\w-]+))$/;(S.fn.init=function(e,t,n){var r,i;if(!e)return this;if(n=n||D,"string"==typeof e){if(!(r="<"===e[0]&&">"===e[e.length-1]&&3<=e.length?[null,e,null]:q.exec(e))||!r[1]&&t)return!t||t.jquery?(t||n).find(e):this.constructor(t).find(e);if(r[1]){if(t=t instanceof S?t[0]:t,S.merge(this,S.parseHTML(r[1],t&&t.nodeType?t.ownerDocument||t:E,!0)),N.test(r[1])&&S.isPlainObject(t))for(r in t)m(this[r])?this[r](t[r]):this.attr(r,t[r]);return this}return(i=E.getElementById(r[2]))&&(this[0]=i,this.length=1),this}return e.nodeType?(this[0]=e,this.length=1,this):m(e)?void 0!==n.ready?n.ready(e):e(S):S.makeArray(e,this)}).prototype=S.fn,D=S(E);var L=/^(?:parents|prev(?:Until|All))/,H={children:!0,contents:!0,next:!0,prev:!0};function O(e,t){while((e=e[t])&&1!==e.nodeType);return e}S.fn.extend({has:function(e){var t=S(e,this),n=t.length;return this.filter(function(){for(var e=0;e\x20\t\r\n\f]*)/i,he=/^$|^module$|\/(?:java|ecma)script/i;ce=E.createDocumentFragment().appendChild(E.createElement("div")),(fe=E.createElement("input")).setAttribute("type","radio"),fe.setAttribute("checked","checked"),fe.setAttribute("name","t"),ce.appendChild(fe),y.checkClone=ce.cloneNode(!0).cloneNode(!0).lastChild.checked,ce.innerHTML="",y.noCloneChecked=!!ce.cloneNode(!0).lastChild.defaultValue,ce.innerHTML="",y.option=!!ce.lastChild;var ge={thead:[1,"","
"],col:[2,"","
"],tr:[2,"","
"],td:[3,"","
"],_default:[0,"",""]};function ve(e,t){var n;return n="undefined"!=typeof e.getElementsByTagName?e.getElementsByTagName(t||"*"):"undefined"!=typeof e.querySelectorAll?e.querySelectorAll(t||"*"):[],void 0===t||t&&A(e,t)?S.merge([e],n):n}function ye(e,t){for(var n=0,r=e.length;n",""]);var me=/<|&#?\w+;/;function xe(e,t,n,r,i){for(var o,a,s,u,l,c,f=t.createDocumentFragment(),p=[],d=0,h=e.length;d\s*$/g;function je(e,t){return A(e,"table")&&A(11!==t.nodeType?t:t.firstChild,"tr")&&S(e).children("tbody")[0]||e}function De(e){return e.type=(null!==e.getAttribute("type"))+"/"+e.type,e}function qe(e){return"true/"===(e.type||"").slice(0,5)?e.type=e.type.slice(5):e.removeAttribute("type"),e}function Le(e,t){var n,r,i,o,a,s;if(1===t.nodeType){if(Y.hasData(e)&&(s=Y.get(e).events))for(i in Y.remove(t,"handle events"),s)for(n=0,r=s[i].length;n").attr(n.scriptAttrs||{}).prop({charset:n.scriptCharset,src:n.url}).on("load error",i=function(e){r.remove(),i=null,e&&t("error"===e.type?404:200,e.type)}),E.head.appendChild(r[0])},abort:function(){i&&i()}}});var _t,zt=[],Ut=/(=)\?(?=&|$)|\?\?/;S.ajaxSetup({jsonp:"callback",jsonpCallback:function(){var e=zt.pop()||S.expando+"_"+wt.guid++;return this[e]=!0,e}}),S.ajaxPrefilter("json jsonp",function(e,t,n){var r,i,o,a=!1!==e.jsonp&&(Ut.test(e.url)?"url":"string"==typeof e.data&&0===(e.contentType||"").indexOf("application/x-www-form-urlencoded")&&Ut.test(e.data)&&"data");if(a||"jsonp"===e.dataTypes[0])return r=e.jsonpCallback=m(e.jsonpCallback)?e.jsonpCallback():e.jsonpCallback,a?e[a]=e[a].replace(Ut,"$1"+r):!1!==e.jsonp&&(e.url+=(Tt.test(e.url)?"&":"?")+e.jsonp+"="+r),e.converters["script json"]=function(){return o||S.error(r+" was not called"),o[0]},e.dataTypes[0]="json",i=C[r],C[r]=function(){o=arguments},n.always(function(){void 0===i?S(C).removeProp(r):C[r]=i,e[r]&&(e.jsonpCallback=t.jsonpCallback,zt.push(r)),o&&m(i)&&i(o[0]),o=i=void 0}),"script"}),y.createHTMLDocument=((_t=E.implementation.createHTMLDocument("").body).innerHTML="
",2===_t.childNodes.length),S.parseHTML=function(e,t,n){return"string"!=typeof e?[]:("boolean"==typeof t&&(n=t,t=!1),t||(y.createHTMLDocument?((r=(t=E.implementation.createHTMLDocument("")).createElement("base")).href=E.location.href,t.head.appendChild(r)):t=E),o=!n&&[],(i=N.exec(e))?[t.createElement(i[1])]:(i=xe([e],t,o),o&&o.length&&S(o).remove(),S.merge([],i.childNodes)));var r,i,o},S.fn.load=function(e,t,n){var r,i,o,a=this,s=e.indexOf(" ");return-1").append(S.parseHTML(e)).find(r):e)}).always(n&&function(e,t){a.each(function(){n.apply(this,o||[e.responseText,t,e])})}),this},S.expr.pseudos.animated=function(t){return S.grep(S.timers,function(e){return t===e.elem}).length},S.offset={setOffset:function(e,t,n){var r,i,o,a,s,u,l=S.css(e,"position"),c=S(e),f={};"static"===l&&(e.style.position="relative"),s=c.offset(),o=S.css(e,"top"),u=S.css(e,"left"),("absolute"===l||"fixed"===l)&&-1<(o+u).indexOf("auto")?(a=(r=c.position()).top,i=r.left):(a=parseFloat(o)||0,i=parseFloat(u)||0),m(t)&&(t=t.call(e,n,S.extend({},s))),null!=t.top&&(f.top=t.top-s.top+a),null!=t.left&&(f.left=t.left-s.left+i),"using"in t?t.using.call(e,f):c.css(f)}},S.fn.extend({offset:function(t){if(arguments.length)return void 0===t?this:this.each(function(e){S.offset.setOffset(this,t,e)});var e,n,r=this[0];return r?r.getClientRects().length?(e=r.getBoundingClientRect(),n=r.ownerDocument.defaultView,{top:e.top+n.pageYOffset,left:e.left+n.pageXOffset}):{top:0,left:0}:void 0},position:function(){if(this[0]){var e,t,n,r=this[0],i={top:0,left:0};if("fixed"===S.css(r,"position"))t=r.getBoundingClientRect();else{t=this.offset(),n=r.ownerDocument,e=r.offsetParent||n.documentElement;while(e&&(e===n.body||e===n.documentElement)&&"static"===S.css(e,"position"))e=e.parentNode;e&&e!==r&&1===e.nodeType&&((i=S(e).offset()).top+=S.css(e,"borderTopWidth",!0),i.left+=S.css(e,"borderLeftWidth",!0))}return{top:t.top-i.top-S.css(r,"marginTop",!0),left:t.left-i.left-S.css(r,"marginLeft",!0)}}},offsetParent:function(){return this.map(function(){var e=this.offsetParent;while(e&&"static"===S.css(e,"position"))e=e.offsetParent;return e||re})}}),S.each({scrollLeft:"pageXOffset",scrollTop:"pageYOffset"},function(t,i){var o="pageYOffset"===i;S.fn[t]=function(e){return $(this,function(e,t,n){var r;if(x(e)?r=e:9===e.nodeType&&(r=e.defaultView),void 0===n)return r?r[i]:e[t];r?r.scrollTo(o?r.pageXOffset:n,o?n:r.pageYOffset):e[t]=n},t,e,arguments.length)}}),S.each(["top","left"],function(e,n){S.cssHooks[n]=Fe(y.pixelPosition,function(e,t){if(t)return t=We(e,n),Pe.test(t)?S(e).position()[n]+"px":t})}),S.each({Height:"height",Width:"width"},function(a,s){S.each({padding:"inner"+a,content:s,"":"outer"+a},function(r,o){S.fn[o]=function(e,t){var n=arguments.length&&(r||"boolean"!=typeof e),i=r||(!0===e||!0===t?"margin":"border");return $(this,function(e,t,n){var r;return x(e)?0===o.indexOf("outer")?e["inner"+a]:e.document.documentElement["client"+a]:9===e.nodeType?(r=e.documentElement,Math.max(e.body["scroll"+a],r["scroll"+a],e.body["offset"+a],r["offset"+a],r["client"+a])):void 0===n?S.css(e,t,i):S.style(e,t,n,i)},s,n?e:void 0,n)}})}),S.each(["ajaxStart","ajaxStop","ajaxComplete","ajaxError","ajaxSuccess","ajaxSend"],function(e,t){S.fn[t]=function(e){return this.on(t,e)}}),S.fn.extend({bind:function(e,t,n){return this.on(e,null,t,n)},unbind:function(e,t){return this.off(e,null,t)},delegate:function(e,t,n,r){return this.on(t,e,n,r)},undelegate:function(e,t,n){return 1===arguments.length?this.off(e,"**"):this.off(t,e||"**",n)},hover:function(e,t){return this.mouseenter(e).mouseleave(t||e)}}),S.each("blur focus focusin focusout resize scroll click dblclick mousedown mouseup mousemove mouseover mouseout mouseenter mouseleave change select submit keydown keypress keyup contextmenu".split(" "),function(e,n){S.fn[n]=function(e,t){return 0 6 Output and Plots | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,14 +223,14 @@

-
+

6 Output and Plots

-
+

6.1 Summarizing output

Output is summarized using SSMSE_summary_all():

- +
summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir",
+                  scenarios = c("sample_low", "sample_high"),
+                  run_parallel = TRUE)

Relying on ss3sim::get_results_all(), this function creates:

  • For each scenario, 3 scenario level .csv files
    • @@ -222,11 +240,11 @@

    6.1 Summarizing output

    Note that run_parallel = TRUE is only faster than run_parallel FALSE when there is more than once scenario and none of the scenario-level .csv files have been created yet.

    By default, if a user doesn’t specify scenarios, all scenarios in dir will be summarized.

-
+

6.2 Checking estimation model convergence

One of the first checks to do if using an estimation model after running an MSE analysis is to check that the estimation model has converged. A number of checks could be done, but a basic one is checking the gradients for the estimation model, which are added to the SSMSE_scalar summary sheet.

-
+

6.3 Calculating performance metrics

Typically, a suite of performance metrics are used in MSE. Punt et al. (2016) recommends at least using metrics related to:

    @@ -246,7 +264,6 @@

    6.3 Calculating performance metri

- @@ -265,6 +282,7 @@

6.3 Calculating performance metri "weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -285,6 +303,10 @@

6.3 Calculating performance metri "text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/plotting-output-and-performance-metrics.html b/docs/manual/plotting-output-and-performance-metrics.html index 0eeed22f..cb7ed1c2 100644 --- a/docs/manual/plotting-output-and-performance-metrics.html +++ b/docs/manual/plotting-output-and-performance-metrics.html @@ -6,7 +6,7 @@ 7 Plotting output and performance metrics | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -32,8 +32,10 @@ - - + + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,7 +223,7 @@

-
+

7 Plotting output and performance metrics

Currently, it has been left up to the user to plot summaries and performance metrics, as the potential options for plots and performance metrics users may desire are extensive.

There are currently diagnostic plots to examine how sampled data compares to the operating model values and data sets used to condition the operating models. plot_index_sampling() creates a diagnostic plot for indices, while plot_comp_sampling() creates diagnostic plots for composition data.

@@ -216,12 +234,11 @@

7 Plotting output and performance

- - + +

- @@ -240,6 +257,7 @@

7 Plotting output and performance "weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -260,6 +278,10 @@

7 Plotting output and performance "text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/docs/manual/reference-keys.txt b/docs/manual/reference-keys.txt index bc8b1f48..30c8bfb4 100644 --- a/docs/manual/reference-keys.txt +++ b/docs/manual/reference-keys.txt @@ -46,3 +46,4 @@ summarizing-output checking-estimation-model-convergence calculating-performance-metrics plotting-output-and-performance-metrics +glossary-of-mse-terms diff --git a/docs/manual/search_index.json b/docs/manual/search_index.json index f53fda7d..62f56422 100644 --- a/docs/manual/search_index.json +++ b/docs/manual/search_index.json @@ -1 +1 @@ -[["index.html", "SSMSE user manual Preface R session information", " SSMSE user manual Kathryn Doering and Nathan Vaughan 2021-09-20 Preface Logo for the SSMSE package This is the user manual for SSMSE, an R package for Management Strategy Evaluation with Stock Synthesis Operating models. This document is still a work in progress! If you would like to suggest changes, the R markdown files used to generate the user manual are available in a bookdown folder within the SSMSE repository. Please feel free to open an issue or submit a pull request to suggest changes. R session information sessionInfo() R version 4.0.2 Patched (2020-07-15 r78861) Platform: x86_64-w64-mingw32/x64 (64-bit) Running under: Windows 10 x64 (build 18363) Matrix products: default locale: [1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252 [3] LC_MONETARY=English_United States.1252 LC_NUMERIC=C [5] LC_TIME=English_United States.1252 attached base packages: [1] stats graphics grDevices utils datasets methods base loaded via a namespace (and not attached): [1] ss3sim_1.1.6 r4ss_1.43.0 "],["intro.html", "1 Introduction 1.1 Purpose 1.2 Functions in SSMSE 1.3 Brief description of the SSMSE MSE simulation procedure", " 1 Introduction 1.1 Purpose SSMSE allows users to directly use Stock Synthesis (SS) as an operating model (OM) in an Management Strategy Evaluation (MSE). The approach requires a conditioned Stock Synthesis model, which is treated as the OM. 1.2 Functions in SSMSE The main functions users can call in SSMSE are: Function Description run_SSMSE() Run the MSE simulations SSMSE_summary_all() Summarize MSE output The helper functions to create inputs to run_SSMSE are: Helper Function Description create_sample_struct() Helper function to create a list for future sampling from a model to use as input in run_SSMSE() create_future_om_list() Helper function that provides examples of the structure for the future_om_list input to run_SSMSE(). develop_OMs() Helper function to turn one OM into many Exported functions that can be used for writing custom management strategies are: Function Description run_EM() Run an SS estimation model (uses run_ss_model) run_ss_model() Run an SS model get_bin() Get location of the SS binary. parse_MS() Function that runs the management strategy and returns catch by fleet for the projections. A reference function for those setting up custom management strategies. Finally, some plotting functions are available: Plotting Function Description plot_index_sampling() Plot to compare the sampled index values to the operating model expected values and original operating model conditioning index data. plot_comp_sampling() Plot to compare the sampled composition values to the operating model expected values and original operating model conditioning composition data. 1.3 Brief description of the SSMSE MSE simulation procedure 1.3.1 Conditioning the OM and sampling from the OM For each scenario, SSMSE starts with the user providing a fitted Stock Synthesis model (or selecting an model from the SSMSE package) to use as an OM. For each iteration of the scenario, SSMSE turns the SS fitted model into an OM and runs it once with no estimation with Stock Synthesis in order to get the true values and a bootstrapped data set from SS. Note that any modifications to the OMs parameters specified by the users as it is extended forward in time are also applied. 1.3.2 First run of the management strategy in the MSE simulation The bootstrapped dataset is then used in a Management strategy to project catch by fleet to use for the next \\(n\\) years, where \\(n\\) is the number of years between assessments. 1.3.3 Feedback from managment strategy into OM: extending model years The catch for the next \\(n\\) years before the next assessment is then added to the OM, as well as any recruitment or time varying parameter deviations. The OM is again run with no estimation where it can be used to produce sampled data for the next \\(n\\) years. These new data values are appended to the original dataset. 1.3.4 Subsequent runs of the management strategy The appended data set is then used in the managment strategy again, and new catch by fleet is produced that can then be fed back to the OM. "],["simple.html", "2 A simple example 2.1 Setup R workspace folders 2.2 Create the operating models (OMs) 2.3 Adding process error through recruitment deviations and time-varying selectivity 2.4 Examine the management procedure used 2.5 Run SSMSE 2.6 run_SSMSE output 2.7 Performance metrics 2.8 Summarize results 2.9 Simple Convergence Check 2.10 Plot Spawning Stock Biomass (SSB) 2.11 Example MSE Results 2.12 Delete the files", " 2 A simple example Suppose we want to look at how well we are able to achieve a performance metric under uncertainty in the operating model (OM). We will look 2 scenarios, one where Steepness (h) is specified correctly and one where it is specified incorrectly in an estimation model (EM): Scenario 1. h-ctl: Cod operating model (h = 0.65) with correctly specified cod model EM (fixed h = 0.65). The OM is the same as the EM. Scenario 2. h-1: Cod operating model (h = 1) with misspecified cod model EM (fixed h = 0.65); The OM is not the same as the EM. Note that this is a simple example where the OM and EM structures for both scenarios are identical, except for different steepness between the OM and EM in scenario 2 and some process error we will include in the operating model. We will assume we want to run the MSE loop for 6 years, with a stock assessment occuring every 3 years (and forecasting catch to maintain 40% of unfished spawning stock biomass). The cod models last year is 100, so the OM is initially conditioned through year 100. Then, after conditioning the operating model through year 100, assessments will occur in years 100 and 103. The operating model runs through year 106. We chose not to run the assessment in year 106, as there was no need for its output in this example. 2.1 Setup R workspace folders First, we will load the SSMSE package and create a folder in which to run the example: library(SSMSE) #load the package library(r4ss) #install using remotes::install_github("r4ss/r4ss) library(foreach) #if using run_parallel = TRUE library(doParallel) #if using run_parallel = TRUE # Create a folder for the output in the working directory. run_SSMSE_dir <- file.path("run_SSMSE-ex") dir.create(run_SSMSE_dir) 2.2 Create the operating models (OMs) 2.2.1 Specify alternative values for h The cod model with h = 0.65 (as in scenario 1) is included as external package data in SSMSE. However, we will need to modify it to use as an operating model with h = 1 (as in scenario 2). Note in this case that refit_OM is false, so the model is not being refit, just run through without fitting. To condition the new model on the same data as the input model, refit_OM should be TRUE. First, we identify where the base cod model is stored, modify it such that the steepness parameter is 1, and save the modified cod OM for scenario 2 in a new folder in the run_SSMSE_dir directory. cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE") # develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir # specified develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep", par_vals = 1, refit_OMs = FALSE, hess = FALSE) # OM model for scenario 2 cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1") 2.3 Adding process error through recruitment deviations and time-varying selectivity Recruitment deviations, implementation error, and changes in parameters in the projection period of the OM can be added through the future_om_list input to run_SSMSE. First, well set up the list to add recruitment deviations in the projection period. The same recruitment deviation patterns are used across scenarios, but different patterns are use across iterations in the same scenario. We also want these deviations to have the same standard deviations as the historical deviations with 0 mean (the assumed default). # Start from a list created by a helper function template_mod_change <- create_future_om_list() # add recruitment deviations rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" # apply change to rec devs rec_dev_specify$scen <- c("replicate", "all") # using 1 to 100 means the sd or mean will be calculated by taking the sd across years # from 1 to 100 rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" # this change is for the sd # no input value needed since it will be calclated from the historical rec devs. rec_dev_specify$input$value <- NA rec_dev_specify $pars [1] "rec_devs" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param 1 1 100 100 101 sd method value 1 absolute NA Next, suppose we want to allow selectivity to vary annually for 1 selectivity parameter of the fishery throughout the projection period. The following specifies that the value for selectivity varies randomly around the base value with a sd of 0.2. # put together the change for selectivity (random values around the orig val, with # an sd of 0.2) mod_change_sel <- template_mod_change[[1]] mod_change_sel$scen[2] <- "all" # apply to all scenarios # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. # historical values are NA in this case, because they are not used to determine # the sd to use. mod_change_sel$input$last_yr_orig_val <- 100 mod_change_sel$input$first_yr_final_val <- 101 mod_change_sel$input$ts_param <- "sd" # this change is for the sd mod_change_sel$input$value <- 0.2 # se to use in the projection period mod_change_sel $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param 1 NA NA 100 101 sd method value 1 absolute 0.2 Finally, add these two changes together into an object to pass to run_SSMSE future_om_list_recdevs_sel <- list(rec_dev_specify, mod_change_sel) 2.3.1 Add observation error through sampling from OM The argument sample_struct specifies the structure for sampling from the OM (and passing to the EM). The function create_sample_struct can be used to construct a simple sampling structure consistent with an input data file: datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this dataframe. sample_struct_1_scen $catch Yr Seas FltSvy SE 1 101 1 1 0.005 2 102 1 1 0.005 3 103 1 1 0.005 4 104 1 1 0.005 5 105 1 1 0.005 6 106 1 1 0.005 $CPUE Yr Seas FltSvy SE 1 105 7 2 0.2 $lencomp Yr Seas FltSvy Sex Part Nsamp 1 NA 1 1 0 0 125 $agecomp Yr Seas FltSvy Sex Part Ageerr Lbin_lo Lbin_hi Nsamp 1 105 1 2 0 0 1 -1 -1 500 $meanbodywt [1] NA $MeanSize_at_Age_obs [1] NA By default, create_sample_struct identifies sampling patterns from the historical period of the OM and replicates those patterns in the projection period. In our cod example, the sample structure specifies that catch will be added to the estimation model every year (years 101 to 106), but an index of abundance (i.e., CPUE) and age composition (i.e., agecomp) will only be added in year 105. We will use the same sampling scheme for both scenarios, but it is possible to specify different sampling for each scenario. The user could modify this sampling strategy (for example, maybe age composition should also be sampled from FltSvy 2 in Yr 102; the user could add another line to the dataframe in sample_struct$agecomp). Note that length comp (lencomp) includes an NA value for year. This is because no consistent pattern was identified, so the user must define their own input. In this case, we will remove sampling length comps all together: sample_struct_1_scen$lencomp <- NULL # don't use length sampling The same sampling structure will be used for both scenarios, which we define in a list below: sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen) 2.4 Examine the management procedure used We will use the same management procedure for both scenarios: Conduct a stock assessment every 3 years to get stock status. Project from this stock assessment using the SS forecast file to get projected future catch. Put this projected catch (without implementation error, in the case of this example) back into the OM. Extend the OM forward in time to get the true values for the population. Lets take a look at step 2 in the management procedure, which is implemented using the forecasting module in SS. We will examine the forecast file for the estimation model to better understand how catches will be forecasted from the assessment. We will use the same management procedure in both of these scenarios, although for a full MSE analysis, it is likely that multiple management procedures would be compared. fore <- r4ss::SS_readforecast( system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"), verbose = FALSE) fore$Forecast [1] 3 fore$Btarget [1] 0.4 fore$Forecast = 3 means our forecasts from the assessment will use fishing mortality (F) to attmpt to achieve a relative (to unfished) spawning stock biomass. Based on fore$Btarget, the relative biomass target is 40% of unfished spawning stock biomass. Note also that the control rule fore$BforconstantF and fore$BfornoF values are set low to make it unlikely that they will be used (these parameters are used for a ramp harvest control rule, which we do not want to use here): fore$BforconstantF [1] 0.03 fore$BfornoF [1] 0.01 Futhermore, fore$Flimitfraction is set to 1 so that the forecasted catch is set equal to the overfishing limit (for simplicity): fore$Flimitfraction [1] 1 Note that the number of forecast years is 1: fore$Nforecastyrs [1] 1 However, an assessment will be conducted every 3 years and thus 3 years of projections is required. SSMSE will automatically modify this value in the estimation model to the appropriate number of forecasting years. More information on using the forecast module in SS to forecast catches is available in the Stock Synthesis users manual. Users can also specify their own [custom management procedures] 2.5 Run SSMSE Now, we create a directory to store our results, and use run_SSMSE to run the MSE analysis loop (note this will take some time to run, ~ 20 min): run_res_path <- file.path(run_SSMSE_dir, "results") dir.create(run_res_path) res <- run_SSMSE( scen_name_vec = c("h-ctl", "h-1"),# name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = c(5,5), # run with 5 iterations each OM_name_vec = NULL, # specify directories instead OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files EM_name_vec = c("cod", "cod"), # cod is included in package data MS_vec = c("EM","EM"), # The management strategy is specified in the EM nyrs_vec = c(6, 6), # Years to project OM forward nyrs_assess_vec = c(3, 3), # Years between assessments future_om_list = future_om_list_recdevs_sel, run_parallel = TRUE, # Run iterations in parallel sample_struct_list = sample_struct_list_all, # How to sample data for running the EM. sample_struct_hist_list = NULL, # because this is null, will just use sampling # as in the current OM data file for the historical period. seed = 12345) #Set a fixed integer seed that allows replication See ?run_SSMSE for more details on function arguments. In a real MSE analysis, running 100+ iterations to reflect the full range of uncertainty (given observation and process errors) in the results would be preferred. However, we are only running 5 iterations per scenario in this demonstration to reduce computing time. 2.6 run_SSMSE output run_SSMSE will create new folders in the folders specified in out_dir_scen_vec (note that in this case, we are running both scenarios in the same folder). After is complete, there will be a folder for each scenario in run_res_path (since out_dir_scen_vec = run_res_path in this example). Within each scenario is a folder for each scenario. And within each scenario folder, there are folders containing the SS models that were run by run_SSMSE. There should be 1 folder for the OM, which is run multiple times in this same folder during the MSE analysis. There are multiple folders for the EMs, as a new folder is created each time an assessment is done. The first run is the folder with a name ending in init; then, each assessment after is named for the updated end year of the model. With many iterations, the number of files adds up; in the future, we hope to add options to save less output. 2.7 Performance metrics Quantitative performance metrics should be specified before conducting an MSE. Typically, a suite of performance metrics will be examined; however, for simplicity in this example, we will only look at what the achieved relative biomass was for the last 3 years of projection in the MSE to determine how it compares to the intended management target of 40% of unfished Spawning Stock Biomass. Note that we are only running our MSE projection for 6 years, but longer projections are typical in MSE analyses. 2.8 Summarize results The function SSMSE_summary_all can be used to summarize the model results in a list of 3 dataframes, one for scalar outputs (named scalar), one for timeseries outputs (ts), one for derived quantities (dq). This function also creates summary csv files in the folder where the results are stored. # Summarize 1 iteration of output summary <- SSMSE_summary_all(run_res_path) ## Extracting results from 2 scenarios ## Starting h-1 with 5 iterations ## Starting h-ctl with 5 iterations Plotting and data manipulation can then be done with these summaries. For example, SSB over time by model can be plotted. The models include the Operating Model (cod_OM), Estimation model (EM) for the historical period of years 0-100 (cod_EM_init), and the EM run with last year of data in year 103 (cod_EM_103). The operating models are shown in blue or black (depending on the scenario), and the estimation model runs are shown in orange, and the scenarios are shown on different subplots: library(ggplot2) # use install.packages("ggplot2") to install package if needed Warning: package 'ggplot2' was built under R version 4.0.5 library(tidyr) # use install.packages("tidyr") to install package if needed Warning: package 'tidyr' was built under R version 4.0.5 library(dplyr) # use install.packages("dplyr") to install package if needed Warning: package 'dplyr' was built under R version 4.0.5 Attaching package: 'dplyr' The following objects are masked from 'package:stats': filter, lag The following objects are masked from 'package:base': intersect, setdiff, setequal, union 2.9 Simple Convergence Check Check there are no params on bounds or SSB that is way too small or way too large check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) { require(dplyr) # note: not the best way to do this if(any(!is.na(summary$scalar$params_on_bound))) { warning("Params on bounds") } else { message("No params on bounds") } summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM") calc_SSB <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% select(iteration, scenario, year, model_run, model_type, SpawnBio) OM_vals <- calc_SSB %>% filter(model_type == "OM") %>% rename(SpawnBio_OM = SpawnBio ) %>% select(iteration, scenario, year, SpawnBio_OM) EM_vals <- calc_SSB %>% filter(model_type == "EM") %>% rename(SpawnBio_EM = SpawnBio) %>% select(iteration, scenario, year, model_run, SpawnBio_EM) bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM) filter_SSB <- bind_vals %>% filter(SSB_ratio > 2 | SSB_ratio < 0.5) if(nrow(filter_SSB) > 0 ) { warning("Some large/small SSBs relative to OM") } else { message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM") } return_val <- bind_vals } values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5) No params on bounds All SSBs in EM are no greater than double and no less than half SSB vals in the OM 2.10 Plot Spawning Stock Biomass (SSB) This plot shows that SSB estimated does not match perfectly with the operating model. A similar plot could be made for any parameter of interest. # plot SSB by year and model run ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), ggplot2::aes(x = year, y = SpawnBio)) + ggplot2::geom_vline(xintercept = 100, color = "gray") + ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+ ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) + ggplot2::scale_linetype_manual(values = rep("solid", 50)) + ggplot2::guides(linetype = FALSE) + ggplot2::facet_wrap(. ~ scenario) + ggplot2::theme_classic() Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> = "none")` instead. Now, we calculate and plot the performance metric, which is average spawning stock biomass (SSB) from years 104 to 106. # get_SSB_avg calculates the SSB in each year for each # iteration of the operating model, then takes the average over the years from # min_yr, to max_year. It uses the summary object as input to do these # calculations. get_SSB_avg <- function(summary, min_yr, max_yr) { OM_vals <- unique(summary$ts$model_run) OM_vals <- grep("_OM$", OM_vals, value = TRUE) SSB_yr <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% filter(model_run %in% OM_vals) %>% select(iteration, scenario, year, SpawnBio) %>% group_by(iteration, scenario) %>% summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% ungroup() SSB_yr } avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106) # function to summarize data in plot data_summary <- function(x) { m <- mean(x) ymin <- m - sd(x) ymax <- m + sd(x) return(c(y = m, ymin = ymin, ymax = ymax)) } # Now, plot the average relative spawning stock biomass for years 104 - 106 ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) + stat_summary(fun.data = data_summary, position = position_dodge(width = 0.9), color = "blue") + labs(title = "Long-term average SSB\\n(years 104-106)", x = "Scenario", y = "SSB") + theme_classic() From the above plot, we see differences in the average SSb between the 2 scenarios. 2.11 Example MSE Results We can see from the performance metric that mis-specifying the value of steepness will results in higher realized relative spawning stock biomass than correctly specifying it. This gives us some idea of the consequences of misspecifying steepness in the stock assessment. 2.12 Delete the files If you wish to delete the files created from this example, you can use: unlink(run_SSMSE_dir, recursive = TRUE) "],["advanced-options-use-a-custom-management-strategyprocedure.html", "3 Advanced options: use a custom management strategy/procedure", " 3 Advanced options: use a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } The function should be created in a separate file. In this case, assume this function is available in a file custom_funs.R. This function can then be used in a call to run_SSMSE: run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = 1, OM_name_vec = "cod", # specify directories instead OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use the custom function custom_MS_source = "custom_funs.R", # File where the custom function is available. nyrs_vec = 6, # Years to project OM forward nyrs_assess_vec = 3, # Years between assessments future_om_list = future_om_list_recdevs_sel, sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS. seed = 12345) #Set a fixed integer seed that allows replication "],["SSMSE.html", "4 Options for run_SSMSE 4.1 Scenarios in SSMSE 4.2 Operating model 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) 4.4 Sampling options 4.5 Future changes to the operating model (#future)", " 4 Options for run_SSMSE Many inputs are possible for the run_SSMSE() option. Here, we will describe some of the options available. For detailed documentation, type ?SSMSE::run_SSMSE() into the R console. 4.1 Scenarios in SSMSE Note that multiple scenarios can be called in run_SSMSE(), often through vector inputs to run_SSMSE(). Below, we will describe inputs needed to run 1 scenario. 4.2 Operating model The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = \"cod\". Otherwise, the path to the OM model should be specified in OM_in_dir_vec. 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) The management strategy (and EM) can be specified in one of two ways: Using an SS model Using a custom procedure via a function in R In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet. 4.3.1 Specify the Management Strategy in a SS model An SS model can be set up as the EM for the MSE. To use this option, specify \"EM\" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify \"cod\" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec. Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user. 4.3.2 Using a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } Lets assume this function is saved in a file within the working directory named constant_catch_MS.R. This function can then be used in a call to run_SSMSE(): # define sample structure datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning sample_struct$lencomp <- NULL # don't use length sampling # run the SSMSE routine run_result_custom <- run_SSMSE( scen_name_vec = "constant-catch", out_dir_scen_vec = "my_results", iter_vec = 1, OM_name_vec = "cod", OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use custom fun custom_MS_source = "constant_catch_MS.R", use_SS_boot_vec = TRUE, nyrs_vec = 6, nyrs_assess_vec = 3, run_EM_last_yr = FALSE, run_parallel = FALSE, sample_struct_list = list(sample_struct), seed = 12345) 4.4 Sampling options Currently, the only available sampling option is to use the bootstrapping module within SS itself. This means specifying use_SS_boot_vec = TRUE. Details on how sampling is done using the bootstrapping module in SS is available in the Bootstrap Data Files section of the SS user manual. Users also need to specify how and which data types should be sampled for each future year in the simulation in sample_struct_list. sample_struct_list is a list of lists. The first level is a list for each scenario; then, for the scenario, there is a list of dataframes, each of which specifying which years and fleets of data should be sampled in the future as well as which standard errors or sample sizes should be used. The helper function create_sample_struct() can be used to help users generate the list of dataframes for a scenario. See an example of this functions use in the simple example or by typing ?SSMSE::create_sample_struct() into the R console. Users can specify the sampling during the historical period of the model through the sample_struct_hist input to run_SSMSE. This is an optional input and has the same structure as sample_struct_list, but the years will be before the original end year of the OM. 4.5 Future changes to the operating model (#future) By default, SSMSE will simply extend the structure of the OM into the future using the same parameter values as in the last year of the model. However, the user may want to allow changes in parameter values to occur as the OM is extended forward in time. Users can input values into the future_om_list to accomplish this. This input is a list of lists. For the first level of lists, each represents a separate change to make. Within the first level, there are 4 list components that outline the details of the change to make. There are 2 main choices: 1) to specify model changes by telling SSMSE how values should be sampled or 2) to input your own custom values for future values of a given parameter. 4.5.1 The strucutre of future_om_list For example, here is an example list containing just one future change for the model. It shows that the model should be changed to 4.5 in year 103 and afterwards: my_future_om_list <- create_future_om_list(list_length = 1) my_future_om_list [[1]] [[1]]$pars [1] "SizeSel_P_3_Fishery(1)" [[1]]$scen [1] "replicate" "scen2" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param 1 NA NA 102 103 mean method value 1 absolute 4.5 length(my_future_om_list) # note has length 1 b/c 1 change [1] 1 length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change [1] 4 Note there is just one change specified here. For the change, there are four list items that are required for any specified change. The first list item is named pars. It contains a vector of parameter name(s) to apply the change to. The names should be the same as the names in r4ss::SS_read_pars() or can include the following: \"rec_devs\" - For specifying recruitment deviations \"impl_error - For specifying implementation error in transforming a management procedures specified future catch into realized catch. \"all\" - Apply the change to all parameters in the model, with the exception of SR_sigmaR, SR_regime, SR_autocorr, and impl_error parameters. Changing the first 3 stock recruitment related parameters would conflict with changes in recruitment deviations. Since implementation error is not a parameter specified in the control file and SSMSE does not rely on the implementation error parameter created through the forecasting file, including the implementation error in all did not seem appropriate. In this case, we just want to apply the change to the SizeSel_P_3_Fishery(1) parameter of the model. The second item is scen, which contains a vector of information about how to apply the changes within and across scenarios. The first value is an option to specify how the change should be applied among scenarios, either randomize (use a different value for each iteration of each scenario) or replicate (use the same set of values across scenarios for the same number iteration, but each value will be different across iterations within the same scenario). Note that the same values will be applied across iterations and scenarios if there isnt any stochasticity in the values being drawn (e.g., standard deviation set at 0), regardless if randomize or replicate is chosen. In the example above, there is no stochasticity, so specifying randomize or replicate does not matter. Following the first value are the names of the scenarios to apply the change to. If the change should be applied to all of the scenarios, all can be used in place of naming every scenario. In this example, The change will only be applied to the scenario named scen2, so the input for scen is c(\"randomize\", \"scen2\"). The pattern is a vector of character inputs. The first value should be either model_change, meaning that SSMSE will calculate the change values, or custom which allows the user to directly put in the values that they want in the model. In this case, model_change is used. A second input can also be added when the model_change pattern is used, which specifies the distribution to pull from. This could be normal or lognormal, but if no input is provided, a normal distribution will be used for sampling. The fourth item, named input, is a dataframe, which contains different column name values depending on if the model_change pattern is used or if the custom pattern is used. For the model_change options, a dataframe in input specifies a change for the parameter of a distribution. Because we are using the model change option, we have the following columns: first_yr_averaging: The first year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_averaging: The last year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_orig_val: The last year of the future deviations with the original model value. This value will not be changed, but the following years value will be. first_yr_final_val: The first year where the final value as specified will be reached. If no future changes are made, the final value will continue forward in the OM through all projected years. For step changes, the first_yr_final_val is just 1 year after the last_yr_orig_val. However, if a more gradual adjustment toward the final value is desired, this could be any number of years after the original value. ts_param: The sampling parameter to modify. Options are mean, ar_1_phi (if using autocorrelation), sd, and cv. If not included in the dataframe, mean is assumed to be the same as in the previous model year, sd is assumed to be 0, and we assume no autocorrelation (as specified through an autoregressive AR1 process). Phi should be between -1 and 1 (this creates a stationary process) and if phi = 1, this creates a random walk (as random walk is a special case of an AR1 model). Note that both sd and cv cannot be specified within the same input dataframe. method: How to apply the change relative to the historical (if specified) or previous years value (if first_yr_averaging and last_yr_averaging for that row are NA). Options are multiplicative, additive, and absolute. This example uses absolute, meaning that the number in value is directly used. value. The value of the parameter change. This may be NA if historical averaging is used. 4.5.2 Example of future_om_list denoting a gradual change Suppose we wanted to change the value of SizeSel_P_3_Fishery(1) in scen2 over 4 years after year 102 to arrive at a value of 4.5 in year 106. We can do this by using my_future_om_list, but changing the value in the first row of first_yr_final_val to 106: my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106 my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param 1 NA NA 102 106 mean method value 1 absolute 4.5 4.5.3 Example of future_om_list with a random deviations Suppose we now wanted the value of SizeSel_P_3_Fishery(1) to change randomly according to a normal distribution with a standard deviation of 0.1 around a mean of 4.5 from 104 onwards. This can be done by adding a line specifying a change in standard deviation (which for now, has been assumed to be 0) to the data frame: new_vals <- data.frame(first_yr_averaging = NA, last_yr_averaging = NA, last_yr_orig_val = 103, first_yr_final_val = 104, ts_param = "sd", method = "absolute", value = 0.1) my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]], new_vals) my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param 1 NA NA 102 106 mean 2 NA NA 103 104 sd method value 1 absolute 4.5 2 absolute 0.1 Note that the last_yr_orig_val and first_yr_final_val are different than the line for the mean, which is allowed. 4.5.4 Example of using historical values for determining parameter values This example applies random annual deviations to all parameters for scenarios scen2 and scen3. future_om_list_2 <- vector(mode = "list", length = 1) future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4)) names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input") future_om_list_2[[1]][["pars"]] <- "all" future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3") future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1), last_yr_averaging = c(100, 100), last_yr_orig_val = c(100, 100), first_yr_final_val = c(101, 101), ts_param = c("cv", "mean"), method = c("absolute", "multiplier"), value = c(0.1, 1)) Note that the choice of year range for historical values does not matter unless the parameter is already time-varying in the original operating model or has become time varying through a previous change. Otherwise, the base model parameter will be applied. If no historical years are included, then the base parameter value will be the basis of comparison for relative changes (i.e., method = multiplier or additive). 4.5.5 Example using custom pattern instead of model_change custom_future_om_list <- create_future_om_list(example_type = "custom", list_length = 1) custom_future_om_list [[1]] [[1]]$pars [1] "impl_error" [[1]]$scen [1] "randomize" "all" [[1]]$pattern [1] "custom" [[1]]$input par scen iter yr value 1 impl_error scen1 1 101 1.05 2 impl_error scen1 2 101 1.05 3 impl_error scen1 3 101 1.05 4 impl_error scen1 4 101 1.05 5 impl_error scen1 5 101 1.05 6 impl_error scen1 1 102 1.05 7 impl_error scen1 2 102 1.05 8 impl_error scen1 3 102 1.05 9 impl_error scen1 4 102 1.05 10 impl_error scen1 5 102 1.05 11 impl_error scen1 1 103 1.05 12 impl_error scen1 2 103 1.05 13 impl_error scen1 3 103 1.05 14 impl_error scen1 4 103 1.05 15 impl_error scen1 5 103 1.05 16 impl_error scen1 1 104 1.05 17 impl_error scen1 2 104 1.05 18 impl_error scen1 3 104 1.05 19 impl_error scen1 4 104 1.05 20 impl_error scen1 5 104 1.05 21 impl_error scen1 1 105 1.05 22 impl_error scen1 2 105 1.05 23 impl_error scen1 3 105 1.05 24 impl_error scen1 4 105 1.05 25 impl_error scen1 5 105 1.05 26 impl_error scen1 1 106 1.05 27 impl_error scen1 2 106 1.05 28 impl_error scen1 3 106 1.05 29 impl_error scen1 4 106 1.05 30 impl_error scen1 5 106 1.05 31 impl_error scen2 1 101 1.10 32 impl_error scen2 2 101 1.10 33 impl_error scen2 3 101 1.10 34 impl_error scen2 4 101 1.10 35 impl_error scen2 5 101 1.10 36 impl_error scen2 1 102 1.10 37 impl_error scen2 2 102 1.10 38 impl_error scen2 3 102 1.10 39 impl_error scen2 4 102 1.10 40 impl_error scen2 5 102 1.10 41 impl_error scen2 1 103 1.10 42 impl_error scen2 2 103 1.10 43 impl_error scen2 3 103 1.10 44 impl_error scen2 4 103 1.10 45 impl_error scen2 5 103 1.10 46 impl_error scen2 1 104 1.10 47 impl_error scen2 2 104 1.10 48 impl_error scen2 3 104 1.10 49 impl_error scen2 4 104 1.10 50 impl_error scen2 5 104 1.10 51 impl_error scen2 1 105 1.10 52 impl_error scen2 2 105 1.10 53 impl_error scen2 3 105 1.10 54 impl_error scen2 4 105 1.10 55 impl_error scen2 5 105 1.10 56 impl_error scen2 1 106 1.10 57 impl_error scen2 2 106 1.10 58 impl_error scen2 3 106 1.10 59 impl_error scen2 4 106 1.10 60 impl_error scen2 5 106 1.10 61 impl_error scen3 1 101 1.10 62 impl_error scen3 2 101 1.10 63 impl_error scen3 3 101 1.10 64 impl_error scen3 4 101 1.10 65 impl_error scen3 5 101 1.10 66 impl_error scen3 1 102 1.10 67 impl_error scen3 2 102 1.10 68 impl_error scen3 3 102 1.10 69 impl_error scen3 4 102 1.10 70 impl_error scen3 5 102 1.10 71 impl_error scen3 1 103 1.10 72 impl_error scen3 2 103 1.10 73 impl_error scen3 3 103 1.10 74 impl_error scen3 4 103 1.10 75 impl_error scen3 5 103 1.10 76 impl_error scen3 1 104 1.10 77 impl_error scen3 2 104 1.10 78 impl_error scen3 3 104 1.10 79 impl_error scen3 4 104 1.10 80 impl_error scen3 5 104 1.10 81 impl_error scen3 1 105 1.10 82 impl_error scen3 2 105 1.10 83 impl_error scen3 3 105 1.10 84 impl_error scen3 4 105 1.10 85 impl_error scen3 5 105 1.10 86 impl_error scen3 1 106 1.10 87 impl_error scen3 2 106 1.10 88 impl_error scen3 3 106 1.10 89 impl_error scen3 4 106 1.10 90 impl_error scen3 5 106 1.10 The inputs required for pattern = custom are similar to using pattern = model_change, but there is no need to specify a distribution as the second vector input of pattern and the column names in input are different. Also that the change is randomized, which indicates that a value for each scenario and each iteration are necessary, whereas if the first value of scen was replicate, separate values for each scenario should not be specified, but rather a single value for all should be used. The columns in the dataframe are: par: the parameter name or all if it should be applied to all parameters in the pars input scen: which scenario the value should apply to, or all if replicate is used as the scenario input iter: which number iteration the value should apply to. These are absolute iteration numbers. yr: the year the value applies to. These should be after the original end year of the OM. value: the value to apply to the model 4.5.6 How is the operating modified to accomodate changes as specified in the future_OM_list? All changes are made by converting the parameter(s) with changes to be time varing by using additive annual parameter deviations. Because this is an operating model, Stock Synthesis is not run with estimation, so to get the changes into the OM, values (drawn or calculated in model_changes or specified by the user using custom) are directly input as parameter deviations into the ss.par file. If an OM already contains time varying parameters, these parameters will also be converted to additive parameter deviations before applying the changes specified in the future_OM_list. 4.5.7 Example of specifying recruitment deviations This shows some code for putting recruitment deviations with a mean of 0 and the same standard deviation as the historical recruitment deviations in years 1 to 100: template_mod_change <- create_future_om_list(list_length = 1) rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" rec_dev_specify$scen <- c("replicate", "all") rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" rec_dev_specify$input$value <- NA rec_dev_list <- list(rec_dev_specify) rec_dev_list [[1]] [[1]]$pars [1] "rec_devs" [[1]]$scen [1] "replicate" "all" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param 1 1 100 100 101 sd method value 1 absolute NA "],["helper.html", "5 Helper functions in SSMSE 5.1 Creating multiple operating models with develop_OMs() 5.2 Set up sampling for a scenario with create_sample_struct() 5.3 Get examples of the future_om_list input with create_future_om_list()", " 5 Helper functions in SSMSE run_SSMSE() requires some detailed inputs that are time consuming to make. In the interest of automating routines as much as possible, helper functions are available in SSMSE. 5.1 Creating multiple operating models with develop_OMs() This function allows users to change a parameter in a Stock Synthesis OM and, if desired, refit the operating model to data. Typically, users will want to create multiple operating models to use in different scenarios in order to account for uncertainties in parameter values, such as biological parameters relating to natural mortality or growth. 5.2 Set up sampling for a scenario with create_sample_struct() run_SSMSE() requires the input sample_struct, which outlines the future sampling structure to use to generate data sets. This object could be difficult to create manually and users may want to just continue sampling as previously done in an existing SS model. create_sample_struct() will use the time patterns in sampling for different data types found in an SS data file and extend it forward in time. If no pattern is found, the function will return NAs and provide a warning to the user. If any NAs are returned (except to indicate that an entire data type is missing), the user will need to remove or fill in the NA value before usign the list as an input to run_SSMSE(). See use of create_sample_struct() in simple example. 5.3 Get examples of the future_om_list input with create_future_om_list() Users can modify the future structure of their operating model through the future_om_list input to run_SSMSE. The structure and names of this object matter, so examples are provided by calling create_future_om_list(). See more details about the specification of future_om_list objects. "],["output.html", "6 Output and Plots 6.1 Summarizing output 6.2 Checking estimation model convergence 6.3 Calculating performance metrics", " 6 Output and Plots 6.1 Summarizing output Output is summarized using SSMSE_summary_all(): summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir", scenarios = c("sample_low", "sample_high"), run_parallel = TRUE) Relying on ss3sim::get_results_all(), this function creates: For each scenario, 3 scenario level .csv files For all scenarios, 2 cross-scenario .csv files named by default to SSMSE_tsand SSMSE_scalar. For all scenarios, the function returns a list object containing data frames of timeseries (ts), scalar, and derived quantities (dq) summaries. Note that run_parallel = TRUE is only faster than run_parallel FALSE when there is more than once scenario and none of the scenario-level .csv files have been created yet. By default, if a user doesnt specify scenarios, all scenarios in dir will be summarized. 6.2 Checking estimation model convergence One of the first checks to do if using an estimation model after running an MSE analysis is to check that the estimation model has converged. A number of checks could be done, but a basic one is checking the gradients for the estimation model, which are added to the SSMSE_scalar summary sheet. 6.3 Calculating performance metrics Typically, a suite of performance metrics are used in MSE. Punt et al. (2016) recommends at least using metrics related to: average catch variation in catch over time population size "],["plotting-output-and-performance-metrics.html", "7 Plotting output and performance metrics", " 7 Plotting output and performance metrics Currently, it has been left up to the user to plot summaries and performance metrics, as the potential options for plots and performance metrics users may desire are extensive. There are currently diagnostic plots to examine how sampled data compares to the operating model values and data sets used to condition the operating models. plot_index_sampling() creates a diagnostic plot for indices, while plot_comp_sampling() creates diagnostic plots for composition data. "]] +[["index.html", "SSMSE user manual Preface R session information", " SSMSE user manual Kathryn Doering and Nathan Vaughan 2021-09-21 Preface Logo for the SSMSE package This is the user manual for SSMSE, an R package for Management Strategy Evaluation with Stock Synthesis Operating models. This document is still a work in progress! If you would like to suggest changes, the R markdown files used to generate the user manual are available in a bookdown folder within the SSMSE repository. Please feel free to open an issue or submit a pull request to suggest changes. R session information sessionInfo() R version 4.1.1 (2021-08-10) Platform: x86_64-w64-mingw32/x64 (64-bit) Running under: Windows 10 x64 (build 19042) Matrix products: default locale: [1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252 LC_MONETARY=English_United States.1252 [4] LC_NUMERIC=C LC_TIME=English_United States.1252 attached base packages: [1] stats graphics grDevices utils datasets methods base loaded via a namespace (and not attached): [1] ss3sim_1.1.6 r4ss_1.43.0 "],["intro.html", "1 Introduction 1.1 Purpose 1.2 Functions in SSMSE 1.3 Brief description of the SSMSE MSE simulation procedure", " 1 Introduction 1.1 Purpose SSMSE allows users to directly use Stock Synthesis (SS) as an operating model (OM) in an Management Strategy Evaluation (MSE). The approach requires a conditioned Stock Synthesis model, which is treated as the OM. 1.2 Functions in SSMSE The main functions users can call in SSMSE are: Function Description run_SSMSE() Run the MSE simulations SSMSE_summary_all() Summarize MSE output The helper functions to create inputs to run_SSMSE are: Helper Function Description create_sample_struct() Helper function to create a list for future sampling from a model to use as input in run_SSMSE() create_future_om_list() Helper function that provides examples of the structure for the future_om_list input to run_SSMSE(). develop_OMs() Helper function to turn one OM into many Exported functions that can be used for writing custom management strategies are: Function Description run_EM() Run an SS estimation model (uses run_ss_model) run_ss_model() Run an SS model get_bin() Get location of the SS binary. parse_MS() Function that runs the management strategy and returns catch by fleet for the projections. A reference function for those setting up custom management strategies. Finally, some plotting functions are available: Plotting Function Description plot_index_sampling() Plot to compare the sampled index values to the operating model expected values and original operating model conditioning index data. plot_comp_sampling() Plot to compare the sampled composition values to the operating model expected values and original operating model conditioning composition data. 1.3 Brief description of the SSMSE MSE simulation procedure 1.3.1 Conditioning the OM and sampling from the OM For each scenario, SSMSE starts with the user providing a fitted Stock Synthesis model (or selecting an model from the SSMSE package) to use as an OM. For each iteration of the scenario, SSMSE turns the SS fitted model into an OM and runs it once with no estimation with Stock Synthesis in order to get the true values and a bootstrapped data set from SS. Note that any modifications to the OMs parameters specified by the users as it is extended forward in time are also applied. 1.3.2 First run of the management strategy in the MSE simulation The bootstrapped dataset is then used in a Management strategy to project catch by fleet to use for the next \\(n\\) years, where \\(n\\) is the number of years between assessments. 1.3.3 Feedback from managment strategy into OM: extending model years The catch for the next \\(n\\) years before the next assessment is then added to the OM, as well as any recruitment or time varying parameter deviations. The OM is again run with no estimation where it can be used to produce sampled data for the next \\(n\\) years. These new data values are appended to the original dataset. 1.3.4 Subsequent runs of the management strategy The appended data set is then used in the managment strategy again, and new catch by fleet is produced that can then be fed back to the OM. "],["simple.html", "2 A simple example 2.1 Setup R workspace folders 2.2 Create the operating models (OMs) 2.3 Adding process error through recruitment deviations and time-varying selectivity 2.4 Examine the management procedure used 2.5 Run SSMSE 2.6 run_SSMSE output 2.7 Performance metrics 2.8 Summarize results 2.9 Simple Convergence Check 2.10 Plot Spawning Stock Biomass (SSB) 2.11 Example MSE Results 2.12 Delete the files", " 2 A simple example Suppose we want to look at how well we are able to achieve a performance metric under uncertainty in the operating model (OM). We will look 2 scenarios, one where Steepness (h) is specified correctly and one where it is specified incorrectly in an estimation model (EM): Scenario 1. h-ctl: Cod operating model (h = 0.65) with correctly specified cod model EM (fixed h = 0.65). The OM is the same as the EM. Scenario 2. h-1: Cod operating model (h = 1) with misspecified cod model EM (fixed h = 0.65); The OM is not the same as the EM. Note that this is a simple example where the OM and EM structures for both scenarios are identical, except for different steepness between the OM and EM in scenario 2 and some process error we will include in the operating model. We will assume we want to run the MSE loop for 6 years, with a stock assessment occuring every 3 years (and forecasting catch to maintain 40% of unfished spawning stock biomass). The cod models last year is 100, so the OM is initially conditioned through year 100. Then, after conditioning the operating model through year 100, assessments will occur in years 100 and 103. The operating model runs through year 106. We chose not to run the assessment in year 106, as there was no need for its output in this example. 2.1 Setup R workspace folders First, we will load the SSMSE package and create a folder in which to run the example: library(SSMSE) #load the package library(r4ss) #install using remotes::install_github("r4ss/r4ss) library(foreach) #if using run_parallel = TRUE library(doParallel) #if using run_parallel = TRUE # Create a folder for the output in the working directory. run_SSMSE_dir <- file.path("run_SSMSE-ex") dir.create(run_SSMSE_dir) Warning in dir.create(run_SSMSE_dir): 'run_SSMSE-ex' already exists 2.2 Create the operating models (OMs) 2.2.1 Specify alternative values for h The cod model with h = 0.65 (as in scenario 1) is included as external package data in SSMSE. However, we will need to modify it to use as an operating model with h = 1 (as in scenario 2). Note in this case that refit_OM is false, so the model is not being refit, just run through without fitting. To condition the new model on the same data as the input model, refit_OM should be TRUE. First, we identify where the base cod model is stored, modify it such that the steepness parameter is 1, and save the modified cod OM for scenario 2 in a new folder in the run_SSMSE_dir directory. cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE") # develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir # specified develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep", par_vals = 1, refit_OMs = FALSE, hess = FALSE) Warning in file.rename(from = file.path(out_dir, basename(OM_in_dir)), to = tmp_mod_path): cannot rename file 'run_SSMSE- ex/cod' to 'run_SSMSE-ex/cod_SR_BH_steep_1', reason 'Access is denied' # OM model for scenario 2 cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1") 2.3 Adding process error through recruitment deviations and time-varying selectivity Recruitment deviations, implementation error, and changes in parameters in the projection period of the OM can be added through the future_om_list input to run_SSMSE. First, well set up the list to add recruitment deviations in the projection period. The same recruitment deviation patterns are used across scenarios, but different patterns are use across iterations in the same scenario. We also want these deviations to have the same standard deviations as the historical deviations with 0 mean (the assumed default). # Start from a list created by a helper function template_mod_change <- create_future_om_list() # add recruitment deviations rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" # apply change to rec devs rec_dev_specify$scen <- c("replicate", "all") # using 1 to 100 means the sd or mean will be calculated by taking the sd across years # from 1 to 100 rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" # this change is for the sd # no input value needed since it will be calclated from the historical rec devs. rec_dev_specify$input$value <- NA rec_dev_specify $pars [1] "rec_devs" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 1 100 100 101 sd absolute NA Next, suppose we want to allow selectivity to vary annually for 1 selectivity parameter of the fishery throughout the projection period. The following specifies that the value for selectivity varies randomly around the base value with a sd of 0.2. # put together the change for selectivity (random values around the orig val, with # an sd of 0.2) mod_change_sel <- template_mod_change[[1]] mod_change_sel$scen[2] <- "all" # apply to all scenarios # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. # historical values are NA in this case, because they are not used to determine # the sd to use. mod_change_sel$input$last_yr_orig_val <- 100 mod_change_sel$input$first_yr_final_val <- 101 mod_change_sel$input$ts_param <- "sd" # this change is for the sd mod_change_sel$input$value <- 0.2 # se to use in the projection period mod_change_sel $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 100 101 sd absolute 0.2 Finally, add these two changes together into an object to pass to run_SSMSE future_om_list_recdevs_sel <- list(rec_dev_specify, mod_change_sel) 2.3.1 Add observation error through sampling from OM The argument sample_struct specifies the structure for sampling from the OM (and passing to the EM). The function create_sample_struct can be used to construct a simple sampling structure consistent with an input data file: datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this dataframe. sample_struct_1_scen $catch Yr Seas FltSvy SE 1 101 1 1 0.005 2 102 1 1 0.005 3 103 1 1 0.005 4 104 1 1 0.005 5 105 1 1 0.005 6 106 1 1 0.005 $CPUE Yr Seas FltSvy SE 1 105 7 2 0.2 $lencomp Yr Seas FltSvy Sex Part Nsamp 1 NA 1 1 0 0 125 $agecomp Yr Seas FltSvy Sex Part Ageerr Lbin_lo Lbin_hi Nsamp 1 105 1 2 0 0 1 -1 -1 500 $meanbodywt [1] NA $MeanSize_at_Age_obs [1] NA By default, create_sample_struct identifies sampling patterns from the historical period of the OM and replicates those patterns in the projection period. In our cod example, the sample structure specifies that catch will be added to the estimation model every year (years 101 to 106), but an index of abundance (i.e., CPUE) and age composition (i.e., agecomp) will only be added in year 105. We will use the same sampling scheme for both scenarios, but it is possible to specify different sampling for each scenario. The user could modify this sampling strategy (for example, maybe age composition should also be sampled from FltSvy 2 in Yr 102; the user could add another line to the dataframe in sample_struct$agecomp). Note that length comp (lencomp) includes an NA value for year. This is because no consistent pattern was identified, so the user must define their own input. In this case, we will remove sampling length comps all together: sample_struct_1_scen$lencomp <- NULL # don't use length sampling The same sampling structure will be used for both scenarios, which we define in a list below: sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen) 2.4 Examine the management procedure used We will use the same management procedure for both scenarios: Conduct a stock assessment every 3 years to get stock status. Project from this stock assessment using the SS forecast file to get projected future catch. Put this projected catch (without implementation error, in the case of this example) back into the OM. Extend the OM forward in time to get the true values for the population. Lets take a look at step 2 in the management procedure, which is implemented using the forecasting module in SS. We will examine the forecast file for the estimation model to better understand how catches will be forecasted from the assessment. We will use the same management procedure in both of these scenarios, although for a full MSE analysis, it is likely that multiple management procedures would be compared. fore <- r4ss::SS_readforecast( system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"), verbose = FALSE) fore$Forecast [1] 3 fore$Btarget [1] 0.4 fore$Forecast = 3 means our forecasts from the assessment will use fishing mortality (F) to attmpt to achieve a relative (to unfished) spawning stock biomass. Based on fore$Btarget, the relative biomass target is 40% of unfished spawning stock biomass. Note also that the control rule fore$BforconstantF and fore$BfornoF values are set low to make it unlikely that they will be used (these parameters are used for a ramp harvest control rule, which we do not want to use here): fore$BforconstantF [1] 0.03 fore$BfornoF [1] 0.01 Futhermore, fore$Flimitfraction is set to 1 so that the forecasted catch is set equal to the overfishing limit (for simplicity): fore$Flimitfraction [1] 1 Note that the number of forecast years is 1: fore$Nforecastyrs [1] 1 However, an assessment will be conducted every 3 years and thus 3 years of projections is required. SSMSE will automatically modify this value in the estimation model to the appropriate number of forecasting years. More information on using the forecast module in SS to forecast catches is available in the Stock Synthesis users manual. Users can also specify their own [custom management procedures] 2.5 Run SSMSE Now, we create a directory to store our results, and use run_SSMSE to run the MSE analysis loop (note this will take some time to run, ~ 20 min): run_res_path <- file.path(run_SSMSE_dir, "results") dir.create(run_res_path) res <- run_SSMSE( scen_name_vec = c("h-ctl", "h-1"),# name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = c(5,5), # run with 5 iterations each OM_name_vec = NULL, # specify directories instead OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files EM_name_vec = c("cod", "cod"), # cod is included in package data MS_vec = c("EM","EM"), # The management strategy is specified in the EM nyrs_vec = c(6, 6), # Years to project OM forward nyrs_assess_vec = c(3, 3), # Years between assessments future_om_list = future_om_list_recdevs_sel, run_parallel = TRUE, # Run iterations in parallel sample_struct_list = sample_struct_list_all, # How to sample data for running the EM. sample_struct_hist_list = NULL, # because this is null, will just use sampling # as in the current OM data file for the historical period. seed = 12345) #Set a fixed integer seed that allows replication See ?run_SSMSE for more details on function arguments. In a real MSE analysis, running 100+ iterations to reflect the full range of uncertainty (given observation and process errors) in the results would be preferred. However, we are only running 5 iterations per scenario in this demonstration to reduce computing time. 2.6 run_SSMSE output run_SSMSE will create new folders in the folders specified in out_dir_scen_vec (note that in this case, we are running both scenarios in the same folder). After is complete, there will be a folder for each scenario in run_res_path (since out_dir_scen_vec = run_res_path in this example). Within each scenario is a folder for each scenario. And within each scenario folder, there are folders containing the SS models that were run by run_SSMSE. There should be 1 folder for the OM, which is run multiple times in this same folder during the MSE analysis. There are multiple folders for the EMs, as a new folder is created each time an assessment is done. The first run is the folder with a name ending in init; then, each assessment after is named for the updated end year of the model. With many iterations, the number of files adds up; in the future, we hope to add options to save less output. 2.7 Performance metrics Quantitative performance metrics should be specified before conducting an MSE. Typically, a suite of performance metrics will be examined; however, for simplicity in this example, we will only look at what the achieved relative biomass was for the last 3 years of projection in the MSE to determine how it compares to the intended management target of 40% of unfished Spawning Stock Biomass. Note that we are only running our MSE projection for 6 years, but longer projections are typical in MSE analyses. 2.8 Summarize results The function SSMSE_summary_all can be used to summarize the model results in a list of 3 dataframes, one for scalar outputs (named scalar), one for timeseries outputs (ts), one for derived quantities (dq). This function also creates summary csv files in the folder where the results are stored. # Summarize 1 iteration of output summary <- SSMSE_summary_all(run_res_path) ## Extracting results from 2 scenarios ## Starting h-1 with 5 iterations ## Starting h-ctl with 5 iterations Plotting and data manipulation can then be done with these summaries. For example, SSB over time by model can be plotted. The models include the Operating Model (cod_OM), Estimation model (EM) for the historical period of years 0-100 (cod_EM_init), and the EM run with last year of data in year 103 (cod_EM_103). The operating models are shown in blue or black (depending on the scenario), and the estimation model runs are shown in orange, and the scenarios are shown on different subplots: library(ggplot2) # use install.packages("ggplot2") to install package if needed library(tidyr) # use install.packages("tidyr") to install package if needed library(dplyr) # use install.packages("dplyr") to install package if needed Attaching package: 'dplyr' The following objects are masked from 'package:stats': filter, lag The following objects are masked from 'package:base': intersect, setdiff, setequal, union 2.9 Simple Convergence Check Check there are no params on bounds or SSB that is way too small or way too large check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) { require(dplyr) # note: not the best way to do this if(any(!is.na(summary$scalar$params_on_bound))) { warning("Params on bounds") } else { message("No params on bounds") } summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM") calc_SSB <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% select(iteration, scenario, year, model_run, model_type, SpawnBio) OM_vals <- calc_SSB %>% filter(model_type == "OM") %>% rename(SpawnBio_OM = SpawnBio ) %>% select(iteration, scenario, year, SpawnBio_OM) EM_vals <- calc_SSB %>% filter(model_type == "EM") %>% rename(SpawnBio_EM = SpawnBio) %>% select(iteration, scenario, year, model_run, SpawnBio_EM) bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM) filter_SSB <- bind_vals %>% filter(SSB_ratio > 2 | SSB_ratio < 0.5) if(nrow(filter_SSB) > 0 ) { warning("Some large/small SSBs relative to OM") } else { message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM") } return_val <- bind_vals } values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5) No params on bounds All SSBs in EM are no greater than double and no less than half SSB vals in the OM 2.10 Plot Spawning Stock Biomass (SSB) This plot shows that SSB estimated does not match perfectly with the operating model. A similar plot could be made for any parameter of interest. # plot SSB by year and model run ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), ggplot2::aes(x = year, y = SpawnBio)) + ggplot2::geom_vline(xintercept = 100, color = "gray") + ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+ ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) + ggplot2::scale_linetype_manual(values = rep("solid", 50)) + ggplot2::guides(linetype = FALSE) + ggplot2::facet_wrap(. ~ scenario) + ggplot2::theme_classic() Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> = "none")` instead. Now, we calculate and plot the performance metric, which is average spawning stock biomass (SSB) from years 104 to 106. # get_SSB_avg calculates the SSB in each year for each # iteration of the operating model, then takes the average over the years from # min_yr, to max_year. It uses the summary object as input to do these # calculations. get_SSB_avg <- function(summary, min_yr, max_yr) { OM_vals <- unique(summary$ts$model_run) OM_vals <- grep("_OM$", OM_vals, value = TRUE) SSB_yr <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% filter(model_run %in% OM_vals) %>% select(iteration, scenario, year, SpawnBio) %>% group_by(iteration, scenario) %>% summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% ungroup() SSB_yr } avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106) # function to summarize data in plot data_summary <- function(x) { m <- mean(x) ymin <- m - sd(x) ymax <- m + sd(x) return(c(y = m, ymin = ymin, ymax = ymax)) } # Now, plot the average relative spawning stock biomass for years 104 - 106 ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) + stat_summary(fun.data = data_summary, position = position_dodge(width = 0.9), color = "blue") + labs(title = "Long-term average SSB\\n(years 104-106)", x = "Scenario", y = "SSB") + theme_classic() From the above plot, we see differences in the average SSb between the 2 scenarios. 2.11 Example MSE Results We can see from the performance metric that mis-specifying the value of steepness will results in higher realized relative spawning stock biomass than correctly specifying it. This gives us some idea of the consequences of misspecifying steepness in the stock assessment. 2.12 Delete the files If you wish to delete the files created from this example, you can use: unlink(run_SSMSE_dir, recursive = TRUE) "],["advanced-options-use-a-custom-management-strategyprocedure.html", "3 Advanced options: use a custom management strategy/procedure", " 3 Advanced options: use a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } The function should be created in a separate file. In this case, assume this function is available in a file custom_funs.R. This function can then be used in a call to run_SSMSE: run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = 1, OM_name_vec = "cod", # specify directories instead OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use the custom function custom_MS_source = "custom_funs.R", # File where the custom function is available. nyrs_vec = 6, # Years to project OM forward nyrs_assess_vec = 3, # Years between assessments future_om_list = future_om_list_recdevs_sel, sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS. seed = 12345) #Set a fixed integer seed that allows replication "],["SSMSE.html", "4 Options for run_SSMSE 4.1 Scenarios in SSMSE 4.2 Operating model 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) 4.4 Sampling options 4.5 Future changes to the operating model (#future)", " 4 Options for run_SSMSE Many inputs are possible for the run_SSMSE() option. Here, we will describe some of the options available. For detailed documentation, type ?SSMSE::run_SSMSE() into the R console. 4.1 Scenarios in SSMSE Note that multiple scenarios can be called in run_SSMSE(), often through vector inputs to run_SSMSE(). Below, we will describe inputs needed to run 1 scenario. 4.2 Operating model The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = \"cod\". Otherwise, the path to the OM model should be specified in OM_in_dir_vec. 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) The management strategy (and EM) can be specified in one of two ways: Using an SS model Using a custom procedure via a function in R In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet. 4.3.1 Specify the Management Strategy in a SS model An SS model can be set up as the EM for the MSE. To use this option, specify \"EM\" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify \"cod\" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec. Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user. 4.3.2 Using a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } Lets assume this function is saved in a file within the working directory named constant_catch_MS.R. This function can then be used in a call to run_SSMSE(): # define sample structure datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning sample_struct$lencomp <- NULL # don't use length sampling # run the SSMSE routine run_result_custom <- run_SSMSE( scen_name_vec = "constant-catch", out_dir_scen_vec = "my_results", iter_vec = 1, OM_name_vec = "cod", OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use custom fun custom_MS_source = "constant_catch_MS.R", use_SS_boot_vec = TRUE, nyrs_vec = 6, nyrs_assess_vec = 3, run_EM_last_yr = FALSE, run_parallel = FALSE, sample_struct_list = list(sample_struct), seed = 12345) 4.4 Sampling options Currently, the only available sampling option is to use the bootstrapping module within SS itself. This means specifying use_SS_boot_vec = TRUE. Details on how sampling is done using the bootstrapping module in SS is available in the Bootstrap Data Files section of the SS user manual. Users also need to specify how and which data types should be sampled for each future year in the simulation in sample_struct_list. sample_struct_list is a list of lists. The first level is a list for each scenario; then, for the scenario, there is a list of dataframes, each of which specifying which years and fleets of data should be sampled in the future as well as which standard errors or sample sizes should be used. The helper function create_sample_struct() can be used to help users generate the list of dataframes for a scenario. See an example of this functions use in the simple example or by typing ?SSMSE::create_sample_struct() into the R console. Users can specify the sampling during the historical period of the model through the sample_struct_hist input to run_SSMSE. This is an optional input and has the same structure as sample_struct_list, but the years will be before the original end year of the OM. 4.5 Future changes to the operating model (#future) By default, SSMSE will simply extend the structure of the OM into the future using the same parameter values as in the last year of the model. However, the user may want to allow changes in parameter values to occur as the OM is extended forward in time. Users can input values into the future_om_list to accomplish this. This input is a list of lists. For the first level of lists, each represents a separate change to make. Within the first level, there are 4 list components that outline the details of the change to make. There are 2 main choices: 1) to specify model changes by telling SSMSE how values should be sampled or 2) to input your own custom values for future values of a given parameter. 4.5.1 The strucutre of future_om_list For example, here is an example list containing just one future change for the model. It shows that the model should be changed to 4.5 in year 103 and afterwards: my_future_om_list <- create_future_om_list(list_length = 1) my_future_om_list [[1]] [[1]]$pars [1] "SizeSel_P_3_Fishery(1)" [[1]]$scen [1] "replicate" "scen2" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 103 mean absolute 4.5 length(my_future_om_list) # note has length 1 b/c 1 change [1] 1 length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change [1] 4 Note there is just one change specified here. For the change, there are four list items that are required for any specified change. The first list item is named pars. It contains a vector of parameter name(s) to apply the change to. The names should be the same as the names in r4ss::SS_read_pars() or can include the following: \"rec_devs\" - For specifying recruitment deviations \"impl_error - For specifying implementation error in transforming a management procedures specified future catch into realized catch. \"all\" - Apply the change to all parameters in the model, with the exception of SR_sigmaR, SR_regime, SR_autocorr, and impl_error parameters. Changing the first 3 stock recruitment related parameters would conflict with changes in recruitment deviations. Since implementation error is not a parameter specified in the control file and SSMSE does not rely on the implementation error parameter created through the forecasting file, including the implementation error in all did not seem appropriate. In this case, we just want to apply the change to the SizeSel_P_3_Fishery(1) parameter of the model. The second item is scen, which contains a vector of information about how to apply the changes within and across scenarios. The first value is an option to specify how the change should be applied among scenarios, either randomize (use a different value for each iteration of each scenario) or replicate (use the same set of values across scenarios for the same number iteration, but each value will be different across iterations within the same scenario). Note that the same values will be applied across iterations and scenarios if there isnt any stochasticity in the values being drawn (e.g., standard deviation set at 0), regardless if randomize or replicate is chosen. In the example above, there is no stochasticity, so specifying randomize or replicate does not matter. Following the first value are the names of the scenarios to apply the change to. If the change should be applied to all of the scenarios, all can be used in place of naming every scenario. In this example, The change will only be applied to the scenario named scen2, so the input for scen is c(\"randomize\", \"scen2\"). The pattern is a vector of character inputs. The first value should be either model_change, meaning that SSMSE will calculate the change values, or custom which allows the user to directly put in the values that they want in the model. In this case, model_change is used. A second input can also be added when the model_change pattern is used, which specifies the distribution to pull from. This could be normal or lognormal, but if no input is provided, a normal distribution will be used for sampling. The fourth item, named input, is a dataframe, which contains different column name values depending on if the model_change pattern is used or if the custom pattern is used. For the model_change options, a dataframe in input specifies a change for the parameter of a distribution. Because we are using the model change option, we have the following columns: first_yr_averaging: The first year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_averaging: The last year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_orig_val: The last year of the future deviations with the original model value. This value will not be changed, but the following years value will be. first_yr_final_val: The first year where the final value as specified will be reached. If no future changes are made, the final value will continue forward in the OM through all projected years. For step changes, the first_yr_final_val is just 1 year after the last_yr_orig_val. However, if a more gradual adjustment toward the final value is desired, this could be any number of years after the original value. ts_param: The sampling parameter to modify. Options are mean, ar_1_phi (if using autocorrelation), sd, and cv. If not included in the dataframe, mean is assumed to be the same as in the previous model year, sd is assumed to be 0, and we assume no autocorrelation (as specified through an autoregressive AR1 process). Phi should be between -1 and 1 (this creates a stationary process) and if phi = 1, this creates a random walk (as random walk is a special case of an AR1 model). Note that both sd and cv cannot be specified within the same input dataframe. method: How to apply the change relative to the historical (if specified) or previous years value (if first_yr_averaging and last_yr_averaging for that row are NA). Options are multiplicative, additive, and absolute. This example uses absolute, meaning that the number in value is directly used. value. The value of the parameter change. This may be NA if historical averaging is used. 4.5.2 Example of future_om_list denoting a gradual change Suppose we wanted to change the value of SizeSel_P_3_Fishery(1) in scen2 over 4 years after year 102 to arrive at a value of 4.5 in year 106. We can do this by using my_future_om_list, but changing the value in the first row of first_yr_final_val to 106: my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106 my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 106 mean absolute 4.5 4.5.3 Example of future_om_list with a random deviations Suppose we now wanted the value of SizeSel_P_3_Fishery(1) to change randomly according to a normal distribution with a standard deviation of 0.1 around a mean of 4.5 from 104 onwards. This can be done by adding a line specifying a change in standard deviation (which for now, has been assumed to be 0) to the data frame: new_vals <- data.frame(first_yr_averaging = NA, last_yr_averaging = NA, last_yr_orig_val = 103, first_yr_final_val = 104, ts_param = "sd", method = "absolute", value = 0.1) my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]], new_vals) my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 106 mean absolute 4.5 2 NA NA 103 104 sd absolute 0.1 Note that the last_yr_orig_val and first_yr_final_val are different than the line for the mean, which is allowed. 4.5.4 Example of using historical values for determining parameter values This example applies random annual deviations to all parameters for scenarios scen2 and scen3. future_om_list_2 <- vector(mode = "list", length = 1) future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4)) names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input") future_om_list_2[[1]][["pars"]] <- "all" future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3") future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1), last_yr_averaging = c(100, 100), last_yr_orig_val = c(100, 100), first_yr_final_val = c(101, 101), ts_param = c("cv", "mean"), method = c("absolute", "multiplier"), value = c(0.1, 1)) Note that the choice of year range for historical values does not matter unless the parameter is already time-varying in the original operating model or has become time varying through a previous change. Otherwise, the base model parameter will be applied. If no historical years are included, then the base parameter value will be the basis of comparison for relative changes (i.e., method = multiplier or additive). 4.5.5 Example using custom pattern instead of model_change custom_future_om_list <- create_future_om_list(example_type = "custom", list_length = 1) custom_future_om_list [[1]] [[1]]$pars [1] "impl_error" [[1]]$scen [1] "randomize" "all" [[1]]$pattern [1] "custom" [[1]]$input par scen iter yr value 1 impl_error scen1 1 101 1.05 2 impl_error scen1 2 101 1.05 3 impl_error scen1 3 101 1.05 4 impl_error scen1 4 101 1.05 5 impl_error scen1 5 101 1.05 6 impl_error scen1 1 102 1.05 7 impl_error scen1 2 102 1.05 8 impl_error scen1 3 102 1.05 9 impl_error scen1 4 102 1.05 10 impl_error scen1 5 102 1.05 11 impl_error scen1 1 103 1.05 12 impl_error scen1 2 103 1.05 13 impl_error scen1 3 103 1.05 14 impl_error scen1 4 103 1.05 15 impl_error scen1 5 103 1.05 16 impl_error scen1 1 104 1.05 17 impl_error scen1 2 104 1.05 18 impl_error scen1 3 104 1.05 19 impl_error scen1 4 104 1.05 20 impl_error scen1 5 104 1.05 21 impl_error scen1 1 105 1.05 22 impl_error scen1 2 105 1.05 23 impl_error scen1 3 105 1.05 24 impl_error scen1 4 105 1.05 25 impl_error scen1 5 105 1.05 26 impl_error scen1 1 106 1.05 27 impl_error scen1 2 106 1.05 28 impl_error scen1 3 106 1.05 29 impl_error scen1 4 106 1.05 30 impl_error scen1 5 106 1.05 31 impl_error scen2 1 101 1.10 32 impl_error scen2 2 101 1.10 33 impl_error scen2 3 101 1.10 34 impl_error scen2 4 101 1.10 35 impl_error scen2 5 101 1.10 36 impl_error scen2 1 102 1.10 37 impl_error scen2 2 102 1.10 38 impl_error scen2 3 102 1.10 39 impl_error scen2 4 102 1.10 40 impl_error scen2 5 102 1.10 41 impl_error scen2 1 103 1.10 42 impl_error scen2 2 103 1.10 43 impl_error scen2 3 103 1.10 44 impl_error scen2 4 103 1.10 45 impl_error scen2 5 103 1.10 46 impl_error scen2 1 104 1.10 47 impl_error scen2 2 104 1.10 48 impl_error scen2 3 104 1.10 49 impl_error scen2 4 104 1.10 50 impl_error scen2 5 104 1.10 51 impl_error scen2 1 105 1.10 52 impl_error scen2 2 105 1.10 53 impl_error scen2 3 105 1.10 54 impl_error scen2 4 105 1.10 55 impl_error scen2 5 105 1.10 56 impl_error scen2 1 106 1.10 57 impl_error scen2 2 106 1.10 58 impl_error scen2 3 106 1.10 59 impl_error scen2 4 106 1.10 60 impl_error scen2 5 106 1.10 61 impl_error scen3 1 101 1.10 62 impl_error scen3 2 101 1.10 63 impl_error scen3 3 101 1.10 64 impl_error scen3 4 101 1.10 65 impl_error scen3 5 101 1.10 66 impl_error scen3 1 102 1.10 67 impl_error scen3 2 102 1.10 68 impl_error scen3 3 102 1.10 69 impl_error scen3 4 102 1.10 70 impl_error scen3 5 102 1.10 71 impl_error scen3 1 103 1.10 72 impl_error scen3 2 103 1.10 73 impl_error scen3 3 103 1.10 74 impl_error scen3 4 103 1.10 75 impl_error scen3 5 103 1.10 76 impl_error scen3 1 104 1.10 77 impl_error scen3 2 104 1.10 78 impl_error scen3 3 104 1.10 79 impl_error scen3 4 104 1.10 80 impl_error scen3 5 104 1.10 81 impl_error scen3 1 105 1.10 82 impl_error scen3 2 105 1.10 83 impl_error scen3 3 105 1.10 84 impl_error scen3 4 105 1.10 85 impl_error scen3 5 105 1.10 86 impl_error scen3 1 106 1.10 87 impl_error scen3 2 106 1.10 88 impl_error scen3 3 106 1.10 89 impl_error scen3 4 106 1.10 90 impl_error scen3 5 106 1.10 The inputs required for pattern = custom are similar to using pattern = model_change, but there is no need to specify a distribution as the second vector input of pattern and the column names in input are different. Also that the change is randomized, which indicates that a value for each scenario and each iteration are necessary, whereas if the first value of scen was replicate, separate values for each scenario should not be specified, but rather a single value for all should be used. The columns in the dataframe are: par: the parameter name or all if it should be applied to all parameters in the pars input scen: which scenario the value should apply to, or all if replicate is used as the scenario input iter: which number iteration the value should apply to. These are absolute iteration numbers. yr: the year the value applies to. These should be after the original end year of the OM. value: the value to apply to the model 4.5.6 How is the operating modified to accomodate changes as specified in the future_OM_list? All changes are made by converting the parameter(s) with changes to be time varing by using additive annual parameter deviations. Because this is an operating model, Stock Synthesis is not run with estimation, so to get the changes into the OM, values (drawn or calculated in model_changes or specified by the user using custom) are directly input as parameter deviations into the ss.par file. If an OM already contains time varying parameters, these parameters will also be converted to additive parameter deviations before applying the changes specified in the future_OM_list. 4.5.7 Example of specifying recruitment deviations This shows some code for putting recruitment deviations with a mean of 0 and the same standard deviation as the historical recruitment deviations in years 1 to 100: template_mod_change <- create_future_om_list(list_length = 1) rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" rec_dev_specify$scen <- c("replicate", "all") rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" rec_dev_specify$input$value <- NA rec_dev_list <- list(rec_dev_specify) rec_dev_list [[1]] [[1]]$pars [1] "rec_devs" [[1]]$scen [1] "replicate" "all" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 1 100 100 101 sd absolute NA "],["helper.html", "5 Helper functions in SSMSE 5.1 Creating multiple operating models with develop_OMs() 5.2 Set up sampling for a scenario with create_sample_struct() 5.3 Get examples of the future_om_list input with create_future_om_list()", " 5 Helper functions in SSMSE run_SSMSE() requires some detailed inputs that are time consuming to make. In the interest of automating routines as much as possible, helper functions are available in SSMSE. 5.1 Creating multiple operating models with develop_OMs() This function allows users to change a parameter in a Stock Synthesis OM and, if desired, refit the operating model to data. Typically, users will want to create multiple operating models to use in different scenarios in order to account for uncertainties in parameter values, such as biological parameters relating to natural mortality or growth. 5.2 Set up sampling for a scenario with create_sample_struct() run_SSMSE() requires the input sample_struct, which outlines the future sampling structure to use to generate data sets. This object could be difficult to create manually and users may want to just continue sampling as previously done in an existing SS model. create_sample_struct() will use the time patterns in sampling for different data types found in an SS data file and extend it forward in time. If no pattern is found, the function will return NAs and provide a warning to the user. If any NAs are returned (except to indicate that an entire data type is missing), the user will need to remove or fill in the NA value before usign the list as an input to run_SSMSE(). See use of create_sample_struct() in simple example. 5.3 Get examples of the future_om_list input with create_future_om_list() Users can modify the future structure of their operating model through the future_om_list input to run_SSMSE. The structure and names of this object matter, so examples are provided by calling create_future_om_list(). See more details about the specification of future_om_list objects. "],["output.html", "6 Output and Plots 6.1 Summarizing output 6.2 Checking estimation model convergence 6.3 Calculating performance metrics", " 6 Output and Plots 6.1 Summarizing output Output is summarized using SSMSE_summary_all(): summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir", scenarios = c("sample_low", "sample_high"), run_parallel = TRUE) Relying on ss3sim::get_results_all(), this function creates: For each scenario, 3 scenario level .csv files For all scenarios, 2 cross-scenario .csv files named by default to SSMSE_tsand SSMSE_scalar. For all scenarios, the function returns a list object containing data frames of timeseries (ts), scalar, and derived quantities (dq) summaries. Note that run_parallel = TRUE is only faster than run_parallel FALSE when there is more than once scenario and none of the scenario-level .csv files have been created yet. By default, if a user doesnt specify scenarios, all scenarios in dir will be summarized. 6.2 Checking estimation model convergence One of the first checks to do if using an estimation model after running an MSE analysis is to check that the estimation model has converged. A number of checks could be done, but a basic one is checking the gradients for the estimation model, which are added to the SSMSE_scalar summary sheet. 6.3 Calculating performance metrics Typically, a suite of performance metrics are used in MSE. Punt et al. (2016) recommends at least using metrics related to: average catch variation in catch over time population size "],["plotting-output-and-performance-metrics.html", "7 Plotting output and performance metrics", " 7 Plotting output and performance metrics Currently, it has been left up to the user to plot summaries and performance metrics, as the potential options for plots and performance metrics users may desire are extensive. There are currently diagnostic plots to examine how sampled data compares to the operating model values and data sets used to condition the operating models. plot_index_sampling() creates a diagnostic plot for indices, while plot_comp_sampling() creates diagnostic plots for composition data. "],["glossary-of-mse-terms.html", "8 Glossary of MSE terms", " 8 Glossary of MSE terms This glossary is based off of Rademeyer et al. (2007) and Punt et al. (2016). Assessment error - Error that occurs during the process of conducting an assessment, specifically error which inform the catch control rule that is being evaluated using the MSE: management advice for any system is based on uncertain data. Consequently, the data that inform catch control rules need to be generated in a manner which is as realistic as possible. Uncertainty arises when the model used for conducting assessments and providing management advice differs from the operating model, or the data are too noisy to estimate all key parameters reliably (Punt et al. 2016). Assessment model (AM) - A fitted and well scrutinized Stock Synthesis stock assessment model. The AM is input for SSMSE so that it can build the operating model. Bootstrapped dataset - After running an SS model (and if the user turns on the option in the starter file), the output file data.ss_new contains a dataset that has the same form as the input data.ss file, but instead of the input values, contains a sampled dataset. This is derived from using the expected values given the parameter estimates and model structure (see glossary entry on Expected values), uncertainty either estimated or input in the model, and an assumed distribution for sampling. As many bootstrapped data sets as the user desires can be output from an SS model. These bootstrap datasets are the third or greater set of values in the data.ss_new file. Estimation model (EM) - This refers to the model used within the MSE procedure to represent the stock assessment process. Note that an estimation model proxy could also be used to represent the stock assessment process. Expected values - After running an SS model (and if the user turns on teh option in the starter file), the output file data.ss_new contains a dataset that has the same form as the input data.ss file, but instead of the input values, contains the expected values given the parameter estimates and model structure. This expected values dataset is the second set of values in the data.ss_new file. Implementation error - Also called implementation uncertainty or outcome uncertainty. Broadly, this includes error of implementing the management action(s), which is often not done perfectly. Definition from Punt et al. (2016): The most obvious form of this type of uncertainty is when catches are not the same as the TACs typically more is taken or the decision-makers do not implement the TACs suggested by the management strategy. However, there are many other sources of outcome uncertainty, such as that associated with catch limits set for recreational fisheries and regulating discards. Index - An index of abundance. In SS files, this is sometimes used interchangably with CPUE. Plural, indices. Management strategy - Following Punt et al. (2016), there are 2 types: model-based management strategies and empirical management strategies. Ideally, SSMSE will allow for users to use either of these management strategies. Model-based management strategies include conducting a stock assessment and using output for determining harvest control rules. Empirical management strategies do not involve conducting a stock assessment but rather setting regulations from data (although summarization is possible). In some cases, management strategies are computationally intensive/time consuming to formally include in the context of an MSE; to speed up the process, it is commen to use management strategy proxies, typically an assumed error distribution about the operating model values. Management strategies are also known as management procedures. Model uncertainty - Definition from Punt et al. (2016): the form of relationships within an operating model will always be subject to uncertainty. The simplest type of model uncertainty involves, for example, whether the stockrecruitment relationship is BevertonHolt or Ricker, whether a fixed value for a model parameter is correct, or whether fishery selectivity is asymptotic or dome-shaped. However, there are other more complicated types of model structure uncertainty such as how many stocks are present in the area modelled, the error structure of the data used for assessment purposes, the impact of future climate change on biological relationships such as the stockrecruitment function, and ecosystem impacts on biological and fishery processes. Observation error - Error that results from not observing the true dynamics of the system. See also bootstrapped dataset. Operating model (OM) - This is a Stock Synthesis model that defines the assumed true dynamics of the population and its associated fisheries for the purposes of management strategy evaluation. Most often more than one operating model is necessary in order to adequately characterize the uncertainty in the true dynamics of the system. The Stock Synthesis OM may also define how sampling from the true dynamics is done, as SS produces expected values and, if desired, bootstrapped data sets based on sampling assumptions implicit in the model. For more information, see the glossary entries on Expected values and bootstrapped dataset Parameter uncertainty - Definition from Punt et al. (2016): many operating models are fit to the data available, but the values estimated for the parameters of those operating models (e.g. fishery selectivity-at-age, the parameters of the stockrecruitment relationship and historical deviations in recruitment about the stockrecruitment relationship) are subject to error. Performance statistics - Definition from Rademeyer et al. (2007): Statistics that summarize different aspects of the results of a simulation trial used to evaluate how well a specific [management strategy] achieves some or all of the general objectives for management for a particular scenario. Performance statistics usually fall into one of three categories: catch-related, stability related, or risk related. Process uncertainty - Definition from Punt et al. (2016): variation (usually assumed to be random, though sometimes incorporating autocorrelation) in parameters often considered fixed in stock assessments such as natural mortality, future recruitment about a stockrecruitment relationship and selectivity. Stock Synthesis (SS, SS3) - A integrative, general population dynamics modeling program used to assess the effects of fishing on population. Available at the Stock Synthesis website Uncertainty - Incorporating uncertainty into an MSE procedure is extremely important. There are several potential sources of uncertainty, which we have divided as done in Punt et al. (2016) in addition to adding observation error. For more details, see the glossary entries on Process Uncertainty, Parameter uncertainty, Model uncertainty, Assessment error, Implementation error, and observation error. Punt et al. (2016) suggest that MSE should at least consider process uncertainty (particularly deviations from the stock recruitment relationship), parameter uncertainty (particularly which relates to productivity and stock size), and observation error. Note that Rademeyer et al. (2007) divides error into estimation error, implementation error, observation error and process error, which could be used instead. These may be more natural divisions in sources of error. "],["404.html", "Page not found", " Page not found The page you requested cannot be found (perhaps it was moved or renamed). You may want to try searching to find the page's new location, or use the table of contents to find the page you are looking for. "]] diff --git a/docs/manual/simple.html b/docs/manual/simple.html index 71ad6c74..9eb82f9b 100644 --- a/docs/manual/simple.html +++ b/docs/manual/simple.html @@ -6,7 +6,7 @@ 2 A simple example | SSMSE user manual - + @@ -24,7 +24,7 @@ - + @@ -33,7 +33,9 @@ - + + + @@ -49,29 +51,32 @@ - + + + @@ -127,25 +133,31 @@ @@ -205,60 +223,63 @@

-
+

2 A simple example

Suppose we want to look at how well we are able to achieve a performance metric under uncertainty in the operating model (OM). We will look 2 scenarios, one where Steepness (h) is specified correctly and one where it is specified incorrectly in an estimation model (EM):

Scenario 1. h-ctl: Cod operating model (h = 0.65) with correctly specified cod model EM (fixed h = 0.65). The OM is the same as the EM.

Scenario 2. h-1: Cod operating model (h = 1) with misspecified cod model EM (fixed h = 0.65); The OM is not the same as the EM.

Note that this is a simple example where the OM and EM structures for both scenarios are identical, except for different steepness between the OM and EM in scenario 2 and some process error we will include in the operating model. We will assume we want to run the MSE loop for 6 years, with a stock assessment occuring every 3 years (and forecasting catch to maintain 40% of unfished spawning stock biomass). The cod model’s last year is 100, so the OM is initially conditioned through year 100. Then, after conditioning the operating model through year 100, assessments will occur in years 100 and 103. The operating model runs through year 106. We chose not to run the assessment in year 106, as there was no need for its output in this example.

-
+

2.1 Setup R workspace folders

First, we will load the SSMSE package and create a folder in which to run the example:

- - +
library(SSMSE) #load the package
+library(r4ss) #install using remotes::install_github("r4ss/r4ss)
+library(foreach) #if using run_parallel = TRUE
+library(doParallel) #if using run_parallel = TRUE
+
# Create a folder for the output in the working directory.
+run_SSMSE_dir <- file.path("run_SSMSE-ex")
+dir.create(run_SSMSE_dir)
+
Warning in dir.create(run_SSMSE_dir): 'run_SSMSE-ex' already exists
-
+

2.2 Create the operating models (OMs)

-
+

2.2.1 Specify alternative values for h

The cod model with h = 0.65 (as in scenario 1) is included as external package data in SSMSE. However, we will need to modify it to use as an operating model with h = 1 (as in scenario 2). Note in this case that refit_OM is false, so the model is not being refit, just run through without fitting. To condition the new model on the same data as the input model, refit_OM should be TRUE.

First, we identify where the base cod model is stored, modify it such that the steepness parameter is 1, and save the modified cod OM for scenario 2 in a new folder in the run_SSMSE_dir directory.

- +
cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE")
+# develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir
+# specified
+develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep",
+            par_vals = 1, refit_OMs = FALSE, hess = FALSE)
+
Warning in file.rename(from = file.path(out_dir, basename(OM_in_dir)), to = tmp_mod_path): cannot rename file 'run_SSMSE-
+ex/cod' to 'run_SSMSE-ex/cod_SR_BH_steep_1', reason 'Access is denied'
+
# OM model for scenario 2
+cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1")
-
+

2.3 Adding process error through recruitment deviations and time-varying selectivity

Recruitment deviations, implementation error, and changes in parameters in the projection period of the OM can be added through the future_om_list input to run_SSMSE.

First, we’ll set up the list to add recruitment deviations in the projection period. The same recruitment deviation patterns are used across scenarios, but different patterns are use across iterations in the same scenario. We also want these deviations to have the same standard deviations as the historical deviations with 0 mean (the assumed default).

- +
# Start from a list created by a helper function
+template_mod_change <- create_future_om_list() 
+# add recruitment deviations
+rec_dev_specify <- template_mod_change[[1]]
+rec_dev_specify$pars <- "rec_devs" # apply change to rec devs
+rec_dev_specify$scen <- c("replicate", "all")
+# using 1 to 100 means the sd or mean will be calculated by taking the sd across years
+# from 1 to 100
+rec_dev_specify$input$first_yr_averaging <- 1
+rec_dev_specify$input$last_yr_averaging <- 100
+# The following 2 lines suggest that this change is immediately applied in year
+# 101, with no transitory period for using sd 0 to the new sd.
+rec_dev_specify$input$last_yr_orig_val <- 100
+rec_dev_specify$input$first_yr_final_val <- 101
+rec_dev_specify$input$ts_param <- "sd" # this change is for the sd
+# no input value needed since it will be calclated from the historical rec devs.
+rec_dev_specify$input$value <- NA
+rec_dev_specify
$pars
 [1] "rec_devs"
 
@@ -269,24 +290,22 @@ 
2.3 Adding process error through
 [1] "model_change"
 
 $input
-  first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param
-1                  1               100              100                101       sd
-    method value
-1 absolute    NA
+ first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value +1 1 100 100 101 sd absolute NA

Next, suppose we want to allow selectivity to vary annually for 1 selectivity parameter of the fishery throughout the projection period. The following specifies that the value for selectivity varies randomly around the base value with a sd of 0.2.

- +
# put together the change for selectivity (random values around the orig val, with
+# an sd of 0.2)
+mod_change_sel <- template_mod_change[[1]]
+mod_change_sel$scen[2] <- "all" # apply to all scenarios
+# The following 2 lines suggest that this change is immediately applied in year
+# 101, with no transitory period for using sd 0 to the new sd.
+# historical values are NA in this case, because they are not used to determine
+# the sd to use.
+mod_change_sel$input$last_yr_orig_val <- 100
+mod_change_sel$input$first_yr_final_val <- 101
+mod_change_sel$input$ts_param <- "sd" # this change is for the sd
+mod_change_sel$input$value <- 0.2 # se to use in the projection period
+mod_change_sel
$pars
 [1] "SizeSel_P_3_Fishery(1)"
 
@@ -297,21 +316,18 @@ 
2.3 Adding process error through
 [1] "model_change"
 
 $input
-  first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param
-1                 NA                NA              100                101       sd
-    method value
-1 absolute   0.2
+ first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value +1 NA NA 100 101 sd absolute 0.2

Finally, add these two changes together into an object to pass to run_SSMSE

- -
+
future_om_list_recdevs_sel <- list(rec_dev_specify, 
+                                   mod_change_sel)
+

2.3.1 Add observation error through sampling from OM

The argument sample_struct specifies the structure for sampling from the OM (and passing to the EM). The function create_sample_struct can be used to construct a simple sampling structure consistent with an input data file:

- -
Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1.
-Returning NA for Yr in this dataframe.
- +
datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE")
+sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning
+
Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this dataframe.
+
sample_struct_1_scen
$catch
    Yr Seas FltSvy    SE
 1 101    1      1 0.005
@@ -342,12 +358,12 @@ 
2.3.1 Add observation error throu
 
Note that length comp (lencomp) includes an NA value for year. This is because
 no consistent pattern was identified, so the user must define their own input.
 In this case, we will remove sampling length comps all together:

-
+
sample_struct_1_scen$lencomp <- NULL # don't use length sampling

 
The same sampling structure will be used for both scenarios, which we define in a list below:

-
+
sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen)
-
+

2.4 Examine the management procedure used

We will use the same management procedure for both scenarios:

    @@ -356,77 +372,74 @@

    2.4 Examine the management proced
  1. Put this projected catch (without implementation error, in the case of this example) back into the OM. Extend the OM forward in time to get the true values for the population.

Let’s take a look at step 2 in the management procedure, which is implemented using the forecasting module in SS. We will examine the forecast file for the estimation model to better understand how catches will be forecasted from the assessment. We will use the same management procedure in both of these scenarios, although for a full MSE analysis, it is likely that multiple management procedures would be compared.

- +
fore <- r4ss::SS_readforecast(
+  system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"),
+  verbose = FALSE)
+fore$Forecast 
[1] 3
- +
fore$Btarget
[1] 0.4

fore$Forecast = 3 means our forecasts from the assessment will use fishing mortality (F) to attmpt to achieve a relative (to unfished) spawning stock biomass. Based on fore$Btarget, the relative biomass target is 40% of unfished spawning stock biomass. Note also that the control rule fore$BforconstantF and fore$BfornoF values are set low to make it unlikely that they will be used (these parameters are used for a ramp harvest control rule, which we do not want to use here):

- +
fore$BforconstantF
[1] 0.03
- +
fore$BfornoF
[1] 0.01

Futhermore, fore$Flimitfraction is set to 1 so that the forecasted catch is set equal to the overfishing limit (for simplicity):

- +
fore$Flimitfraction
[1] 1

Note that the number of forecast years is 1:

- +
fore$Nforecastyrs
[1] 1

However, an assessment will be conducted every 3 years and thus 3 years of projections is required. SSMSE will automatically modify this value in the estimation model to the appropriate number of forecasting years.

More information on using the forecast module in SS to forecast catches is available in the Stock Synthesis users manual.

Users can also specify their own [custom management procedures]

-
+

2.5 Run SSMSE

Now, we create a directory to store our results, and use run_SSMSE to run the MSE analysis loop (note this will take some time to run, ~ 20 min):

- +
run_res_path <- file.path(run_SSMSE_dir, "results")
+dir.create(run_res_path)
+res <- run_SSMSE(
+    scen_name_vec = c("h-ctl", "h-1"),# name of the scenario
+    out_dir_scen_vec = run_res_path, # directory in which to run the scenario
+    iter_vec = c(5,5), # run with 5 iterations each
+    OM_name_vec = NULL, # specify directories instead
+    OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files
+    EM_name_vec = c("cod", "cod"), # cod is included in package data
+    MS_vec = c("EM","EM"), # The management strategy is specified in the EM
+    nyrs_vec = c(6, 6), # Years to project OM forward
+    nyrs_assess_vec = c(3, 3), # Years between assessments
+    future_om_list = future_om_list_recdevs_sel,
+    run_parallel = TRUE, # Run iterations in parallel
+    sample_struct_list = sample_struct_list_all, # How to sample data for running the EM.
+    sample_struct_hist_list = NULL, # because this is null, will just use sampling
+    # as in the current OM data file for the historical period.
+    seed = 12345) #Set a fixed integer seed that allows replication

See ?run_SSMSE for more details on function arguments. In a real MSE analysis, running 100+ iterations to reflect the full range of uncertainty (given observation and process errors) in the results would be preferred. However, we are only running 5 iterations per scenario in this demonstration to reduce computing time.

-
+

2.6 run_SSMSE output

run_SSMSE will create new folders in the folders specified in out_dir_scen_vec (note that in this case, we are running both scenarios in the same folder). After is complete, there will be a folder for each scenario in run_res_path (since out_dir_scen_vec = run_res_path in this example). Within each scenario is a folder for each scenario.

And within each scenario folder, there are folders containing the SS models that were run by run_SSMSE.

There should be 1 folder for the OM, which is run multiple times in this same folder during the MSE analysis. There are multiple folders for the EMs, as a new folder is created each time an assessment is done. The first run is the folder with a name ending in init; then, each assessment after is named for the updated end year of the model.

With many iterations, the number of files adds up; in the future, we hope to add options to save less output.

-
+

2.7 Performance metrics

Quantitative performance metrics should be specified before conducting an MSE. Typically, a suite of performance metrics will be examined; however, for simplicity in this example, we will only look at what the achieved relative biomass was for the last 3 years of projection in the MSE to determine how it compares to the intended management target of 40% of unfished Spawning Stock Biomass. Note that we are only running our MSE projection for 6 years, but longer projections are typical in MSE analyses.

-
+

2.8 Summarize results

The function SSMSE_summary_all can be used to summarize the model results in a list of 3 dataframes, one for scalar outputs (named scalar), one for timeseries outputs (ts), one for derived quantities (dq). This function also creates summary csv files in the folder where the results are stored.

- +
# Summarize 1 iteration of output
+summary <- SSMSE_summary_all(run_res_path)
+## Extracting results from 2 scenarios
+## Starting h-1 with 5 iterations
+## Starting h-ctl with 5 iterations

Plotting and data manipulation can then be done with these summaries. For example, SSB over time by model can be plotted. The models include the Operating Model (cod_OM), Estimation model (EM) for the historical period of years 0-100 (cod_EM_init), and the EM run with last year of data in year 103 (cod_EM_103). The operating models are shown in blue or black (depending on the scenario), and the estimation model runs are shown in orange, and the scenarios are shown on different subplots:

- -
Warning: package 'ggplot2' was built under R version 4.0.5
- -
Warning: package 'tidyr' was built under R version 4.0.5
- -
Warning: package 'dplyr' was built under R version 4.0.5
+
library(ggplot2) # use install.packages("ggplot2") to install package if needed
+library(tidyr) # use install.packages("tidyr") to install package if needed
+library(dplyr) # use install.packages("dplyr") to install package if needed

 Attaching package: 'dplyr'
The following objects are masked from 'package:stats':
@@ -436,103 +449,102 @@ 
2.8 Summarize results

 
     intersect, setdiff, setequal, union
-
+

2.9 Simple Convergence Check

Check there are no params on bounds or SSB that is way too small or way too large

- +
check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) {
+  require(dplyr) # note: not the best way to do this
+  if(any(!is.na(summary$scalar$params_on_bound))) {
+    warning("Params on bounds")
+  } else {
+    message("No params on bounds")
+  }
+  summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM")
+  calc_SSB <- summary$ts %>%
+    filter(year >= min_yr & year <= max_yr) %>% 
+    select(iteration, scenario, year, model_run, model_type, SpawnBio)
+  OM_vals <- calc_SSB %>% 
+              filter(model_type == "OM") %>% 
+              rename(SpawnBio_OM = SpawnBio ) %>% 
+              select(iteration, scenario, year, SpawnBio_OM)
+  EM_vals <- calc_SSB %>% 
+               filter(model_type == "EM") %>% 
+               rename(SpawnBio_EM = SpawnBio) %>% 
+               select(iteration, scenario, year, model_run, SpawnBio_EM)
+  bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% 
+                  mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM)
+  filter_SSB <- bind_vals %>% 
+    filter(SSB_ratio > 2 | SSB_ratio < 0.5)
+  if(nrow(filter_SSB) > 0 ) {
+    warning("Some large/small SSBs relative to OM")
+  } else {
+    message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM")
+  }
+  return_val <- bind_vals
+}
+values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5)
No params on bounds
All SSBs in EM are no greater than double and no less than half SSB vals in the OM
-
+

2.10 Plot Spawning Stock Biomass (SSB)

This plot shows that SSB estimated does not match perfectly with the operating model. A similar plot could be made for any parameter of interest.

- -
Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> =
-"none")` instead.
+
# plot SSB by year and model run
+ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), 
+                ggplot2::aes(x = year, y = SpawnBio)) +
+  ggplot2::geom_vline(xintercept = 100, color = "gray") +
+  ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+
+  ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) +
+  ggplot2::scale_linetype_manual(values = rep("solid", 50)) +
+  ggplot2::guides(linetype = FALSE) +
+  ggplot2::facet_wrap(. ~ scenario) +
+  ggplot2::theme_classic()
+
Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> = "none")` instead.

Now, we calculate and plot the performance metric, which is average spawning stock biomass (SSB) from years 104 to 106.

- +
# get_SSB_avg calculates the SSB in each year for each
+# iteration of the operating model, then takes the average over the years from
+# min_yr, to max_year. It uses the summary object as input to do these
+# calculations.
+get_SSB_avg <- function(summary, min_yr, max_yr) {
+  OM_vals <- unique(summary$ts$model_run)
+  OM_vals <- grep("_OM$", OM_vals, value = TRUE)
+  SSB_yr <- summary$ts %>% 
+          filter(year >= min_yr & year <= max_yr) %>% 
+          filter(model_run %in% OM_vals) %>% 
+          select(iteration, scenario, year, SpawnBio) %>% 
+          group_by(iteration, scenario) %>% 
+          summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% 
+          ungroup()
+  SSB_yr
+}
+avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106)
+
+# function to summarize data in plot
+data_summary <- function(x) {
+  m <- mean(x)
+  ymin <- m - sd(x)
+  ymax <- m + sd(x)
+  return(c(y = m, ymin = ymin, ymax = ymax))
+}
+# Now, plot the average relative spawning stock biomass for years 104 - 106
+ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) +
+  stat_summary(fun.data = data_summary, 
+               position = position_dodge(width = 0.9), color = "blue") +
+  labs(title = "Long-term average  SSB\n(years 104-106)", 
+       x = "Scenario", y = "SSB") +
+  theme_classic()

From the above plot, we see differences in the average SSb between the 2 scenarios.

-
+

2.11 Example MSE Results

We can see from the performance metric that mis-specifying the value of steepness will results in higher realized relative spawning stock biomass than correctly specifying it. This gives us some idea of the consequences of misspecifying steepness in the stock assessment.

-
+

2.12 Delete the files

If you wish to delete the files created from this example, you can use:

- +
unlink(run_SSMSE_dir, recursive = TRUE)
@@ -545,7 +557,6 @@

2.12 Delete the files

- @@ -564,6 +575,7 @@

2.12 Delete the files

"weibo": false, "instapaper": false, "vk": false, +"whatsapp": false, "all": ["facebook", "twitter", "linkedin", "weibo", "instapaper"] }, "fontsettings": { @@ -584,6 +596,10 @@

2.12 Delete the files

"text": null }, "download": null, +"search": { +"engine": "fuse", +"options": null +}, "toc": { "collapse": "subsection" } diff --git a/inst/bookdown/06-glossary.Rmd b/inst/bookdown/06-glossary.Rmd new file mode 100644 index 00000000..0c8f5fd5 --- /dev/null +++ b/inst/bookdown/06-glossary.Rmd @@ -0,0 +1,21 @@ +# Glossary of MSE terms + +This glossary is based off of [Rademeyer et al. (2007)](https://doi.org/10.1093/icesjms/fsm050) and [Punt et al. (2016)](https://doi.org/10.1111/faf.12104). + +* **Assessment error** - Error that occurs during the process of conducting an assessment, specifically error "which inform the catch control rule that is being evaluated using the MSE: management advice for any system is based on uncertain data. Consequently, the data that inform catch control rules need to be generated in a manner which is as realistic as possible. Uncertainty arises when the model used for conducting assessments and providing management advice differs from the operating model, or the data are too noisy to estimate all key parameters reliably [(Punt et al. 2016)](https://doi.org/10.1111/faf.12104)." +* **Assessment model (AM)** - A fitted and well scrutinized Stock Synthesis stock assessment model. The AM is input for SSMSE so that it can build the operating model. +* **Bootstrapped dataset** - After running an SS model (and if the user turns on the option in the starter file), the output file data.ss_new contains a "dataset" that has the same form as the input data.ss file, but instead of the input values, contains a sampled dataset. This is derived from using the expected values given the parameter estimates and model structure (see glossary entry on **Expected values**), uncertainty either estimated or input in the model, and an assumed distribution for sampling. As many bootstrapped data sets as the user desires can be output from an SS model. These "bootstrap datasets" are the third or greater set of values in the data.ss_new file. +* **Estimation model (EM)** - This refers to the model used within the MSE procedure to represent the stock assessment process. Note that an estimation model proxy could also be used to represent the stock assessment process. +* **Expected values** - After running an SS model (and if the user turns on teh option in the starter file), the output file data.ss_new contains a "dataset" that has the same form as the input data.ss file, but instead of the input values, contains the expected values given the parameter estimates and model structure. This "expected values dataset" is the second set of values in the data.ss_new file. +* **Implementation error** - Also called implementation uncertainty or outcome uncertainty. Broadly, this includes error of implementing the management action(s), which is often not done perfectly. Definition from [Punt et al. (2016)](https://doi.org/10.1111/faf.12104): "The most obvious form of this type of uncertainty is when catches are +not the same as the TACs – typically more is taken or the decision-makers do not implement the TACs suggested by the management strategy. However, there are many other sources of outcome uncertainty, such as that associated with catch limits set for recreational fisheries and regulating discards." +* **Index** - An index of abundance. In SS files, this is sometimes used interchangably with CPUE. Plural, indices. +* **Management strategy** - Following [Punt et al. (2016)](https://doi.org/10.1111/faf.12104), there are 2 types: **model-based management strategies** and **empirical management strategies**. Ideally, SSMSE will allow for users to use either of these management strategies. Model-based management strategies include conducting a stock assessment and using output for determining harvest control rules. Empirical management strategies do not involve conducting a stock assessment but rather setting regulations from data (although summarization is possible). In some cases, management strategies are computationally intensive/time consuming to formally include in the context of an MSE; to speed up the process, it is commen to use **management strategy proxies**, typically an assumed error distribution about the operating model values. Management strategies are also known as management procedures. +* **Model uncertainty** - Definition from [Punt et al. (2016)](https://doi.org/10.1111/faf.12104): "the form of relationships within an operating model will always be subject to uncertainty. The simplest type of model uncertainty involves, for example, whether the stock–recruitment relationship is Beverton–Holt or Ricker, whether a fixed value for a model parameter is correct, or whether fishery selectivity is asymptotic or dome-shaped. However, there are other more complicated types of model structure uncertainty such as how many stocks are present in the area modelled, the error structure of the data used for assessment purposes, the impact of future climate change on biological relationships such as the stock–recruitment function, and ecosystem impacts on biological and fishery processes." +* **Observation error** - Error that results from not observing the true dynamics of the system. See also **bootstrapped dataset**. +* **Operating model (OM)** - This is a Stock Synthesis model that defines the assumed true dynamics of the population and its associated fisheries for the purposes of management strategy evaluation. Most often more than one operating model is necessary in order to adequately characterize the uncertainty in the true dynamics of the system. The Stock Synthesis OM may also define how sampling from the true dynamics is done, as SS produces expected values and, if desired, bootstrapped data sets based on sampling assumptions implicit in the model. For more information, see the glossary entries on **Expected values** and **bootstrapped dataset** +* **Parameter uncertainty** - Definition from [Punt et al. (2016)](https://doi.org/10.1111/faf.12104): "many operating models are fit to the data available, but the values estimated for the parameters of those operating models (e.g. fishery selectivity-at-age, the parameters of the stock–recruitment relationship and historical deviations in recruitment about the stock–recruitment relationship) are subject to error." +* **Performance statistics** - Definition from [Rademeyer et al. (2007)](https://doi.org/10.1093/icesjms/fsm050): "Statistics that summarize different aspects of the results of a simulation trial used to evaluate how well a specific [management strategy] achieves some or all of the general objectives for management for a particular scenario." Performance statistics usually fall into one of three categories: **catch-related**, **stability related**, or **risk related**. +* **Process uncertainty** - Definition from [Punt et al. (2016)](https://doi.org/10.1111/faf.12104): "variation (usually assumed to be random, though sometimes incorporating autocorrelation) in parameters often considered fixed in stock assessments such as natural mortality, future recruitment about a stock–recruitment relationship and selectivity." +* **Stock Synthesis (SS, SS3)** - A integrative, general population dynamics modeling program used to assess the effects of fishing on population. Available at the [Stock Synthesis website](https://vlab.ncep.noaa.gov/web/stock-synthesis/home) +* **Uncertainty** - Incorporating uncertainty into an MSE procedure is extremely important. There are several potential sources of uncertainty, which we have divided as done in [Punt et al. (2016)](https://doi.org/10.1111/faf.12104) in addition to adding **observation error**. For more details, see the glossary entries on **Process Uncertainty**, **Parameter uncertainty**, **Model uncertainty**, **Assessment error**, **Implementation error**, and observation error. [Punt et al. (2016)](https://doi.org/10.1111/faf.12104) suggest that MSE should at least consider process uncertainty (particularly deviations from the stock recruitment relationship), parameter uncertainty (particularly which relates to productivity and stock size), and observation error. *Note that [Rademeyer et al. (2007)](https://doi.org/10.1093/icesjms/fsm050) divides error into estimation error, implementation error, observation error and process error, which could be used instead. These may be more natural divisions in sources of error.* \ No newline at end of file diff --git a/inst/bookdown/README.md b/inst/bookdown/README.md new file mode 100644 index 00000000..129c9653 --- /dev/null +++ b/inst/bookdown/README.md @@ -0,0 +1,3 @@ +# How to update and render the user manual + +See [notes in the wiki](https://github.com/nmfs-fish-tools/SSMSE/wiki/Developers#notes-on-how-to-edit-and-update-the-bookdown-user-manual). From 40802582e40f7a732e8c372561e9587d7c79d4fc Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Tue, 21 Sep 2021 16:51:51 -0400 Subject: [PATCH 02/10] usx, #60: make sure output vectors exist --- R/runSSMSE.R | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/R/runSSMSE.R b/R/runSSMSE.R index 307dfa90..bfb945a6 100644 --- a/R/runSSMSE.R +++ b/R/runSSMSE.R @@ -176,6 +176,11 @@ run_SSMSE <- function(scen_name_vec, ) } } + + + # make sure the output directories exist + result <- lapply(out_dir_scen_vec, function(x) if(!dir.exists(x)) dir.create(x, showWarnings = FALSE)) + # check and add implicit inputs to the future_om_list future_om_list <- check_future_om_list_str(future_om_list = future_om_list) # Note that all input checks are done in the check_scen_list function. From e16dea09d8965f547777fac19d47f27bcbbd5e1a Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Mon, 4 Oct 2021 12:40:56 -0400 Subject: [PATCH 03/10] #46: add warnings about Empirical W at age --- R/initOM.R | 3 +++ R/parse_MS.R | 4 ++++ inst/bookdown/03-runSSMSE.Rmd | 6 +++--- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/R/initOM.R b/R/initOM.R index f86557d9..962a2f91 100644 --- a/R/initOM.R +++ b/R/initOM.R @@ -92,6 +92,9 @@ create_OM <- function(OM_out_dir, datsource = dat, ctlsource = ctl, verbose = FALSE ) + if(ctl[["EmpiricalWAA"]] == 1) { + stop("OM is an empirical weight at age model, which is not yet possible to use.") + } # model checks ---- if (ctl[["F_Method"]] == 1) { stop( diff --git a/R/parse_MS.R b/R/parse_MS.R index f0375443..73fe5c82 100644 --- a/R/parse_MS.R +++ b/R/parse_MS.R @@ -504,6 +504,10 @@ parse_MS <- function(MS, EM_out_dir = NULL, EM_init_dir = NULL, do_checks = TRUE, verbose = verbose ) + ctl <- SS_readctl(start[["ctlfile"]], datlist = new_EM_dat) + if(ctl[["EmpiricalWAA"]] == 1) { + stop("EM uses empirical weight at age, which is not yet possible to use.") + } } else { if (!is.null(sample_struct)) { sample_struct_sub <- lapply(sample_struct, diff --git a/inst/bookdown/03-runSSMSE.Rmd b/inst/bookdown/03-runSSMSE.Rmd index 1bc468c0..d4afa214 100644 --- a/inst/bookdown/03-runSSMSE.Rmd +++ b/inst/bookdown/03-runSSMSE.Rmd @@ -8,20 +8,20 @@ Note that multiple scenarios can be called in `run_SSMSE()`, often through vecto ## Operating model -The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set `OM_name_vec = "cod"`. Otherwise, the path to the OM model should be specified in `OM_in_dir_vec`. +The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model, except for models that use Empirical Weight at age. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set `OM_name_vec = "cod"`. Otherwise, the path to the OM model should be specified in `OM_in_dir_vec`. ## The Management Strategy/procedure and (if applicable) Estimation model (EM) The management strategy (and EM) can be specified in one of two ways: -1. Using an SS model +1. Using an SS model and its forcast file to project catch by fleet 2. Using a custom procedure via a function in R In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet. ### Specify the Management Strategy in a SS model -An SS model can be set up as the EM for the MSE. To use this option, specify `"EM"` as part of `MS_vec`. As with the OM, the built-in cod model could be used; just specify `"cod"` in the `EM_name_vec`. To use any other SS model as the EM, specify the path in `EM_in_dir_vec`. +An SS model can be set up as the EM for the MSE. To use this option, specify `"EM"` as part of `MS_vec`. As with the OM, the built-in cod model could be used; just specify `"cod"` in the `EM_name_vec`. To use any other SS model as the EM, specify the path in `EM_in_dir_vec`.Note that models that use empirical weight at age can not yet be used as estimation models. Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user. From c410ebcff277399c0af0a35221a0aead94e24975 Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Wed, 6 Oct 2021 15:26:10 -0400 Subject: [PATCH 04/10] #63: document which data types can be used --- inst/bookdown/03-runSSMSE.Rmd | 2 ++ 1 file changed, 2 insertions(+) diff --git a/inst/bookdown/03-runSSMSE.Rmd b/inst/bookdown/03-runSSMSE.Rmd index d4afa214..c495624e 100644 --- a/inst/bookdown/03-runSSMSE.Rmd +++ b/inst/bookdown/03-runSSMSE.Rmd @@ -87,6 +87,8 @@ Users also need to specify how and which data types should be sampled for each f The helper function `create_sample_struct()` can be used to help users generate the list of dataframes for a scenario. See an example of this function's use in the [simple example](#sample) or by typing `?SSMSE::create_sample_struct()` into the R console. +Currently, users can sample data for catches (treated as fixed), CPUE (i.e., indices of abundance), length and age composition, conditional length at age compositions, mean size at age, and mean size. It is not yet possible to sample data for generalized size compositions, tagging data, and morph compositions. + Users can specify the sampling during the historical period of the model through the `sample_struct_hist` input to `run_SSMSE`. This is an optional input and has the same structure as `sample_struct_list`, but the years will be before the original end year of the OM. ## Future changes to the operating model (#future) From 4c2f28e77d9e32c3e5bb74dd25aae908efc5169b Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Wed, 6 Oct 2021 16:34:54 -0400 Subject: [PATCH 05/10] fix: path --- R/parse_MS.R | 3 ++- tests/testthat/test-runSSMSE.R | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/R/parse_MS.R b/R/parse_MS.R index 73fe5c82..6a7a6e32 100644 --- a/R/parse_MS.R +++ b/R/parse_MS.R @@ -504,7 +504,8 @@ parse_MS <- function(MS, EM_out_dir = NULL, EM_init_dir = NULL, do_checks = TRUE, verbose = verbose ) - ctl <- SS_readctl(start[["ctlfile"]], datlist = new_EM_dat) + ctl <- SS_readctl(file.path(EM_out_dir, start[["ctlfile"]]), + datlist = new_EM_dat) if(ctl[["EmpiricalWAA"]] == 1) { stop("EM uses empirical weight at age, which is not yet possible to use.") } diff --git a/tests/testthat/test-runSSMSE.R b/tests/testthat/test-runSSMSE.R index a828f536..bec4acce 100644 --- a/tests/testthat/test-runSSMSE.R +++ b/tests/testthat/test-runSSMSE.R @@ -81,7 +81,6 @@ test_that("run_SSMSE runs with an EM, and works with summary funs", { test_that("run_SSMSE runs multiple iterations/scenarios and works with summary funs", { # This tests takes a while to run, but is really helpful. new_temp_path <- file.path(temp_path, "mult_scenarios") - dir.create(new_temp_path) skip_on_cran() nyrs <- 6 datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") From 1907e892b7c3b036c27ec42d30051accb987f13b Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Wed, 6 Oct 2021 16:52:35 -0400 Subject: [PATCH 06/10] bump version --- DESCRIPTION | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DESCRIPTION b/DESCRIPTION index 97047c03..60eb7114 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -1,7 +1,7 @@ Package: SSMSE Type: Package Title: Management Strategy Evaluation (MSE) using Stock Synthesis (SS) -Version: 0.2.0 +Version: 0.2.1 Authors@R: c( person("Kathryn", "Doering", email = "kathryn.doering@noaa.gov", role = c("aut", "cre")), From e5597f29fd15d3ed33ac93b8591f2785a6ff92c7 Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Wed, 6 Oct 2021 16:53:39 -0400 Subject: [PATCH 07/10] docs: run devtools::document() --- DESCRIPTION | 2 +- NAMESPACE | 6 +----- man/calc_par_trend.Rd | 2 +- man/check_future_om_list_str.Rd | 2 +- man/check_future_om_list_vals.Rd | 2 +- man/convert_future_om_list_to_devs_df.Rd | 12 ++++++------ man/create_OM.Rd | 2 +- man/create_future_om_list.Rd | 6 +++--- man/create_scen_list.Rd | 4 ++-- man/develop_OMs.Rd | 3 ++- man/get_performance_metrics.Rd | 2 +- man/plot_comp_sampling.Rd | 2 +- man/plot_index_sampling.Rd | 2 +- man/rm_vals.Rd | 2 +- man/run_SSMSE.Rd | 6 +++--- man/run_SSMSE_iter.Rd | 6 +++--- man/run_SSMSE_scen.Rd | 6 +++--- man/sample_vals.Rd | 6 +++--- man/update_OM.Rd | 6 +++--- 19 files changed, 38 insertions(+), 41 deletions(-) diff --git a/DESCRIPTION b/DESCRIPTION index 60eb7114..71ff42a0 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -19,7 +19,7 @@ URL: https://github.com/nmfs-fish-tools/SSMSE BugReports: https://github.com/nmfs-fish-tools/SSMSE/issues Encoding: UTF-8 LazyData: true -RoxygenNote: 7.1.1 +RoxygenNote: 7.1.2 Imports: stats, utils, diff --git a/NAMESPACE b/NAMESPACE index f4b22a80..5367b450 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -1,20 +1,16 @@ # Generated by roxygen2: do not edit by hand export(SSMSE_summary_all) -export(SSMSE_summary_iter) -export(SSMSE_summary_scen) export(create_future_om_list) export(create_sample_struct) -export(create_scen_list) export(develop_OMs) export(get_bin) -export(get_input_value) +export(parse_MS) export(plot_comp_sampling) export(plot_index_sampling) export(run_EM) export(run_SSMSE) export(run_ss_model) -export(set_MSE_seeds) import(ggplot2) import(parallel) import(r4ss) diff --git a/man/calc_par_trend.Rd b/man/calc_par_trend.Rd index d0494832..f7007080 100644 --- a/man/calc_par_trend.Rd +++ b/man/calc_par_trend.Rd @@ -24,7 +24,7 @@ calc_par_trend( \item{ref_parm_value}{This is the historic parameter that the end trend value. Can be NA if the there is no line in val_info for the given parameter} -\item{vals_df}{The dataframe of the parameter values by year. Use to get +\item{vals_df}{The dataframe of the parameter values by year. Use to get start val and last year} \item{parname}{Name of the parameter with devs from the SS model. diff --git a/man/check_future_om_list_str.Rd b/man/check_future_om_list_str.Rd index 37157d43..2b0de3ee 100644 --- a/man/check_future_om_list_str.Rd +++ b/man/check_future_om_list_str.Rd @@ -13,7 +13,7 @@ check_future_om_list_str(future_om_list) The future_om_list with implicit arguments made explicit } \description{ -Checks that a future OM list is valid. If any values are implicit, +Checks that a future OM list is valid. If any values are implicit, then add these values. Does not check against arguments in the scenario, just the generic structure } diff --git a/man/check_future_om_list_vals.Rd b/man/check_future_om_list_vals.Rd index f8498af0..98f44ca8 100644 --- a/man/check_future_om_list_vals.Rd +++ b/man/check_future_om_list_vals.Rd @@ -9,7 +9,7 @@ check_future_om_list_vals(future_om_list, scen_list) \arguments{ \item{future_om_list}{The future_om_list object to check.} -\item{scen_list}{The list object of scenarios specifying inputs, typically +\item{scen_list}{The list object of scenarios specifying inputs, typically passed to or created by run_SSMSE.} } \value{ diff --git a/man/convert_future_om_list_to_devs_df.Rd b/man/convert_future_om_list_to_devs_df.Rd index 374dc893..dc7f19d1 100644 --- a/man/convert_future_om_list_to_devs_df.Rd +++ b/man/convert_future_om_list_to_devs_df.Rd @@ -25,19 +25,19 @@ the operating model as it is extended forward in time} \item{nyrs}{The total number of years that the model will be extended forward.} -\item{global_seed}{A global seed to set, then pull new seeds from. Defaults +\item{global_seed}{A global seed to set, then pull new seeds from. Defaults to 123.} } \value{ -A list including 3 dataframes and one list: devs_df, the additive deviations - relative to the base values; base_df, the base values of the parameter with - deviations; abs_df, the absolute future values by year (first col) and +A list including 3 dataframes and one list: devs_df, the additive deviations + relative to the base values; base_df, the base values of the parameter with + deviations; abs_df, the absolute future values by year (first col) and parameter (parameterss in different cols). Also includes a modified version of the future_om_list which includes the seed applied to each list component (note that this is not the ultimate seed used for sampling, as addiitonal) - seeds are generated from this seed based on the scenario, iteration, and + seeds are generated from this seed based on the scenario, iteration, and option for randomness (replicate across scenarios or randomize across scenarios). - Note that no OM files are modified or created as part of this function + Note that no OM files are modified or created as part of this function (i.e., it does not have side effects). } \description{ diff --git a/man/create_OM.Rd b/man/create_OM.Rd index 1e350224..862ab937 100644 --- a/man/create_OM.Rd +++ b/man/create_OM.Rd @@ -49,7 +49,7 @@ recruitment deviations, and implementation errors.} \item{verify_OM}{Should the model be run without estimation and some basic checks done to verify that the OM can run? Defaults to TRUE.} -\item{sample_struct_hist}{The historical sample structure, if specified by +\item{sample_struct_hist}{The historical sample structure, if specified by the user. Defaults to NULL, which means to use the same sample structure as the historical data.} diff --git a/man/create_future_om_list.Rd b/man/create_future_om_list.Rd index c11adbd7..4d1ce92d 100644 --- a/man/create_future_om_list.Rd +++ b/man/create_future_om_list.Rd @@ -18,11 +18,11 @@ For now, just replicates the same list.} } \description{ The future_om_list objects specify changes to make in the future to the -OM as the OM is extended forward in time. In particular, this function helps -users create these objects. For now, just returns examples based on cod +OM as the OM is extended forward in time. In particular, this function helps +users create these objects. For now, just returns examples based on cod model that comes with SSMSE. } \examples{ -example_future_om_list <- +example_future_om_list <- create_future_om_list(example_type = "custom", list_length = 2) } diff --git a/man/create_scen_list.Rd b/man/create_scen_list.Rd index 52c43891..49817bf4 100644 --- a/man/create_scen_list.Rd +++ b/man/create_scen_list.Rd @@ -74,8 +74,8 @@ If NULL, the data structure will try to be infered from the pattern found for each of the datatypes within the EM datafiles. Include this strucutre for the number of years to extend the model out.} -\item{sample_struct_hist_list}{An optional list of lists including which -years should be sampled for the historical period for the data generated +\item{sample_struct_hist_list}{An optional list of lists including which +years should be sampled for the historical period for the data generated from the OM. If this is left as NULL, then the same sampling scheme will be used as in the OM's data file. If it is not NULL, then only the data specified in the list will be sampled.} diff --git a/man/develop_OMs.Rd b/man/develop_OMs.Rd index 53759c8b..a2f7fb24 100644 --- a/man/develop_OMs.Rd +++ b/man/develop_OMs.Rd @@ -16,7 +16,8 @@ develop_OMs( } \arguments{ \item{OM_name}{Name of the original model, as in the SSMSE package. If not -using a model in the package, please specify its path in \code{OM_in_dir}} +using a model in the package, please specify its path in \code{OM_in_dir}. +Can be left NULL if specifying OM_in_dir.} \item{OM_in_dir}{Path to the original operating model. If using a model in the SSMSE package, please specify its name in \code{OM_name} instead.} diff --git a/man/get_performance_metrics.Rd b/man/get_performance_metrics.Rd index d640b672..ff241009 100644 --- a/man/get_performance_metrics.Rd +++ b/man/get_performance_metrics.Rd @@ -11,7 +11,7 @@ get_performance_metrics( ) } \arguments{ -\item{dir}{Path to the directory containing the scenarios, either relative +\item{dir}{Path to the directory containing the scenarios, either relative or absolute. Defaults to the working directory.} \item{use_SSMSE_summary_all}{If it exists, should the summmary files generated diff --git a/man/plot_comp_sampling.Rd b/man/plot_comp_sampling.Rd index 67d91e3c..04091756 100644 --- a/man/plot_comp_sampling.Rd +++ b/man/plot_comp_sampling.Rd @@ -17,7 +17,7 @@ A list containing 2 components: 1) the ggplot object and 2) the dataframe used to make the ggplot object } \description{ -Creates a plot that can be used to see how sampling lines up with +Creates a plot that can be used to see how sampling lines up with data and expected values for the index of abundance } \author{ diff --git a/man/plot_index_sampling.Rd b/man/plot_index_sampling.Rd index b8b18907..79788432 100644 --- a/man/plot_index_sampling.Rd +++ b/man/plot_index_sampling.Rd @@ -15,7 +15,7 @@ A list containing 2 components: 1) the ggplot object and 2) the dataframe used to make the ggplot object } \description{ -Creates a plot that can be used to see how sampling lines up with +Creates a plot that can be used to see how sampling lines up with data and expected values for the index of abundance } \author{ diff --git a/man/rm_vals.Rd b/man/rm_vals.Rd index 5d303ba9..3ecd3b73 100644 --- a/man/rm_vals.Rd +++ b/man/rm_vals.Rd @@ -26,7 +26,7 @@ return_obj[[name_in_obj]], modified to only include elements present in compare_obj[[name_in_obj]]. } \description{ -From 2 list components with the same name, remove vals that aren't in the +From 2 list components with the same name, remove vals that aren't in the compare object } \author{ diff --git a/man/run_SSMSE.Rd b/man/run_SSMSE.Rd index 7967633f..6daf6a44 100644 --- a/man/run_SSMSE.Rd +++ b/man/run_SSMSE.Rd @@ -71,8 +71,8 @@ enviroment and specified by name in MS. For example, if the function is called "my_ms" then the user should make it available in the global environment and specify "my_ms" as a component of MS_vec.} -\item{custom_MS_source}{A file location with the source code for any custom MS -models to be used in the simulation. SSMSE will source this file which should +\item{custom_MS_source}{A file location with the source code for any custom MS +models to be used in the simulation. SSMSE will source this file which should contain a function/s whose name/s match each custom MS models included in MS_vec.} \item{use_SS_boot_vec}{Should a bootstrapped data set generated by SS be used? @@ -101,7 +101,7 @@ that the model will be extended forward in time assuming the original model structure.} \item{sample_struct_hist_list}{An optional list of lists including which years should be -sampled for the historical period for the data generated from the OM. If +sampled for the historical period for the data generated from the OM. If this is left as NULL, then the same sampling scheme will be used as in the OM's data file. If it is not NULL, then each year} diff --git a/man/run_SSMSE_iter.Rd b/man/run_SSMSE_iter.Rd index 7f473362..5f207520 100644 --- a/man/run_SSMSE_iter.Rd +++ b/man/run_SSMSE_iter.Rd @@ -65,8 +65,8 @@ enviroment and specified by name in MS. For example, if the function is called "my_ms" then the user should make it available in the global environment and specify MS = "my_ms".} -\item{custom_MS_source}{A file location with the source code for any custom MS -models to be used in the simulation. SSMSE will source this file which should +\item{custom_MS_source}{A file location with the source code for any custom MS +models to be used in the simulation. SSMSE will source this file which should contain a function/s whose name/s match each custom MS models included in MS_vec.} \item{use_SS_boot}{Should a bootstrapped data set generated by SS be used? @@ -113,7 +113,7 @@ that the model will be extended forward in time assuming the original model structure.} \item{sample_struct_hist}{An optional list including which years should be -sampled for the historical period for the data generated from the OM. If +sampled for the historical period for the data generated from the OM. If this is left as NULL, then the same sampling scheme will be used as in the OM's data file. If it is not NULL, then each year} diff --git a/man/run_SSMSE_scen.Rd b/man/run_SSMSE_scen.Rd index d623983f..783dfba7 100644 --- a/man/run_SSMSE_scen.Rd +++ b/man/run_SSMSE_scen.Rd @@ -73,8 +73,8 @@ enviroment and specified by name in MS. For example, if the function is called "my_ms" then the user should make it available in the global environment and specify MS = "my_ms".} -\item{custom_MS_source}{A file location with the source code for any custom MS -models to be used in the simulation. SSMSE will source this file which should +\item{custom_MS_source}{A file location with the source code for any custom MS +models to be used in the simulation. SSMSE will source this file which should contain a function/s whose name/s match each custom MS models included in MS_vec.} \item{use_SS_boot}{Should a bootstrapped data set generated by SS be used? @@ -105,7 +105,7 @@ that the model will be extended forward in time assuming the original model structure.} \item{sample_struct_hist}{An optional list including which years should be -sampled for the historical period for the data generated from the OM. If +sampled for the historical period for the data generated from the OM. If this is left as NULL, then the same sampling scheme will be used as in the OM's data file. If it is not NULL, then each year} diff --git a/man/sample_vals.Rd b/man/sample_vals.Rd index 3367b753..d88943bb 100644 --- a/man/sample_vals.Rd +++ b/man/sample_vals.Rd @@ -2,7 +2,7 @@ % Please edit documentation in R/initOM_create_devs_list.R \name{sample_vals} \alias{sample_vals} -\title{Sample vals from normal random, lognormal random, or modified AR-1 +\title{Sample vals from normal random, lognormal random, or modified AR-1 process.} \usage{ sample_vals(mean, sd, ar_1_phi = 0, ndevs, dist = c("normal", "lognormal")) @@ -12,7 +12,7 @@ sample_vals(mean, sd, ar_1_phi = 0, ndevs, dist = c("normal", "lognormal")) \item{sd}{A single value or vector of sd parameter} -\item{ar_1_phi}{The phi (coefficient) value for an ar 1 process. Should be +\item{ar_1_phi}{The phi (coefficient) value for an ar 1 process. Should be between -1 and 1. 0 means an AR 1 process will NOT be used. 1 indicates a random walk. A single value or vector.} @@ -21,7 +21,7 @@ random walk. A single value or vector.} \item{dist}{The distribution to sample from.} } \description{ -Sample vals from normal random, lognormal random, or modified AR-1 +Sample vals from normal random, lognormal random, or modified AR-1 process. } \author{ diff --git a/man/update_OM.Rd b/man/update_OM.Rd index 5d5d6179..1b9e6bea 100644 --- a/man/update_OM.Rd +++ b/man/update_OM.Rd @@ -32,13 +32,13 @@ a corresponding catch the OM will assume effort based management an use harvest with implementation error added.} \item{catch_basis}{data frame with columns year, seas, fleet, basis that specifies if catch -should reference retained biomass (1) or dead biomass (2). Any year/season/fleet not listed will assume +should reference retained biomass (1) or dead biomass (2). Any year/season/fleet not listed will assume a value of 1 referencing retained biomass. Entering -99 for any of year, season, or fleet will apply the basis across all values of that variable (i.e. a single row with -99, -99, -99, 1 would implement retained biomass for all cases)} -\item{F_limit}{data frame with columns year, fleet, season, limit that specifies a maximum F -allowed in the OM or a negative value to specify a multiple of the historic maximum F. Any year/season/fleet +\item{F_limit}{data frame with columns year, fleet, season, limit that specifies a maximum F +allowed in the OM or a negative value to specify a multiple of the historic maximum F. Any year/season/fleet not listed will assume a value of 1.5. Entering -99 for any of year, season, or fleet will apply the limit across all values of that variable (i.e. a single row with -99, -99, -99, -2 would implement a cap of twice the historic maximum F for all cases)} From 5001dcc12ec89207c0b682ee3f4c6345ae718205 Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Wed, 6 Oct 2021 16:59:11 -0400 Subject: [PATCH 08/10] docs: build bookdown --- docs/manual/404.html | 2 +- docs/manual/SSMSE.html | 181 +++++----- ...a-custom-management-strategyprocedure.html | 64 ++-- docs/manual/glossary-of-mse-terms.html | 2 +- docs/manual/helper.html | 2 +- docs/manual/index.html | 9 +- docs/manual/intro.html | 2 +- docs/manual/output.html | 8 +- ...otting-output-and-performance-metrics.html | 2 +- docs/manual/search_index.json | 2 +- docs/manual/simple.html | 316 +++++++++--------- 11 files changed, 295 insertions(+), 295 deletions(-) diff --git a/docs/manual/404.html b/docs/manual/404.html index 2f56328e..e1d9ebf9 100644 --- a/docs/manual/404.html +++ b/docs/manual/404.html @@ -24,7 +24,7 @@ - + diff --git a/docs/manual/SSMSE.html b/docs/manual/SSMSE.html index 8920fa0f..377c9ca3 100644 --- a/docs/manual/SSMSE.html +++ b/docs/manual/SSMSE.html @@ -24,7 +24,7 @@ - + @@ -232,66 +232,66 @@

4.1 Scenarios in SSMSE

4.2 Operating model

-

The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = "cod". Otherwise, the path to the OM model should be specified in OM_in_dir_vec.

+

The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model, except for models that use Empirical Weight at age. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = "cod". Otherwise, the path to the OM model should be specified in OM_in_dir_vec.

4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM)

The management strategy (and EM) can be specified in one of two ways:

    -
  1. Using an SS model
    2. +
  2. Using an SS model and its forcast file to project catch by fleet
  3. Using a custom procedure via a function in R

In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet.

4.3.1 Specify the Management Strategy in a SS model

-

An SS model can be set up as the EM for the MSE. To use this option, specify "EM" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify "cod" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec.

+

An SS model can be set up as the EM for the MSE. To use this option, specify "EM" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify "cod" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec.Note that models that use empirical weight at age can not yet be used as estimation models.

Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user.

4.3.2 Using a custom management strategy/procedure

Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year:

-
constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, 
-                              frac_catch = 0.5, ...) { # need to include ... to allow function to work
-  # set catch the same as the previous year (sampled catch).
-  # catch is in the same units as the operating model, in this case it is in
-  # biomass.
-  catch <- data.frame(
-    year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward
-    seas = 1, # hard coded from looking at model 
-    fleet = 1,  # hard coded from looking at model
-    catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch,
-    catch_se = 0.05) # hard coded from looking at model
-  catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL
-  catch_F <- NULL # catch in terms of F, can be left as NULL.
-  discards <- NULL # discards can be left as NULL if there are no discards
-  catch_list <- list(catch = catch,
-                     catch_bio = catch_bio, 
-                     catch_F = catch_F,
-                     discards = discards)
-}
+
constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, 
+                              frac_catch = 0.5, ...) { # need to include ... to allow function to work
+  # set catch the same as the previous year (sampled catch).
+  # catch is in the same units as the operating model, in this case it is in
+  # biomass.
+  catch <- data.frame(
+    year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward
+    seas = 1, # hard coded from looking at model 
+    fleet = 1,  # hard coded from looking at model
+    catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch,
+    catch_se = 0.05) # hard coded from looking at model
+  catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL
+  catch_F <- NULL # catch in terms of F, can be left as NULL.
+  discards <- NULL # discards can be left as NULL if there are no discards
+  catch_list <- list(catch = catch,
+                     catch_bio = catch_bio, 
+                     catch_F = catch_F,
+                     discards = discards)
+}

Let’s assume this function is saved in a file within the working directory named constant_catch_MS.R.

This function can then be used in a call to run_SSMSE():

-
# define sample structure
-datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE")
-sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning
-sample_struct$lencomp <- NULL # don't use length sampling
-
-# run the SSMSE routine
-run_result_custom <- run_SSMSE(
-                              scen_name_vec = "constant-catch",
-                              out_dir_scen_vec = "my_results",
-                              iter_vec = 1,
-                              OM_name_vec = "cod",
-                              OM_in_dir_vec = NULL,
-                              MS_vec = "constant_catch_MS", # use custom fun
-                              custom_MS_source = "constant_catch_MS.R",
-                              use_SS_boot_vec = TRUE,
-                              nyrs_vec = 6,
-                              nyrs_assess_vec = 3,
-                              run_EM_last_yr = FALSE,
-                              run_parallel = FALSE,
-                              sample_struct_list = list(sample_struct),
-                              seed = 12345)
+
# define sample structure
+datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE")
+sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning
+sample_struct$lencomp <- NULL # don't use length sampling
+
+# run the SSMSE routine
+run_result_custom <- run_SSMSE(
+                              scen_name_vec = "constant-catch",
+                              out_dir_scen_vec = "my_results",
+                              iter_vec = 1,
+                              OM_name_vec = "cod",
+                              OM_in_dir_vec = NULL,
+                              MS_vec = "constant_catch_MS", # use custom fun
+                              custom_MS_source = "constant_catch_MS.R",
+                              use_SS_boot_vec = TRUE,
+                              nyrs_vec = 6,
+                              nyrs_assess_vec = 3,
+                              run_EM_last_yr = FALSE,
+                              run_parallel = FALSE,
+                              sample_struct_list = list(sample_struct),
+                              seed = 12345)
@@ -299,6 +299,7 @@

4.4 Sampling options

Currently, the only available sampling option is to use the bootstrapping module within SS itself. This means specifying use_SS_boot_vec = TRUE. Details on how sampling is done using the bootstrapping module in SS is available in the “Bootstrap Data Files” section of the SS user manual.

Users also need to specify how and which data types should be sampled for each future year in the simulation in sample_struct_list. sample_struct_list is a list of lists. The first level is a list for each scenario; then, for the scenario, there is a list of dataframes, each of which specifying which years and fleets of data should be sampled in the future as well as which standard errors or sample sizes should be used.

The helper function create_sample_struct() can be used to help users generate the list of dataframes for a scenario. See an example of this function’s use in the simple example or by typing ?SSMSE::create_sample_struct() into the R console.

+

Currently, users can sample data for catches (treated as fixed), CPUE (i.e., indices of abundance), length and age composition, conditional length at age compositions, mean size at age, and mean size. It is not yet possible to sample data for generalized size compositions, tagging data, and morph compositions.

Users can specify the sampling during the historical period of the model through the sample_struct_hist input to run_SSMSE. This is an optional input and has the same structure as sample_struct_list, but the years will be before the original end year of the OM.

@@ -307,8 +308,8 @@

4.5 Future changes to the operati

4.5.1 The strucutre of future_om_list

For example, here is an example list containing just one future change for the model. It shows that the model should be changed to 4.5 in year 103 and afterwards:

-
my_future_om_list <- create_future_om_list(list_length = 1)
-my_future_om_list
+
my_future_om_list <- create_future_om_list(list_length = 1)
+my_future_om_list
[[1]]
 [[1]]$pars
 [1] "SizeSel_P_3_Fishery(1)"
@@ -322,9 +323,9 @@ 
4.5.1 The strucutre of futu
 [[1]]$input
   first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param   method value
 1                 NA                NA              102                103     mean absolute   4.5
-
length(my_future_om_list) # note has length 1 b/c 1 change
+
length(my_future_om_list) # note has length 1 b/c 1 change
[1] 1
-
length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change
+
length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change
[1] 4

Note there is just one change specified here. For the change, there are four list items that are required for any specified change.

    @@ -354,8 +355,8 @@

    4.5.1 The strucutre of futu

    4.5.2 Example of future_om_list denoting a gradual change

    Suppose we wanted to change the value of “SizeSel_P_3_Fishery(1)” in scen2 over 4 years after year 102 to arrive at a value of 4.5 in year 106. We can do this by using my_future_om_list, but changing the value in the first row of first_yr_final_val to 106:

    -
    my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106
-my_future_om_list[[1]]
    +
    my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106
+my_future_om_list[[1]]
    $pars
 [1] "SizeSel_P_3_Fishery(1)"
 
@@ -372,17 +373,17 @@ 
    4.5.2 Example of future_om_list d
 
    
 
    4.5.3 Example of future_om_list with a random deviations
    
 
    Suppose we now wanted the value of “SizeSel_P_3_Fishery(1)” to change randomly according to a normal distribution with a standard deviation of 0.1 around a mean of 4.5 from 104 onwards. This can be done by adding a line specifying a change in standard deviation (which for now, has been assumed to be 0) to the data frame:
    
-
    new_vals <- data.frame(first_yr_averaging = NA,
-                       last_yr_averaging  = NA, 
-                       last_yr_orig_val   = 103,
-                       first_yr_final_val = 104, 
-                       ts_param = "sd", 
-                       method = "absolute",
-                       value = 0.1)
-
-my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]],
-                                           new_vals)
-my_future_om_list[[1]]
    
+
    new_vals <- data.frame(first_yr_averaging = NA,
+                       last_yr_averaging  = NA, 
+                       last_yr_orig_val   = 103,
+                       first_yr_final_val = 104, 
+                       ts_param = "sd", 
+                       method = "absolute",
+                       value = 0.1)
+
+my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]],
+                                           new_vals)
+my_future_om_list[[1]]
    
 
    $pars
 [1] "SizeSel_P_3_Fishery(1)"
 
@@ -401,27 +402,27 @@ 
    4.5.3 Example of future_om_list w
 
    
 
    4.5.4 Example of using historical values for determining parameter values
    
 
    This example applies random annual deviations to all parameters for scenarios scen2 and scen3.
    
-
    future_om_list_2 <- vector(mode = "list", length = 1)
-future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4))
-names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input")
-
-future_om_list_2[[1]][["pars"]] <- "all"
-future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3")
-future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist
-future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1),
-                                               last_yr_averaging = c(100, 100),
-                                               last_yr_orig_val = c(100, 100),
-                                               first_yr_final_val = c(101, 101), 
-                                               ts_param = c("cv", "mean"),
-                                               method = c("absolute", "multiplier"), 
-                                               value = c(0.1, 1))
    
+
    future_om_list_2 <- vector(mode = "list", length = 1)
+future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4))
+names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input")
+
+future_om_list_2[[1]][["pars"]] <- "all"
+future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3")
+future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist
+future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1),
+                                               last_yr_averaging = c(100, 100),
+                                               last_yr_orig_val = c(100, 100),
+                                               first_yr_final_val = c(101, 101), 
+                                               ts_param = c("cv", "mean"),
+                                               method = c("absolute", "multiplier"), 
+                                               value = c(0.1, 1))
    
 
    Note that the choice of year range for historical values does not matter unless the parameter is already time-varying in the original operating model or has become time varying through a previous change. Otherwise, the base model parameter will be applied. If no historical years are included, then the base parameter value will be the basis of comparison for relative changes (i.e., method = multiplier or additive).
    
 
    
 
    
 
    4.5.5 Example using “custom” pattern instead of “model_change”
    
-
    custom_future_om_list <- create_future_om_list(example_type = "custom", 
-                                               list_length = 1)
-custom_future_om_list
    
+
    custom_future_om_list <- create_future_om_list(example_type = "custom", 
+                                               list_length = 1)
+custom_future_om_list
    
 
    [[1]]
 [[1]]$pars
 [1] "impl_error"
@@ -541,18 +542,18 @@ 
    4.5.6 How is the operating modifi
 
    
 
    4.5.7 Example of specifying recruitment deviations
    
 
    This shows some code for putting recruitment deviations with a mean of 0 and the same standard deviation as the historical recruitment deviations in years 1 to 100:
    
-
    template_mod_change <- create_future_om_list(list_length = 1)
-rec_dev_specify <- template_mod_change[[1]]
-rec_dev_specify$pars <- "rec_devs"
-rec_dev_specify$scen <- c("replicate", "all")
-rec_dev_specify$input$first_yr_averaging <- 1
-rec_dev_specify$input$last_yr_averaging <- 100
-rec_dev_specify$input$last_yr_orig_val <- 100
-rec_dev_specify$input$first_yr_final_val <- 101
-rec_dev_specify$input$ts_param <- "sd"
-rec_dev_specify$input$value <- NA
-rec_dev_list <- list(rec_dev_specify)
-rec_dev_list
    
+
    template_mod_change <- create_future_om_list(list_length = 1)
+rec_dev_specify <- template_mod_change[[1]]
+rec_dev_specify$pars <- "rec_devs"
+rec_dev_specify$scen <- c("replicate", "all")
+rec_dev_specify$input$first_yr_averaging <- 1
+rec_dev_specify$input$last_yr_averaging <- 100
+rec_dev_specify$input$last_yr_orig_val <- 100
+rec_dev_specify$input$first_yr_final_val <- 101
+rec_dev_specify$input$ts_param <- "sd"
+rec_dev_specify$input$value <- NA
+rec_dev_list <- list(rec_dev_specify)
+rec_dev_list
    
 
    [[1]]
 [[1]]$pars
 [1] "rec_devs"
diff --git a/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html b/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html
index af4a21a5..4d5c99cb 100644
--- a/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html
+++ b/docs/manual/advanced-options-use-a-custom-management-strategyprocedure.html
@@ -24,7 +24,7 @@
 
 
 
-
+
 
   
   
@@ -226,39 +226,39 @@ 
    
 
    
 
    3 Advanced options: use a custom management strategy/procedure
    
 
    Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year:
    
-
    constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, 
-                              frac_catch = 0.5, ...) { # need to include ... to allow function to work
-  # set catch the same as the previous year (sampled catch).
-  # catch is in the same units as the operating model, in this case it is in
-  # biomass.
-  catch <- data.frame(
-    year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward
-    seas = 1, # hard coded from looking at model 
-    fleet = 1,  # hard coded from looking at model
-    catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch,
-    catch_se = 0.05) # hard coded from looking at model
-  catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL
-  catch_F <- NULL # catch in terms of F, can be left as NULL.
-  discards <- NULL # discards can be left as NULL if there are no discards
-  catch_list <- list(catch = catch,
-                     catch_bio = catch_bio, 
-                     catch_F = catch_F,
-                     discards = discards)
-}
    
+
    constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, 
+                              frac_catch = 0.5, ...) { # need to include ... to allow function to work
+  # set catch the same as the previous year (sampled catch).
+  # catch is in the same units as the operating model, in this case it is in
+  # biomass.
+  catch <- data.frame(
+    year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward
+    seas = 1, # hard coded from looking at model 
+    fleet = 1,  # hard coded from looking at model
+    catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch,
+    catch_se = 0.05) # hard coded from looking at model
+  catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL
+  catch_F <- NULL # catch in terms of F, can be left as NULL.
+  discards <- NULL # discards can be left as NULL if there are no discards
+  catch_list <- list(catch = catch,
+                     catch_bio = catch_bio, 
+                     catch_F = catch_F,
+                     discards = discards)
+}
    
 
    The function should be created in a separate file. In this case, assume this function is available in a file custom_funs.R.
    
 
    This function can then be used in a call to run_SSMSE:
    
-
    run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario
-                out_dir_scen_vec = run_res_path, # directory in which to run the scenario
-                iter_vec = 1,
-                OM_name_vec = "cod", # specify directories instead
-                OM_in_dir_vec = NULL,
-                MS_vec = "constant_catch_MS", # use the custom function
-                custom_MS_source = "custom_funs.R", # File where the custom function is available.
-                nyrs_vec = 6,        # Years to project OM forward
-                nyrs_assess_vec = 3, # Years between assessments
-                future_om_list = future_om_list_recdevs_sel,
-                sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS.
-                seed = 12345) #Set a fixed integer seed that allows replication 
    
+
    run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario
+                out_dir_scen_vec = run_res_path, # directory in which to run the scenario
+                iter_vec = 1,
+                OM_name_vec = "cod", # specify directories instead
+                OM_in_dir_vec = NULL,
+                MS_vec = "constant_catch_MS", # use the custom function
+                custom_MS_source = "custom_funs.R", # File where the custom function is available.
+                nyrs_vec = 6,        # Years to project OM forward
+                nyrs_assess_vec = 3, # Years between assessments
+                future_om_list = future_om_list_recdevs_sel,
+                sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS.
+                seed = 12345) #Set a fixed integer seed that allows replication

diff --git a/docs/manual/glossary-of-mse-terms.html b/docs/manual/glossary-of-mse-terms.html index 5d71dc5d..2a14a9c7 100644 --- a/docs/manual/glossary-of-mse-terms.html +++ b/docs/manual/glossary-of-mse-terms.html @@ -24,7 +24,7 @@ - + diff --git a/docs/manual/helper.html b/docs/manual/helper.html index 110ca522..03bf4b27 100644 --- a/docs/manual/helper.html +++ b/docs/manual/helper.html @@ -24,7 +24,7 @@ - + diff --git a/docs/manual/index.html b/docs/manual/index.html index 7509ad16..868a05e4 100644 --- a/docs/manual/index.html +++ b/docs/manual/index.html @@ -24,7 +24,7 @@ - + @@ -226,7 +226,7 @@

Preface

@@ -245,8 +245,9 @@

R session information

Matrix products: default locale: -[1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252 LC_MONETARY=English_United States.1252 -[4] LC_NUMERIC=C LC_TIME=English_United States.1252 +[1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252 +[3] LC_MONETARY=English_United States.1252 LC_NUMERIC=C +[5] LC_TIME=English_United States.1252 attached base packages: [1] stats graphics grDevices utils datasets methods base diff --git a/docs/manual/intro.html b/docs/manual/intro.html index 2d9c0d24..c54e1c18 100644 --- a/docs/manual/intro.html +++ b/docs/manual/intro.html @@ -24,7 +24,7 @@ - + diff --git a/docs/manual/output.html b/docs/manual/output.html index 42468b54..aef4f190 100644 --- a/docs/manual/output.html +++ b/docs/manual/output.html @@ -24,7 +24,7 @@ - + @@ -228,9 +228,9 @@

6 Output and Plots

6.1 Summarizing output

Output is summarized using SSMSE_summary_all():

-
summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir",
-                  scenarios = c("sample_low", "sample_high"),
-                  run_parallel = TRUE)
+
summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir",
+                  scenarios = c("sample_low", "sample_high"),
+                  run_parallel = TRUE)

Relying on ss3sim::get_results_all(), this function creates:

  • For each scenario, 3 scenario level .csv files
    • diff --git a/docs/manual/plotting-output-and-performance-metrics.html b/docs/manual/plotting-output-and-performance-metrics.html index cb7ed1c2..86e9bc55 100644 --- a/docs/manual/plotting-output-and-performance-metrics.html +++ b/docs/manual/plotting-output-and-performance-metrics.html @@ -24,7 +24,7 @@ - + diff --git a/docs/manual/search_index.json b/docs/manual/search_index.json index 62f56422..d83d7589 100644 --- a/docs/manual/search_index.json +++ b/docs/manual/search_index.json @@ -1 +1 @@ -[["index.html", "SSMSE user manual Preface R session information", " SSMSE user manual Kathryn Doering and Nathan Vaughan 2021-09-21 Preface Logo for the SSMSE package This is the user manual for SSMSE, an R package for Management Strategy Evaluation with Stock Synthesis Operating models. This document is still a work in progress! If you would like to suggest changes, the R markdown files used to generate the user manual are available in a bookdown folder within the SSMSE repository. Please feel free to open an issue or submit a pull request to suggest changes. R session information sessionInfo() R version 4.1.1 (2021-08-10) Platform: x86_64-w64-mingw32/x64 (64-bit) Running under: Windows 10 x64 (build 19042) Matrix products: default locale: [1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252 LC_MONETARY=English_United States.1252 [4] LC_NUMERIC=C LC_TIME=English_United States.1252 attached base packages: [1] stats graphics grDevices utils datasets methods base loaded via a namespace (and not attached): [1] ss3sim_1.1.6 r4ss_1.43.0 "],["intro.html", "1 Introduction 1.1 Purpose 1.2 Functions in SSMSE 1.3 Brief description of the SSMSE MSE simulation procedure", " 1 Introduction 1.1 Purpose SSMSE allows users to directly use Stock Synthesis (SS) as an operating model (OM) in an Management Strategy Evaluation (MSE). The approach requires a conditioned Stock Synthesis model, which is treated as the OM. 1.2 Functions in SSMSE The main functions users can call in SSMSE are: Function Description run_SSMSE() Run the MSE simulations SSMSE_summary_all() Summarize MSE output The helper functions to create inputs to run_SSMSE are: Helper Function Description create_sample_struct() Helper function to create a list for future sampling from a model to use as input in run_SSMSE() create_future_om_list() Helper function that provides examples of the structure for the future_om_list input to run_SSMSE(). develop_OMs() Helper function to turn one OM into many Exported functions that can be used for writing custom management strategies are: Function Description run_EM() Run an SS estimation model (uses run_ss_model) run_ss_model() Run an SS model get_bin() Get location of the SS binary. parse_MS() Function that runs the management strategy and returns catch by fleet for the projections. A reference function for those setting up custom management strategies. Finally, some plotting functions are available: Plotting Function Description plot_index_sampling() Plot to compare the sampled index values to the operating model expected values and original operating model conditioning index data. plot_comp_sampling() Plot to compare the sampled composition values to the operating model expected values and original operating model conditioning composition data. 1.3 Brief description of the SSMSE MSE simulation procedure 1.3.1 Conditioning the OM and sampling from the OM For each scenario, SSMSE starts with the user providing a fitted Stock Synthesis model (or selecting an model from the SSMSE package) to use as an OM. For each iteration of the scenario, SSMSE turns the SS fitted model into an OM and runs it once with no estimation with Stock Synthesis in order to get the true values and a bootstrapped data set from SS. Note that any modifications to the OMs parameters specified by the users as it is extended forward in time are also applied. 1.3.2 First run of the management strategy in the MSE simulation The bootstrapped dataset is then used in a Management strategy to project catch by fleet to use for the next \\(n\\) years, where \\(n\\) is the number of years between assessments. 1.3.3 Feedback from managment strategy into OM: extending model years The catch for the next \\(n\\) years before the next assessment is then added to the OM, as well as any recruitment or time varying parameter deviations. The OM is again run with no estimation where it can be used to produce sampled data for the next \\(n\\) years. These new data values are appended to the original dataset. 1.3.4 Subsequent runs of the management strategy The appended data set is then used in the managment strategy again, and new catch by fleet is produced that can then be fed back to the OM. "],["simple.html", "2 A simple example 2.1 Setup R workspace folders 2.2 Create the operating models (OMs) 2.3 Adding process error through recruitment deviations and time-varying selectivity 2.4 Examine the management procedure used 2.5 Run SSMSE 2.6 run_SSMSE output 2.7 Performance metrics 2.8 Summarize results 2.9 Simple Convergence Check 2.10 Plot Spawning Stock Biomass (SSB) 2.11 Example MSE Results 2.12 Delete the files", " 2 A simple example Suppose we want to look at how well we are able to achieve a performance metric under uncertainty in the operating model (OM). We will look 2 scenarios, one where Steepness (h) is specified correctly and one where it is specified incorrectly in an estimation model (EM): Scenario 1. h-ctl: Cod operating model (h = 0.65) with correctly specified cod model EM (fixed h = 0.65). The OM is the same as the EM. Scenario 2. h-1: Cod operating model (h = 1) with misspecified cod model EM (fixed h = 0.65); The OM is not the same as the EM. Note that this is a simple example where the OM and EM structures for both scenarios are identical, except for different steepness between the OM and EM in scenario 2 and some process error we will include in the operating model. We will assume we want to run the MSE loop for 6 years, with a stock assessment occuring every 3 years (and forecasting catch to maintain 40% of unfished spawning stock biomass). The cod models last year is 100, so the OM is initially conditioned through year 100. Then, after conditioning the operating model through year 100, assessments will occur in years 100 and 103. The operating model runs through year 106. We chose not to run the assessment in year 106, as there was no need for its output in this example. 2.1 Setup R workspace folders First, we will load the SSMSE package and create a folder in which to run the example: library(SSMSE) #load the package library(r4ss) #install using remotes::install_github("r4ss/r4ss) library(foreach) #if using run_parallel = TRUE library(doParallel) #if using run_parallel = TRUE # Create a folder for the output in the working directory. run_SSMSE_dir <- file.path("run_SSMSE-ex") dir.create(run_SSMSE_dir) Warning in dir.create(run_SSMSE_dir): 'run_SSMSE-ex' already exists 2.2 Create the operating models (OMs) 2.2.1 Specify alternative values for h The cod model with h = 0.65 (as in scenario 1) is included as external package data in SSMSE. However, we will need to modify it to use as an operating model with h = 1 (as in scenario 2). Note in this case that refit_OM is false, so the model is not being refit, just run through without fitting. To condition the new model on the same data as the input model, refit_OM should be TRUE. First, we identify where the base cod model is stored, modify it such that the steepness parameter is 1, and save the modified cod OM for scenario 2 in a new folder in the run_SSMSE_dir directory. cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE") # develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir # specified develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep", par_vals = 1, refit_OMs = FALSE, hess = FALSE) Warning in file.rename(from = file.path(out_dir, basename(OM_in_dir)), to = tmp_mod_path): cannot rename file 'run_SSMSE- ex/cod' to 'run_SSMSE-ex/cod_SR_BH_steep_1', reason 'Access is denied' # OM model for scenario 2 cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1") 2.3 Adding process error through recruitment deviations and time-varying selectivity Recruitment deviations, implementation error, and changes in parameters in the projection period of the OM can be added through the future_om_list input to run_SSMSE. First, well set up the list to add recruitment deviations in the projection period. The same recruitment deviation patterns are used across scenarios, but different patterns are use across iterations in the same scenario. We also want these deviations to have the same standard deviations as the historical deviations with 0 mean (the assumed default). # Start from a list created by a helper function template_mod_change <- create_future_om_list() # add recruitment deviations rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" # apply change to rec devs rec_dev_specify$scen <- c("replicate", "all") # using 1 to 100 means the sd or mean will be calculated by taking the sd across years # from 1 to 100 rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" # this change is for the sd # no input value needed since it will be calclated from the historical rec devs. rec_dev_specify$input$value <- NA rec_dev_specify $pars [1] "rec_devs" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 1 100 100 101 sd absolute NA Next, suppose we want to allow selectivity to vary annually for 1 selectivity parameter of the fishery throughout the projection period. The following specifies that the value for selectivity varies randomly around the base value with a sd of 0.2. # put together the change for selectivity (random values around the orig val, with # an sd of 0.2) mod_change_sel <- template_mod_change[[1]] mod_change_sel$scen[2] <- "all" # apply to all scenarios # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. # historical values are NA in this case, because they are not used to determine # the sd to use. mod_change_sel$input$last_yr_orig_val <- 100 mod_change_sel$input$first_yr_final_val <- 101 mod_change_sel$input$ts_param <- "sd" # this change is for the sd mod_change_sel$input$value <- 0.2 # se to use in the projection period mod_change_sel $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 100 101 sd absolute 0.2 Finally, add these two changes together into an object to pass to run_SSMSE future_om_list_recdevs_sel <- list(rec_dev_specify, mod_change_sel) 2.3.1 Add observation error through sampling from OM The argument sample_struct specifies the structure for sampling from the OM (and passing to the EM). The function create_sample_struct can be used to construct a simple sampling structure consistent with an input data file: datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this dataframe. sample_struct_1_scen $catch Yr Seas FltSvy SE 1 101 1 1 0.005 2 102 1 1 0.005 3 103 1 1 0.005 4 104 1 1 0.005 5 105 1 1 0.005 6 106 1 1 0.005 $CPUE Yr Seas FltSvy SE 1 105 7 2 0.2 $lencomp Yr Seas FltSvy Sex Part Nsamp 1 NA 1 1 0 0 125 $agecomp Yr Seas FltSvy Sex Part Ageerr Lbin_lo Lbin_hi Nsamp 1 105 1 2 0 0 1 -1 -1 500 $meanbodywt [1] NA $MeanSize_at_Age_obs [1] NA By default, create_sample_struct identifies sampling patterns from the historical period of the OM and replicates those patterns in the projection period. In our cod example, the sample structure specifies that catch will be added to the estimation model every year (years 101 to 106), but an index of abundance (i.e., CPUE) and age composition (i.e., agecomp) will only be added in year 105. We will use the same sampling scheme for both scenarios, but it is possible to specify different sampling for each scenario. The user could modify this sampling strategy (for example, maybe age composition should also be sampled from FltSvy 2 in Yr 102; the user could add another line to the dataframe in sample_struct$agecomp). Note that length comp (lencomp) includes an NA value for year. This is because no consistent pattern was identified, so the user must define their own input. In this case, we will remove sampling length comps all together: sample_struct_1_scen$lencomp <- NULL # don't use length sampling The same sampling structure will be used for both scenarios, which we define in a list below: sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen) 2.4 Examine the management procedure used We will use the same management procedure for both scenarios: Conduct a stock assessment every 3 years to get stock status. Project from this stock assessment using the SS forecast file to get projected future catch. Put this projected catch (without implementation error, in the case of this example) back into the OM. Extend the OM forward in time to get the true values for the population. Lets take a look at step 2 in the management procedure, which is implemented using the forecasting module in SS. We will examine the forecast file for the estimation model to better understand how catches will be forecasted from the assessment. We will use the same management procedure in both of these scenarios, although for a full MSE analysis, it is likely that multiple management procedures would be compared. fore <- r4ss::SS_readforecast( system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"), verbose = FALSE) fore$Forecast [1] 3 fore$Btarget [1] 0.4 fore$Forecast = 3 means our forecasts from the assessment will use fishing mortality (F) to attmpt to achieve a relative (to unfished) spawning stock biomass. Based on fore$Btarget, the relative biomass target is 40% of unfished spawning stock biomass. Note also that the control rule fore$BforconstantF and fore$BfornoF values are set low to make it unlikely that they will be used (these parameters are used for a ramp harvest control rule, which we do not want to use here): fore$BforconstantF [1] 0.03 fore$BfornoF [1] 0.01 Futhermore, fore$Flimitfraction is set to 1 so that the forecasted catch is set equal to the overfishing limit (for simplicity): fore$Flimitfraction [1] 1 Note that the number of forecast years is 1: fore$Nforecastyrs [1] 1 However, an assessment will be conducted every 3 years and thus 3 years of projections is required. SSMSE will automatically modify this value in the estimation model to the appropriate number of forecasting years. More information on using the forecast module in SS to forecast catches is available in the Stock Synthesis users manual. Users can also specify their own [custom management procedures] 2.5 Run SSMSE Now, we create a directory to store our results, and use run_SSMSE to run the MSE analysis loop (note this will take some time to run, ~ 20 min): run_res_path <- file.path(run_SSMSE_dir, "results") dir.create(run_res_path) res <- run_SSMSE( scen_name_vec = c("h-ctl", "h-1"),# name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = c(5,5), # run with 5 iterations each OM_name_vec = NULL, # specify directories instead OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files EM_name_vec = c("cod", "cod"), # cod is included in package data MS_vec = c("EM","EM"), # The management strategy is specified in the EM nyrs_vec = c(6, 6), # Years to project OM forward nyrs_assess_vec = c(3, 3), # Years between assessments future_om_list = future_om_list_recdevs_sel, run_parallel = TRUE, # Run iterations in parallel sample_struct_list = sample_struct_list_all, # How to sample data for running the EM. sample_struct_hist_list = NULL, # because this is null, will just use sampling # as in the current OM data file for the historical period. seed = 12345) #Set a fixed integer seed that allows replication See ?run_SSMSE for more details on function arguments. In a real MSE analysis, running 100+ iterations to reflect the full range of uncertainty (given observation and process errors) in the results would be preferred. However, we are only running 5 iterations per scenario in this demonstration to reduce computing time. 2.6 run_SSMSE output run_SSMSE will create new folders in the folders specified in out_dir_scen_vec (note that in this case, we are running both scenarios in the same folder). After is complete, there will be a folder for each scenario in run_res_path (since out_dir_scen_vec = run_res_path in this example). Within each scenario is a folder for each scenario. And within each scenario folder, there are folders containing the SS models that were run by run_SSMSE. There should be 1 folder for the OM, which is run multiple times in this same folder during the MSE analysis. There are multiple folders for the EMs, as a new folder is created each time an assessment is done. The first run is the folder with a name ending in init; then, each assessment after is named for the updated end year of the model. With many iterations, the number of files adds up; in the future, we hope to add options to save less output. 2.7 Performance metrics Quantitative performance metrics should be specified before conducting an MSE. Typically, a suite of performance metrics will be examined; however, for simplicity in this example, we will only look at what the achieved relative biomass was for the last 3 years of projection in the MSE to determine how it compares to the intended management target of 40% of unfished Spawning Stock Biomass. Note that we are only running our MSE projection for 6 years, but longer projections are typical in MSE analyses. 2.8 Summarize results The function SSMSE_summary_all can be used to summarize the model results in a list of 3 dataframes, one for scalar outputs (named scalar), one for timeseries outputs (ts), one for derived quantities (dq). This function also creates summary csv files in the folder where the results are stored. # Summarize 1 iteration of output summary <- SSMSE_summary_all(run_res_path) ## Extracting results from 2 scenarios ## Starting h-1 with 5 iterations ## Starting h-ctl with 5 iterations Plotting and data manipulation can then be done with these summaries. For example, SSB over time by model can be plotted. The models include the Operating Model (cod_OM), Estimation model (EM) for the historical period of years 0-100 (cod_EM_init), and the EM run with last year of data in year 103 (cod_EM_103). The operating models are shown in blue or black (depending on the scenario), and the estimation model runs are shown in orange, and the scenarios are shown on different subplots: library(ggplot2) # use install.packages("ggplot2") to install package if needed library(tidyr) # use install.packages("tidyr") to install package if needed library(dplyr) # use install.packages("dplyr") to install package if needed Attaching package: 'dplyr' The following objects are masked from 'package:stats': filter, lag The following objects are masked from 'package:base': intersect, setdiff, setequal, union 2.9 Simple Convergence Check Check there are no params on bounds or SSB that is way too small or way too large check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) { require(dplyr) # note: not the best way to do this if(any(!is.na(summary$scalar$params_on_bound))) { warning("Params on bounds") } else { message("No params on bounds") } summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM") calc_SSB <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% select(iteration, scenario, year, model_run, model_type, SpawnBio) OM_vals <- calc_SSB %>% filter(model_type == "OM") %>% rename(SpawnBio_OM = SpawnBio ) %>% select(iteration, scenario, year, SpawnBio_OM) EM_vals <- calc_SSB %>% filter(model_type == "EM") %>% rename(SpawnBio_EM = SpawnBio) %>% select(iteration, scenario, year, model_run, SpawnBio_EM) bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM) filter_SSB <- bind_vals %>% filter(SSB_ratio > 2 | SSB_ratio < 0.5) if(nrow(filter_SSB) > 0 ) { warning("Some large/small SSBs relative to OM") } else { message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM") } return_val <- bind_vals } values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5) No params on bounds All SSBs in EM are no greater than double and no less than half SSB vals in the OM 2.10 Plot Spawning Stock Biomass (SSB) This plot shows that SSB estimated does not match perfectly with the operating model. A similar plot could be made for any parameter of interest. # plot SSB by year and model run ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), ggplot2::aes(x = year, y = SpawnBio)) + ggplot2::geom_vline(xintercept = 100, color = "gray") + ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+ ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) + ggplot2::scale_linetype_manual(values = rep("solid", 50)) + ggplot2::guides(linetype = FALSE) + ggplot2::facet_wrap(. ~ scenario) + ggplot2::theme_classic() Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> = "none")` instead. Now, we calculate and plot the performance metric, which is average spawning stock biomass (SSB) from years 104 to 106. # get_SSB_avg calculates the SSB in each year for each # iteration of the operating model, then takes the average over the years from # min_yr, to max_year. It uses the summary object as input to do these # calculations. get_SSB_avg <- function(summary, min_yr, max_yr) { OM_vals <- unique(summary$ts$model_run) OM_vals <- grep("_OM$", OM_vals, value = TRUE) SSB_yr <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% filter(model_run %in% OM_vals) %>% select(iteration, scenario, year, SpawnBio) %>% group_by(iteration, scenario) %>% summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% ungroup() SSB_yr } avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106) # function to summarize data in plot data_summary <- function(x) { m <- mean(x) ymin <- m - sd(x) ymax <- m + sd(x) return(c(y = m, ymin = ymin, ymax = ymax)) } # Now, plot the average relative spawning stock biomass for years 104 - 106 ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) + stat_summary(fun.data = data_summary, position = position_dodge(width = 0.9), color = "blue") + labs(title = "Long-term average SSB\\n(years 104-106)", x = "Scenario", y = "SSB") + theme_classic() From the above plot, we see differences in the average SSb between the 2 scenarios. 2.11 Example MSE Results We can see from the performance metric that mis-specifying the value of steepness will results in higher realized relative spawning stock biomass than correctly specifying it. This gives us some idea of the consequences of misspecifying steepness in the stock assessment. 2.12 Delete the files If you wish to delete the files created from this example, you can use: unlink(run_SSMSE_dir, recursive = TRUE) "],["advanced-options-use-a-custom-management-strategyprocedure.html", "3 Advanced options: use a custom management strategy/procedure", " 3 Advanced options: use a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } The function should be created in a separate file. In this case, assume this function is available in a file custom_funs.R. This function can then be used in a call to run_SSMSE: run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = 1, OM_name_vec = "cod", # specify directories instead OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use the custom function custom_MS_source = "custom_funs.R", # File where the custom function is available. nyrs_vec = 6, # Years to project OM forward nyrs_assess_vec = 3, # Years between assessments future_om_list = future_om_list_recdevs_sel, sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS. seed = 12345) #Set a fixed integer seed that allows replication "],["SSMSE.html", "4 Options for run_SSMSE 4.1 Scenarios in SSMSE 4.2 Operating model 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) 4.4 Sampling options 4.5 Future changes to the operating model (#future)", " 4 Options for run_SSMSE Many inputs are possible for the run_SSMSE() option. Here, we will describe some of the options available. For detailed documentation, type ?SSMSE::run_SSMSE() into the R console. 4.1 Scenarios in SSMSE Note that multiple scenarios can be called in run_SSMSE(), often through vector inputs to run_SSMSE(). Below, we will describe inputs needed to run 1 scenario. 4.2 Operating model The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = \"cod\". Otherwise, the path to the OM model should be specified in OM_in_dir_vec. 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) The management strategy (and EM) can be specified in one of two ways: Using an SS model Using a custom procedure via a function in R In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet. 4.3.1 Specify the Management Strategy in a SS model An SS model can be set up as the EM for the MSE. To use this option, specify \"EM\" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify \"cod\" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec. Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user. 4.3.2 Using a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } Lets assume this function is saved in a file within the working directory named constant_catch_MS.R. This function can then be used in a call to run_SSMSE(): # define sample structure datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning sample_struct$lencomp <- NULL # don't use length sampling # run the SSMSE routine run_result_custom <- run_SSMSE( scen_name_vec = "constant-catch", out_dir_scen_vec = "my_results", iter_vec = 1, OM_name_vec = "cod", OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use custom fun custom_MS_source = "constant_catch_MS.R", use_SS_boot_vec = TRUE, nyrs_vec = 6, nyrs_assess_vec = 3, run_EM_last_yr = FALSE, run_parallel = FALSE, sample_struct_list = list(sample_struct), seed = 12345) 4.4 Sampling options Currently, the only available sampling option is to use the bootstrapping module within SS itself. This means specifying use_SS_boot_vec = TRUE. Details on how sampling is done using the bootstrapping module in SS is available in the Bootstrap Data Files section of the SS user manual. Users also need to specify how and which data types should be sampled for each future year in the simulation in sample_struct_list. sample_struct_list is a list of lists. The first level is a list for each scenario; then, for the scenario, there is a list of dataframes, each of which specifying which years and fleets of data should be sampled in the future as well as which standard errors or sample sizes should be used. The helper function create_sample_struct() can be used to help users generate the list of dataframes for a scenario. See an example of this functions use in the simple example or by typing ?SSMSE::create_sample_struct() into the R console. Users can specify the sampling during the historical period of the model through the sample_struct_hist input to run_SSMSE. This is an optional input and has the same structure as sample_struct_list, but the years will be before the original end year of the OM. 4.5 Future changes to the operating model (#future) By default, SSMSE will simply extend the structure of the OM into the future using the same parameter values as in the last year of the model. However, the user may want to allow changes in parameter values to occur as the OM is extended forward in time. Users can input values into the future_om_list to accomplish this. This input is a list of lists. For the first level of lists, each represents a separate change to make. Within the first level, there are 4 list components that outline the details of the change to make. There are 2 main choices: 1) to specify model changes by telling SSMSE how values should be sampled or 2) to input your own custom values for future values of a given parameter. 4.5.1 The strucutre of future_om_list For example, here is an example list containing just one future change for the model. It shows that the model should be changed to 4.5 in year 103 and afterwards: my_future_om_list <- create_future_om_list(list_length = 1) my_future_om_list [[1]] [[1]]$pars [1] "SizeSel_P_3_Fishery(1)" [[1]]$scen [1] "replicate" "scen2" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 103 mean absolute 4.5 length(my_future_om_list) # note has length 1 b/c 1 change [1] 1 length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change [1] 4 Note there is just one change specified here. For the change, there are four list items that are required for any specified change. The first list item is named pars. It contains a vector of parameter name(s) to apply the change to. The names should be the same as the names in r4ss::SS_read_pars() or can include the following: \"rec_devs\" - For specifying recruitment deviations \"impl_error - For specifying implementation error in transforming a management procedures specified future catch into realized catch. \"all\" - Apply the change to all parameters in the model, with the exception of SR_sigmaR, SR_regime, SR_autocorr, and impl_error parameters. Changing the first 3 stock recruitment related parameters would conflict with changes in recruitment deviations. Since implementation error is not a parameter specified in the control file and SSMSE does not rely on the implementation error parameter created through the forecasting file, including the implementation error in all did not seem appropriate. In this case, we just want to apply the change to the SizeSel_P_3_Fishery(1) parameter of the model. The second item is scen, which contains a vector of information about how to apply the changes within and across scenarios. The first value is an option to specify how the change should be applied among scenarios, either randomize (use a different value for each iteration of each scenario) or replicate (use the same set of values across scenarios for the same number iteration, but each value will be different across iterations within the same scenario). Note that the same values will be applied across iterations and scenarios if there isnt any stochasticity in the values being drawn (e.g., standard deviation set at 0), regardless if randomize or replicate is chosen. In the example above, there is no stochasticity, so specifying randomize or replicate does not matter. Following the first value are the names of the scenarios to apply the change to. If the change should be applied to all of the scenarios, all can be used in place of naming every scenario. In this example, The change will only be applied to the scenario named scen2, so the input for scen is c(\"randomize\", \"scen2\"). The pattern is a vector of character inputs. The first value should be either model_change, meaning that SSMSE will calculate the change values, or custom which allows the user to directly put in the values that they want in the model. In this case, model_change is used. A second input can also be added when the model_change pattern is used, which specifies the distribution to pull from. This could be normal or lognormal, but if no input is provided, a normal distribution will be used for sampling. The fourth item, named input, is a dataframe, which contains different column name values depending on if the model_change pattern is used or if the custom pattern is used. For the model_change options, a dataframe in input specifies a change for the parameter of a distribution. Because we are using the model change option, we have the following columns: first_yr_averaging: The first year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_averaging: The last year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_orig_val: The last year of the future deviations with the original model value. This value will not be changed, but the following years value will be. first_yr_final_val: The first year where the final value as specified will be reached. If no future changes are made, the final value will continue forward in the OM through all projected years. For step changes, the first_yr_final_val is just 1 year after the last_yr_orig_val. However, if a more gradual adjustment toward the final value is desired, this could be any number of years after the original value. ts_param: The sampling parameter to modify. Options are mean, ar_1_phi (if using autocorrelation), sd, and cv. If not included in the dataframe, mean is assumed to be the same as in the previous model year, sd is assumed to be 0, and we assume no autocorrelation (as specified through an autoregressive AR1 process). Phi should be between -1 and 1 (this creates a stationary process) and if phi = 1, this creates a random walk (as random walk is a special case of an AR1 model). Note that both sd and cv cannot be specified within the same input dataframe. method: How to apply the change relative to the historical (if specified) or previous years value (if first_yr_averaging and last_yr_averaging for that row are NA). Options are multiplicative, additive, and absolute. This example uses absolute, meaning that the number in value is directly used. value. The value of the parameter change. This may be NA if historical averaging is used. 4.5.2 Example of future_om_list denoting a gradual change Suppose we wanted to change the value of SizeSel_P_3_Fishery(1) in scen2 over 4 years after year 102 to arrive at a value of 4.5 in year 106. We can do this by using my_future_om_list, but changing the value in the first row of first_yr_final_val to 106: my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106 my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 106 mean absolute 4.5 4.5.3 Example of future_om_list with a random deviations Suppose we now wanted the value of SizeSel_P_3_Fishery(1) to change randomly according to a normal distribution with a standard deviation of 0.1 around a mean of 4.5 from 104 onwards. This can be done by adding a line specifying a change in standard deviation (which for now, has been assumed to be 0) to the data frame: new_vals <- data.frame(first_yr_averaging = NA, last_yr_averaging = NA, last_yr_orig_val = 103, first_yr_final_val = 104, ts_param = "sd", method = "absolute", value = 0.1) my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]], new_vals) my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 106 mean absolute 4.5 2 NA NA 103 104 sd absolute 0.1 Note that the last_yr_orig_val and first_yr_final_val are different than the line for the mean, which is allowed. 4.5.4 Example of using historical values for determining parameter values This example applies random annual deviations to all parameters for scenarios scen2 and scen3. future_om_list_2 <- vector(mode = "list", length = 1) future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4)) names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input") future_om_list_2[[1]][["pars"]] <- "all" future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3") future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1), last_yr_averaging = c(100, 100), last_yr_orig_val = c(100, 100), first_yr_final_val = c(101, 101), ts_param = c("cv", "mean"), method = c("absolute", "multiplier"), value = c(0.1, 1)) Note that the choice of year range for historical values does not matter unless the parameter is already time-varying in the original operating model or has become time varying through a previous change. Otherwise, the base model parameter will be applied. If no historical years are included, then the base parameter value will be the basis of comparison for relative changes (i.e., method = multiplier or additive). 4.5.5 Example using custom pattern instead of model_change custom_future_om_list <- create_future_om_list(example_type = "custom", list_length = 1) custom_future_om_list [[1]] [[1]]$pars [1] "impl_error" [[1]]$scen [1] "randomize" "all" [[1]]$pattern [1] "custom" [[1]]$input par scen iter yr value 1 impl_error scen1 1 101 1.05 2 impl_error scen1 2 101 1.05 3 impl_error scen1 3 101 1.05 4 impl_error scen1 4 101 1.05 5 impl_error scen1 5 101 1.05 6 impl_error scen1 1 102 1.05 7 impl_error scen1 2 102 1.05 8 impl_error scen1 3 102 1.05 9 impl_error scen1 4 102 1.05 10 impl_error scen1 5 102 1.05 11 impl_error scen1 1 103 1.05 12 impl_error scen1 2 103 1.05 13 impl_error scen1 3 103 1.05 14 impl_error scen1 4 103 1.05 15 impl_error scen1 5 103 1.05 16 impl_error scen1 1 104 1.05 17 impl_error scen1 2 104 1.05 18 impl_error scen1 3 104 1.05 19 impl_error scen1 4 104 1.05 20 impl_error scen1 5 104 1.05 21 impl_error scen1 1 105 1.05 22 impl_error scen1 2 105 1.05 23 impl_error scen1 3 105 1.05 24 impl_error scen1 4 105 1.05 25 impl_error scen1 5 105 1.05 26 impl_error scen1 1 106 1.05 27 impl_error scen1 2 106 1.05 28 impl_error scen1 3 106 1.05 29 impl_error scen1 4 106 1.05 30 impl_error scen1 5 106 1.05 31 impl_error scen2 1 101 1.10 32 impl_error scen2 2 101 1.10 33 impl_error scen2 3 101 1.10 34 impl_error scen2 4 101 1.10 35 impl_error scen2 5 101 1.10 36 impl_error scen2 1 102 1.10 37 impl_error scen2 2 102 1.10 38 impl_error scen2 3 102 1.10 39 impl_error scen2 4 102 1.10 40 impl_error scen2 5 102 1.10 41 impl_error scen2 1 103 1.10 42 impl_error scen2 2 103 1.10 43 impl_error scen2 3 103 1.10 44 impl_error scen2 4 103 1.10 45 impl_error scen2 5 103 1.10 46 impl_error scen2 1 104 1.10 47 impl_error scen2 2 104 1.10 48 impl_error scen2 3 104 1.10 49 impl_error scen2 4 104 1.10 50 impl_error scen2 5 104 1.10 51 impl_error scen2 1 105 1.10 52 impl_error scen2 2 105 1.10 53 impl_error scen2 3 105 1.10 54 impl_error scen2 4 105 1.10 55 impl_error scen2 5 105 1.10 56 impl_error scen2 1 106 1.10 57 impl_error scen2 2 106 1.10 58 impl_error scen2 3 106 1.10 59 impl_error scen2 4 106 1.10 60 impl_error scen2 5 106 1.10 61 impl_error scen3 1 101 1.10 62 impl_error scen3 2 101 1.10 63 impl_error scen3 3 101 1.10 64 impl_error scen3 4 101 1.10 65 impl_error scen3 5 101 1.10 66 impl_error scen3 1 102 1.10 67 impl_error scen3 2 102 1.10 68 impl_error scen3 3 102 1.10 69 impl_error scen3 4 102 1.10 70 impl_error scen3 5 102 1.10 71 impl_error scen3 1 103 1.10 72 impl_error scen3 2 103 1.10 73 impl_error scen3 3 103 1.10 74 impl_error scen3 4 103 1.10 75 impl_error scen3 5 103 1.10 76 impl_error scen3 1 104 1.10 77 impl_error scen3 2 104 1.10 78 impl_error scen3 3 104 1.10 79 impl_error scen3 4 104 1.10 80 impl_error scen3 5 104 1.10 81 impl_error scen3 1 105 1.10 82 impl_error scen3 2 105 1.10 83 impl_error scen3 3 105 1.10 84 impl_error scen3 4 105 1.10 85 impl_error scen3 5 105 1.10 86 impl_error scen3 1 106 1.10 87 impl_error scen3 2 106 1.10 88 impl_error scen3 3 106 1.10 89 impl_error scen3 4 106 1.10 90 impl_error scen3 5 106 1.10 The inputs required for pattern = custom are similar to using pattern = model_change, but there is no need to specify a distribution as the second vector input of pattern and the column names in input are different. Also that the change is randomized, which indicates that a value for each scenario and each iteration are necessary, whereas if the first value of scen was replicate, separate values for each scenario should not be specified, but rather a single value for all should be used. The columns in the dataframe are: par: the parameter name or all if it should be applied to all parameters in the pars input scen: which scenario the value should apply to, or all if replicate is used as the scenario input iter: which number iteration the value should apply to. These are absolute iteration numbers. yr: the year the value applies to. These should be after the original end year of the OM. value: the value to apply to the model 4.5.6 How is the operating modified to accomodate changes as specified in the future_OM_list? All changes are made by converting the parameter(s) with changes to be time varing by using additive annual parameter deviations. Because this is an operating model, Stock Synthesis is not run with estimation, so to get the changes into the OM, values (drawn or calculated in model_changes or specified by the user using custom) are directly input as parameter deviations into the ss.par file. If an OM already contains time varying parameters, these parameters will also be converted to additive parameter deviations before applying the changes specified in the future_OM_list. 4.5.7 Example of specifying recruitment deviations This shows some code for putting recruitment deviations with a mean of 0 and the same standard deviation as the historical recruitment deviations in years 1 to 100: template_mod_change <- create_future_om_list(list_length = 1) rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" rec_dev_specify$scen <- c("replicate", "all") rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" rec_dev_specify$input$value <- NA rec_dev_list <- list(rec_dev_specify) rec_dev_list [[1]] [[1]]$pars [1] "rec_devs" [[1]]$scen [1] "replicate" "all" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 1 100 100 101 sd absolute NA "],["helper.html", "5 Helper functions in SSMSE 5.1 Creating multiple operating models with develop_OMs() 5.2 Set up sampling for a scenario with create_sample_struct() 5.3 Get examples of the future_om_list input with create_future_om_list()", " 5 Helper functions in SSMSE run_SSMSE() requires some detailed inputs that are time consuming to make. In the interest of automating routines as much as possible, helper functions are available in SSMSE. 5.1 Creating multiple operating models with develop_OMs() This function allows users to change a parameter in a Stock Synthesis OM and, if desired, refit the operating model to data. Typically, users will want to create multiple operating models to use in different scenarios in order to account for uncertainties in parameter values, such as biological parameters relating to natural mortality or growth. 5.2 Set up sampling for a scenario with create_sample_struct() run_SSMSE() requires the input sample_struct, which outlines the future sampling structure to use to generate data sets. This object could be difficult to create manually and users may want to just continue sampling as previously done in an existing SS model. create_sample_struct() will use the time patterns in sampling for different data types found in an SS data file and extend it forward in time. If no pattern is found, the function will return NAs and provide a warning to the user. If any NAs are returned (except to indicate that an entire data type is missing), the user will need to remove or fill in the NA value before usign the list as an input to run_SSMSE(). See use of create_sample_struct() in simple example. 5.3 Get examples of the future_om_list input with create_future_om_list() Users can modify the future structure of their operating model through the future_om_list input to run_SSMSE. The structure and names of this object matter, so examples are provided by calling create_future_om_list(). See more details about the specification of future_om_list objects. "],["output.html", "6 Output and Plots 6.1 Summarizing output 6.2 Checking estimation model convergence 6.3 Calculating performance metrics", " 6 Output and Plots 6.1 Summarizing output Output is summarized using SSMSE_summary_all(): summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir", scenarios = c("sample_low", "sample_high"), run_parallel = TRUE) Relying on ss3sim::get_results_all(), this function creates: For each scenario, 3 scenario level .csv files For all scenarios, 2 cross-scenario .csv files named by default to SSMSE_tsand SSMSE_scalar. For all scenarios, the function returns a list object containing data frames of timeseries (ts), scalar, and derived quantities (dq) summaries. Note that run_parallel = TRUE is only faster than run_parallel FALSE when there is more than once scenario and none of the scenario-level .csv files have been created yet. By default, if a user doesnt specify scenarios, all scenarios in dir will be summarized. 6.2 Checking estimation model convergence One of the first checks to do if using an estimation model after running an MSE analysis is to check that the estimation model has converged. A number of checks could be done, but a basic one is checking the gradients for the estimation model, which are added to the SSMSE_scalar summary sheet. 6.3 Calculating performance metrics Typically, a suite of performance metrics are used in MSE. Punt et al. (2016) recommends at least using metrics related to: average catch variation in catch over time population size "],["plotting-output-and-performance-metrics.html", "7 Plotting output and performance metrics", " 7 Plotting output and performance metrics Currently, it has been left up to the user to plot summaries and performance metrics, as the potential options for plots and performance metrics users may desire are extensive. There are currently diagnostic plots to examine how sampled data compares to the operating model values and data sets used to condition the operating models. plot_index_sampling() creates a diagnostic plot for indices, while plot_comp_sampling() creates diagnostic plots for composition data. "],["glossary-of-mse-terms.html", "8 Glossary of MSE terms", " 8 Glossary of MSE terms This glossary is based off of Rademeyer et al. (2007) and Punt et al. (2016). Assessment error - Error that occurs during the process of conducting an assessment, specifically error which inform the catch control rule that is being evaluated using the MSE: management advice for any system is based on uncertain data. Consequently, the data that inform catch control rules need to be generated in a manner which is as realistic as possible. Uncertainty arises when the model used for conducting assessments and providing management advice differs from the operating model, or the data are too noisy to estimate all key parameters reliably (Punt et al. 2016). Assessment model (AM) - A fitted and well scrutinized Stock Synthesis stock assessment model. The AM is input for SSMSE so that it can build the operating model. Bootstrapped dataset - After running an SS model (and if the user turns on the option in the starter file), the output file data.ss_new contains a dataset that has the same form as the input data.ss file, but instead of the input values, contains a sampled dataset. This is derived from using the expected values given the parameter estimates and model structure (see glossary entry on Expected values), uncertainty either estimated or input in the model, and an assumed distribution for sampling. As many bootstrapped data sets as the user desires can be output from an SS model. These bootstrap datasets are the third or greater set of values in the data.ss_new file. Estimation model (EM) - This refers to the model used within the MSE procedure to represent the stock assessment process. Note that an estimation model proxy could also be used to represent the stock assessment process. Expected values - After running an SS model (and if the user turns on teh option in the starter file), the output file data.ss_new contains a dataset that has the same form as the input data.ss file, but instead of the input values, contains the expected values given the parameter estimates and model structure. This expected values dataset is the second set of values in the data.ss_new file. Implementation error - Also called implementation uncertainty or outcome uncertainty. Broadly, this includes error of implementing the management action(s), which is often not done perfectly. Definition from Punt et al. (2016): The most obvious form of this type of uncertainty is when catches are not the same as the TACs typically more is taken or the decision-makers do not implement the TACs suggested by the management strategy. However, there are many other sources of outcome uncertainty, such as that associated with catch limits set for recreational fisheries and regulating discards. Index - An index of abundance. In SS files, this is sometimes used interchangably with CPUE. Plural, indices. Management strategy - Following Punt et al. (2016), there are 2 types: model-based management strategies and empirical management strategies. Ideally, SSMSE will allow for users to use either of these management strategies. Model-based management strategies include conducting a stock assessment and using output for determining harvest control rules. Empirical management strategies do not involve conducting a stock assessment but rather setting regulations from data (although summarization is possible). In some cases, management strategies are computationally intensive/time consuming to formally include in the context of an MSE; to speed up the process, it is commen to use management strategy proxies, typically an assumed error distribution about the operating model values. Management strategies are also known as management procedures. Model uncertainty - Definition from Punt et al. (2016): the form of relationships within an operating model will always be subject to uncertainty. The simplest type of model uncertainty involves, for example, whether the stockrecruitment relationship is BevertonHolt or Ricker, whether a fixed value for a model parameter is correct, or whether fishery selectivity is asymptotic or dome-shaped. However, there are other more complicated types of model structure uncertainty such as how many stocks are present in the area modelled, the error structure of the data used for assessment purposes, the impact of future climate change on biological relationships such as the stockrecruitment function, and ecosystem impacts on biological and fishery processes. Observation error - Error that results from not observing the true dynamics of the system. See also bootstrapped dataset. Operating model (OM) - This is a Stock Synthesis model that defines the assumed true dynamics of the population and its associated fisheries for the purposes of management strategy evaluation. Most often more than one operating model is necessary in order to adequately characterize the uncertainty in the true dynamics of the system. The Stock Synthesis OM may also define how sampling from the true dynamics is done, as SS produces expected values and, if desired, bootstrapped data sets based on sampling assumptions implicit in the model. For more information, see the glossary entries on Expected values and bootstrapped dataset Parameter uncertainty - Definition from Punt et al. (2016): many operating models are fit to the data available, but the values estimated for the parameters of those operating models (e.g. fishery selectivity-at-age, the parameters of the stockrecruitment relationship and historical deviations in recruitment about the stockrecruitment relationship) are subject to error. Performance statistics - Definition from Rademeyer et al. (2007): Statistics that summarize different aspects of the results of a simulation trial used to evaluate how well a specific [management strategy] achieves some or all of the general objectives for management for a particular scenario. Performance statistics usually fall into one of three categories: catch-related, stability related, or risk related. Process uncertainty - Definition from Punt et al. (2016): variation (usually assumed to be random, though sometimes incorporating autocorrelation) in parameters often considered fixed in stock assessments such as natural mortality, future recruitment about a stockrecruitment relationship and selectivity. Stock Synthesis (SS, SS3) - A integrative, general population dynamics modeling program used to assess the effects of fishing on population. Available at the Stock Synthesis website Uncertainty - Incorporating uncertainty into an MSE procedure is extremely important. There are several potential sources of uncertainty, which we have divided as done in Punt et al. (2016) in addition to adding observation error. For more details, see the glossary entries on Process Uncertainty, Parameter uncertainty, Model uncertainty, Assessment error, Implementation error, and observation error. Punt et al. (2016) suggest that MSE should at least consider process uncertainty (particularly deviations from the stock recruitment relationship), parameter uncertainty (particularly which relates to productivity and stock size), and observation error. Note that Rademeyer et al. (2007) divides error into estimation error, implementation error, observation error and process error, which could be used instead. These may be more natural divisions in sources of error. "],["404.html", "Page not found", " Page not found The page you requested cannot be found (perhaps it was moved or renamed). You may want to try searching to find the page's new location, or use the table of contents to find the page you are looking for. "]] +[["index.html", "SSMSE user manual Preface R session information", " SSMSE user manual Kathryn Doering and Nathan Vaughan 2021-10-06 Preface Logo for the SSMSE package This is the user manual for SSMSE, an R package for Management Strategy Evaluation with Stock Synthesis Operating models. This document is still a work in progress! If you would like to suggest changes, the R markdown files used to generate the user manual are available in a bookdown folder within the SSMSE repository. Please feel free to open an issue or submit a pull request to suggest changes. R session information sessionInfo() R version 4.1.1 (2021-08-10) Platform: x86_64-w64-mingw32/x64 (64-bit) Running under: Windows 10 x64 (build 19042) Matrix products: default locale: [1] LC_COLLATE=English_United States.1252 LC_CTYPE=English_United States.1252 [3] LC_MONETARY=English_United States.1252 LC_NUMERIC=C [5] LC_TIME=English_United States.1252 attached base packages: [1] stats graphics grDevices utils datasets methods base loaded via a namespace (and not attached): [1] ss3sim_1.1.6 r4ss_1.43.0 "],["intro.html", "1 Introduction 1.1 Purpose 1.2 Functions in SSMSE 1.3 Brief description of the SSMSE MSE simulation procedure", " 1 Introduction 1.1 Purpose SSMSE allows users to directly use Stock Synthesis (SS) as an operating model (OM) in an Management Strategy Evaluation (MSE). The approach requires a conditioned Stock Synthesis model, which is treated as the OM. 1.2 Functions in SSMSE The main functions users can call in SSMSE are: Function Description run_SSMSE() Run the MSE simulations SSMSE_summary_all() Summarize MSE output The helper functions to create inputs to run_SSMSE are: Helper Function Description create_sample_struct() Helper function to create a list for future sampling from a model to use as input in run_SSMSE() create_future_om_list() Helper function that provides examples of the structure for the future_om_list input to run_SSMSE(). develop_OMs() Helper function to turn one OM into many Exported functions that can be used for writing custom management strategies are: Function Description run_EM() Run an SS estimation model (uses run_ss_model) run_ss_model() Run an SS model get_bin() Get location of the SS binary. parse_MS() Function that runs the management strategy and returns catch by fleet for the projections. A reference function for those setting up custom management strategies. Finally, some plotting functions are available: Plotting Function Description plot_index_sampling() Plot to compare the sampled index values to the operating model expected values and original operating model conditioning index data. plot_comp_sampling() Plot to compare the sampled composition values to the operating model expected values and original operating model conditioning composition data. 1.3 Brief description of the SSMSE MSE simulation procedure 1.3.1 Conditioning the OM and sampling from the OM For each scenario, SSMSE starts with the user providing a fitted Stock Synthesis model (or selecting an model from the SSMSE package) to use as an OM. For each iteration of the scenario, SSMSE turns the SS fitted model into an OM and runs it once with no estimation with Stock Synthesis in order to get the true values and a bootstrapped data set from SS. Note that any modifications to the OMs parameters specified by the users as it is extended forward in time are also applied. 1.3.2 First run of the management strategy in the MSE simulation The bootstrapped dataset is then used in a Management strategy to project catch by fleet to use for the next \\(n\\) years, where \\(n\\) is the number of years between assessments. 1.3.3 Feedback from managment strategy into OM: extending model years The catch for the next \\(n\\) years before the next assessment is then added to the OM, as well as any recruitment or time varying parameter deviations. The OM is again run with no estimation where it can be used to produce sampled data for the next \\(n\\) years. These new data values are appended to the original dataset. 1.3.4 Subsequent runs of the management strategy The appended data set is then used in the managment strategy again, and new catch by fleet is produced that can then be fed back to the OM. "],["simple.html", "2 A simple example 2.1 Setup R workspace folders 2.2 Create the operating models (OMs) 2.3 Adding process error through recruitment deviations and time-varying selectivity 2.4 Examine the management procedure used 2.5 Run SSMSE 2.6 run_SSMSE output 2.7 Performance metrics 2.8 Summarize results 2.9 Simple Convergence Check 2.10 Plot Spawning Stock Biomass (SSB) 2.11 Example MSE Results 2.12 Delete the files", " 2 A simple example Suppose we want to look at how well we are able to achieve a performance metric under uncertainty in the operating model (OM). We will look 2 scenarios, one where Steepness (h) is specified correctly and one where it is specified incorrectly in an estimation model (EM): Scenario 1. h-ctl: Cod operating model (h = 0.65) with correctly specified cod model EM (fixed h = 0.65). The OM is the same as the EM. Scenario 2. h-1: Cod operating model (h = 1) with misspecified cod model EM (fixed h = 0.65); The OM is not the same as the EM. Note that this is a simple example where the OM and EM structures for both scenarios are identical, except for different steepness between the OM and EM in scenario 2 and some process error we will include in the operating model. We will assume we want to run the MSE loop for 6 years, with a stock assessment occuring every 3 years (and forecasting catch to maintain 40% of unfished spawning stock biomass). The cod models last year is 100, so the OM is initially conditioned through year 100. Then, after conditioning the operating model through year 100, assessments will occur in years 100 and 103. The operating model runs through year 106. We chose not to run the assessment in year 106, as there was no need for its output in this example. 2.1 Setup R workspace folders First, we will load the SSMSE package and create a folder in which to run the example: library(SSMSE) #load the package library(r4ss) #install using remotes::install_github("r4ss/r4ss) library(foreach) #if using run_parallel = TRUE library(doParallel) #if using run_parallel = TRUE # Create a folder for the output in the working directory. run_SSMSE_dir <- file.path("run_SSMSE-ex") dir.create(run_SSMSE_dir) 2.2 Create the operating models (OMs) 2.2.1 Specify alternative values for h The cod model with h = 0.65 (as in scenario 1) is included as external package data in SSMSE. However, we will need to modify it to use as an operating model with h = 1 (as in scenario 2). Note in this case that refit_OM is false, so the model is not being refit, just run through without fitting. To condition the new model on the same data as the input model, refit_OM should be TRUE. First, we identify where the base cod model is stored, modify it such that the steepness parameter is 1, and save the modified cod OM for scenario 2 in a new folder in the run_SSMSE_dir directory. cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE") # develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir # specified develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep", par_vals = 1, refit_OMs = FALSE, hess = FALSE) # OM model for scenario 2 cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1") 2.3 Adding process error through recruitment deviations and time-varying selectivity Recruitment deviations, implementation error, and changes in parameters in the projection period of the OM can be added through the future_om_list input to run_SSMSE. First, well set up the list to add recruitment deviations in the projection period. The same recruitment deviation patterns are used across scenarios, but different patterns are use across iterations in the same scenario. We also want these deviations to have the same standard deviations as the historical deviations with 0 mean (the assumed default). # Start from a list created by a helper function template_mod_change <- create_future_om_list() # add recruitment deviations rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" # apply change to rec devs rec_dev_specify$scen <- c("replicate", "all") # using 1 to 100 means the sd or mean will be calculated by taking the sd across years # from 1 to 100 rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" # this change is for the sd # no input value needed since it will be calclated from the historical rec devs. rec_dev_specify$input$value <- NA rec_dev_specify $pars [1] "rec_devs" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 1 100 100 101 sd absolute NA Next, suppose we want to allow selectivity to vary annually for 1 selectivity parameter of the fishery throughout the projection period. The following specifies that the value for selectivity varies randomly around the base value with a sd of 0.2. # put together the change for selectivity (random values around the orig val, with # an sd of 0.2) mod_change_sel <- template_mod_change[[1]] mod_change_sel$scen[2] <- "all" # apply to all scenarios # The following 2 lines suggest that this change is immediately applied in year # 101, with no transitory period for using sd 0 to the new sd. # historical values are NA in this case, because they are not used to determine # the sd to use. mod_change_sel$input$last_yr_orig_val <- 100 mod_change_sel$input$first_yr_final_val <- 101 mod_change_sel$input$ts_param <- "sd" # this change is for the sd mod_change_sel$input$value <- 0.2 # se to use in the projection period mod_change_sel $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "all" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 100 101 sd absolute 0.2 Finally, add these two changes together into an object to pass to run_SSMSE future_om_list_recdevs_sel <- list(rec_dev_specify, mod_change_sel) 2.3.1 Add observation error through sampling from OM The argument sample_struct specifies the structure for sampling from the OM (and passing to the EM). The function create_sample_struct can be used to construct a simple sampling structure consistent with an input data file: datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this dataframe. sample_struct_1_scen $catch Yr Seas FltSvy SE 1 101 1 1 0.005 2 102 1 1 0.005 3 103 1 1 0.005 4 104 1 1 0.005 5 105 1 1 0.005 6 106 1 1 0.005 $CPUE Yr Seas FltSvy SE 1 105 7 2 0.2 $lencomp Yr Seas FltSvy Sex Part Nsamp 1 NA 1 1 0 0 125 $agecomp Yr Seas FltSvy Sex Part Ageerr Lbin_lo Lbin_hi Nsamp 1 105 1 2 0 0 1 -1 -1 500 $meanbodywt [1] NA $MeanSize_at_Age_obs [1] NA By default, create_sample_struct identifies sampling patterns from the historical period of the OM and replicates those patterns in the projection period. In our cod example, the sample structure specifies that catch will be added to the estimation model every year (years 101 to 106), but an index of abundance (i.e., CPUE) and age composition (i.e., agecomp) will only be added in year 105. We will use the same sampling scheme for both scenarios, but it is possible to specify different sampling for each scenario. The user could modify this sampling strategy (for example, maybe age composition should also be sampled from FltSvy 2 in Yr 102; the user could add another line to the dataframe in sample_struct$agecomp). Note that length comp (lencomp) includes an NA value for year. This is because no consistent pattern was identified, so the user must define their own input. In this case, we will remove sampling length comps all together: sample_struct_1_scen$lencomp <- NULL # don't use length sampling The same sampling structure will be used for both scenarios, which we define in a list below: sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen) 2.4 Examine the management procedure used We will use the same management procedure for both scenarios: Conduct a stock assessment every 3 years to get stock status. Project from this stock assessment using the SS forecast file to get projected future catch. Put this projected catch (without implementation error, in the case of this example) back into the OM. Extend the OM forward in time to get the true values for the population. Lets take a look at step 2 in the management procedure, which is implemented using the forecasting module in SS. We will examine the forecast file for the estimation model to better understand how catches will be forecasted from the assessment. We will use the same management procedure in both of these scenarios, although for a full MSE analysis, it is likely that multiple management procedures would be compared. fore <- r4ss::SS_readforecast( system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"), verbose = FALSE) fore$Forecast [1] 3 fore$Btarget [1] 0.4 fore$Forecast = 3 means our forecasts from the assessment will use fishing mortality (F) to attmpt to achieve a relative (to unfished) spawning stock biomass. Based on fore$Btarget, the relative biomass target is 40% of unfished spawning stock biomass. Note also that the control rule fore$BforconstantF and fore$BfornoF values are set low to make it unlikely that they will be used (these parameters are used for a ramp harvest control rule, which we do not want to use here): fore$BforconstantF [1] 0.03 fore$BfornoF [1] 0.01 Futhermore, fore$Flimitfraction is set to 1 so that the forecasted catch is set equal to the overfishing limit (for simplicity): fore$Flimitfraction [1] 1 Note that the number of forecast years is 1: fore$Nforecastyrs [1] 1 However, an assessment will be conducted every 3 years and thus 3 years of projections is required. SSMSE will automatically modify this value in the estimation model to the appropriate number of forecasting years. More information on using the forecast module in SS to forecast catches is available in the Stock Synthesis users manual. Users can also specify their own [custom management procedures] 2.5 Run SSMSE Now, we create a directory to store our results, and use run_SSMSE to run the MSE analysis loop (note this will take some time to run, ~ 20 min): run_res_path <- file.path(run_SSMSE_dir, "results") dir.create(run_res_path) res <- run_SSMSE( scen_name_vec = c("h-ctl", "h-1"),# name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = c(5,5), # run with 5 iterations each OM_name_vec = NULL, # specify directories instead OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files EM_name_vec = c("cod", "cod"), # cod is included in package data MS_vec = c("EM","EM"), # The management strategy is specified in the EM nyrs_vec = c(6, 6), # Years to project OM forward nyrs_assess_vec = c(3, 3), # Years between assessments future_om_list = future_om_list_recdevs_sel, run_parallel = TRUE, # Run iterations in parallel sample_struct_list = sample_struct_list_all, # How to sample data for running the EM. sample_struct_hist_list = NULL, # because this is null, will just use sampling # as in the current OM data file for the historical period. seed = 12345) #Set a fixed integer seed that allows replication See ?run_SSMSE for more details on function arguments. In a real MSE analysis, running 100+ iterations to reflect the full range of uncertainty (given observation and process errors) in the results would be preferred. However, we are only running 5 iterations per scenario in this demonstration to reduce computing time. 2.6 run_SSMSE output run_SSMSE will create new folders in the folders specified in out_dir_scen_vec (note that in this case, we are running both scenarios in the same folder). After is complete, there will be a folder for each scenario in run_res_path (since out_dir_scen_vec = run_res_path in this example). Within each scenario is a folder for each scenario. And within each scenario folder, there are folders containing the SS models that were run by run_SSMSE. There should be 1 folder for the OM, which is run multiple times in this same folder during the MSE analysis. There are multiple folders for the EMs, as a new folder is created each time an assessment is done. The first run is the folder with a name ending in init; then, each assessment after is named for the updated end year of the model. With many iterations, the number of files adds up; in the future, we hope to add options to save less output. 2.7 Performance metrics Quantitative performance metrics should be specified before conducting an MSE. Typically, a suite of performance metrics will be examined; however, for simplicity in this example, we will only look at what the achieved relative biomass was for the last 3 years of projection in the MSE to determine how it compares to the intended management target of 40% of unfished Spawning Stock Biomass. Note that we are only running our MSE projection for 6 years, but longer projections are typical in MSE analyses. 2.8 Summarize results The function SSMSE_summary_all can be used to summarize the model results in a list of 3 dataframes, one for scalar outputs (named scalar), one for timeseries outputs (ts), one for derived quantities (dq). This function also creates summary csv files in the folder where the results are stored. # Summarize 1 iteration of output summary <- SSMSE_summary_all(run_res_path) ## Extracting results from 2 scenarios ## Starting h-1 with 5 iterations ## Starting h-ctl with 5 iterations Plotting and data manipulation can then be done with these summaries. For example, SSB over time by model can be plotted. The models include the Operating Model (cod_OM), Estimation model (EM) for the historical period of years 0-100 (cod_EM_init), and the EM run with last year of data in year 103 (cod_EM_103). The operating models are shown in blue or black (depending on the scenario), and the estimation model runs are shown in orange, and the scenarios are shown on different subplots: library(ggplot2) # use install.packages("ggplot2") to install package if needed library(tidyr) # use install.packages("tidyr") to install package if needed library(dplyr) # use install.packages("dplyr") to install package if needed Attaching package: 'dplyr' The following objects are masked from 'package:stats': filter, lag The following objects are masked from 'package:base': intersect, setdiff, setequal, union 2.9 Simple Convergence Check Check there are no params on bounds or SSB that is way too small or way too large check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) { require(dplyr) # note: not the best way to do this if(any(!is.na(summary$scalar$params_on_bound))) { warning("Params on bounds") } else { message("No params on bounds") } summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM") calc_SSB <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% select(iteration, scenario, year, model_run, model_type, SpawnBio) OM_vals <- calc_SSB %>% filter(model_type == "OM") %>% rename(SpawnBio_OM = SpawnBio ) %>% select(iteration, scenario, year, SpawnBio_OM) EM_vals <- calc_SSB %>% filter(model_type == "EM") %>% rename(SpawnBio_EM = SpawnBio) %>% select(iteration, scenario, year, model_run, SpawnBio_EM) bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM) filter_SSB <- bind_vals %>% filter(SSB_ratio > 2 | SSB_ratio < 0.5) if(nrow(filter_SSB) > 0 ) { warning("Some large/small SSBs relative to OM") } else { message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM") } return_val <- bind_vals } values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5) No params on bounds All SSBs in EM are no greater than double and no less than half SSB vals in the OM 2.10 Plot Spawning Stock Biomass (SSB) This plot shows that SSB estimated does not match perfectly with the operating model. A similar plot could be made for any parameter of interest. # plot SSB by year and model run ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), ggplot2::aes(x = year, y = SpawnBio)) + ggplot2::geom_vline(xintercept = 100, color = "gray") + ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+ ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) + ggplot2::scale_linetype_manual(values = rep("solid", 50)) + ggplot2::guides(linetype = FALSE) + ggplot2::facet_wrap(. ~ scenario) + ggplot2::theme_classic() Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> = "none")` instead. Now, we calculate and plot the performance metric, which is average spawning stock biomass (SSB) from years 104 to 106. # get_SSB_avg calculates the SSB in each year for each # iteration of the operating model, then takes the average over the years from # min_yr, to max_year. It uses the summary object as input to do these # calculations. get_SSB_avg <- function(summary, min_yr, max_yr) { OM_vals <- unique(summary$ts$model_run) OM_vals <- grep("_OM$", OM_vals, value = TRUE) SSB_yr <- summary$ts %>% filter(year >= min_yr & year <= max_yr) %>% filter(model_run %in% OM_vals) %>% select(iteration, scenario, year, SpawnBio) %>% group_by(iteration, scenario) %>% summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% ungroup() SSB_yr } avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106) # function to summarize data in plot data_summary <- function(x) { m <- mean(x) ymin <- m - sd(x) ymax <- m + sd(x) return(c(y = m, ymin = ymin, ymax = ymax)) } # Now, plot the average relative spawning stock biomass for years 104 - 106 ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) + stat_summary(fun.data = data_summary, position = position_dodge(width = 0.9), color = "blue") + labs(title = "Long-term average SSB\\n(years 104-106)", x = "Scenario", y = "SSB") + theme_classic() From the above plot, we see differences in the average SSb between the 2 scenarios. 2.11 Example MSE Results We can see from the performance metric that mis-specifying the value of steepness will results in higher realized relative spawning stock biomass than correctly specifying it. This gives us some idea of the consequences of misspecifying steepness in the stock assessment. 2.12 Delete the files If you wish to delete the files created from this example, you can use: unlink(run_SSMSE_dir, recursive = TRUE) "],["advanced-options-use-a-custom-management-strategyprocedure.html", "3 Advanced options: use a custom management strategy/procedure", " 3 Advanced options: use a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } The function should be created in a separate file. In this case, assume this function is available in a file custom_funs.R. This function can then be used in a call to run_SSMSE: run_result_custom <- run_SSMSE(scen_name_vec = "constant-catch", # name of the scenario out_dir_scen_vec = run_res_path, # directory in which to run the scenario iter_vec = 1, OM_name_vec = "cod", # specify directories instead OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use the custom function custom_MS_source = "custom_funs.R", # File where the custom function is available. nyrs_vec = 6, # Years to project OM forward nyrs_assess_vec = 3, # Years between assessments future_om_list = future_om_list_recdevs_sel, sample_struct_list = list(sample_struct_list[[1]]), # How to sample data for running the MS. seed = 12345) #Set a fixed integer seed that allows replication "],["SSMSE.html", "4 Options for run_SSMSE 4.1 Scenarios in SSMSE 4.2 Operating model 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) 4.4 Sampling options 4.5 Future changes to the operating model (#future)", " 4 Options for run_SSMSE Many inputs are possible for the run_SSMSE() option. Here, we will describe some of the options available. For detailed documentation, type ?SSMSE::run_SSMSE() into the R console. 4.1 Scenarios in SSMSE Note that multiple scenarios can be called in run_SSMSE(), often through vector inputs to run_SSMSE(). Below, we will describe inputs needed to run 1 scenario. 4.2 Operating model The operating model (OM) for SSMSE should be a Stock Synthesis model. This could be any fitted Stock Synthesis model, except for models that use Empirical Weight at age. There is one built-in OM that comes with SSMSE, which is a cod-like model. To use the cod model in run_SSMSE, set OM_name_vec = \"cod\". Otherwise, the path to the OM model should be specified in OM_in_dir_vec. 4.3 The Management Strategy/procedure and (if applicable) Estimation model (EM) The management strategy (and EM) can be specified in one of two ways: Using an SS model and its forcast file to project catch by fleet Using a custom procedure via a function in R In theory, any management strategy should work, as long as it can take the data file produced by the OM as output and provide back to SSMSE future catches by fleet. 4.3.1 Specify the Management Strategy in a SS model An SS model can be set up as the EM for the MSE. To use this option, specify \"EM\" as part of MS_vec. As with the OM, the built-in cod model could be used; just specify \"cod\" in the EM_name_vec. To use any other SS model as the EM, specify the path in EM_in_dir_vec.Note that models that use empirical weight at age can not yet be used as estimation models. Future catches will be determined from the forecasting file settings of the SS model. SSMSE will change the number of forecast years to match the number of years between assessments, but other specifications need to be made by the user. 4.3.2 Using a custom management strategy/procedure Users can outline a custom managment strategy as an R function to use. As long as the correct inputs and outputs are used, any estimation method and management procedure can be used. For example, here is a simple function that just sets future catches as half the sampled catches in a specified year: constant_catch_MS <- function(OM_dat, nyrs_assess, catch_yr = 100, frac_catch = 0.5, ...) { # need to include ... to allow function to work # set catch the same as the previous year (sampled catch). # catch is in the same units as the operating model, in this case it is in # biomass. catch <- data.frame( year = (OM_dat$endyr + 1):(OM_dat$endyr + nyrs_assess), # the years to project the model forward seas = 1, # hard coded from looking at model fleet = 1, # hard coded from looking at model catch = OM_dat$catch[OM_dat$catch$year == catch_yr, "catch"]*frac_catch, catch_se = 0.05) # hard coded from looking at model catch_bio <- catch # catch in biomass. In this case, catch is in biomass for both. Could also be left as NULL catch_F <- NULL # catch in terms of F, can be left as NULL. discards <- NULL # discards can be left as NULL if there are no discards catch_list <- list(catch = catch, catch_bio = catch_bio, catch_F = catch_F, discards = discards) } Lets assume this function is saved in a file within the working directory named constant_catch_MS.R. This function can then be used in a call to run_SSMSE(): # define sample structure datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE") sample_struct <- create_sample_struct(dat = datfile, nyrs = 6) # note warning sample_struct$lencomp <- NULL # don't use length sampling # run the SSMSE routine run_result_custom <- run_SSMSE( scen_name_vec = "constant-catch", out_dir_scen_vec = "my_results", iter_vec = 1, OM_name_vec = "cod", OM_in_dir_vec = NULL, MS_vec = "constant_catch_MS", # use custom fun custom_MS_source = "constant_catch_MS.R", use_SS_boot_vec = TRUE, nyrs_vec = 6, nyrs_assess_vec = 3, run_EM_last_yr = FALSE, run_parallel = FALSE, sample_struct_list = list(sample_struct), seed = 12345) 4.4 Sampling options Currently, the only available sampling option is to use the bootstrapping module within SS itself. This means specifying use_SS_boot_vec = TRUE. Details on how sampling is done using the bootstrapping module in SS is available in the Bootstrap Data Files section of the SS user manual. Users also need to specify how and which data types should be sampled for each future year in the simulation in sample_struct_list. sample_struct_list is a list of lists. The first level is a list for each scenario; then, for the scenario, there is a list of dataframes, each of which specifying which years and fleets of data should be sampled in the future as well as which standard errors or sample sizes should be used. The helper function create_sample_struct() can be used to help users generate the list of dataframes for a scenario. See an example of this functions use in the simple example or by typing ?SSMSE::create_sample_struct() into the R console. Currently, users can sample data for catches (treated as fixed), CPUE (i.e., indices of abundance), length and age composition, conditional length at age compositions, mean size at age, and mean size. It is not yet possible to sample data for generalized size compositions, tagging data, and morph compositions. Users can specify the sampling during the historical period of the model through the sample_struct_hist input to run_SSMSE. This is an optional input and has the same structure as sample_struct_list, but the years will be before the original end year of the OM. 4.5 Future changes to the operating model (#future) By default, SSMSE will simply extend the structure of the OM into the future using the same parameter values as in the last year of the model. However, the user may want to allow changes in parameter values to occur as the OM is extended forward in time. Users can input values into the future_om_list to accomplish this. This input is a list of lists. For the first level of lists, each represents a separate change to make. Within the first level, there are 4 list components that outline the details of the change to make. There are 2 main choices: 1) to specify model changes by telling SSMSE how values should be sampled or 2) to input your own custom values for future values of a given parameter. 4.5.1 The strucutre of future_om_list For example, here is an example list containing just one future change for the model. It shows that the model should be changed to 4.5 in year 103 and afterwards: my_future_om_list <- create_future_om_list(list_length = 1) my_future_om_list [[1]] [[1]]$pars [1] "SizeSel_P_3_Fishery(1)" [[1]]$scen [1] "replicate" "scen2" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 103 mean absolute 4.5 length(my_future_om_list) # note has length 1 b/c 1 change [1] 1 length(my_future_om_list[[1]]) # has length 4 because 4 list components, as for each change [1] 4 Note there is just one change specified here. For the change, there are four list items that are required for any specified change. The first list item is named pars. It contains a vector of parameter name(s) to apply the change to. The names should be the same as the names in r4ss::SS_read_pars() or can include the following: \"rec_devs\" - For specifying recruitment deviations \"impl_error - For specifying implementation error in transforming a management procedures specified future catch into realized catch. \"all\" - Apply the change to all parameters in the model, with the exception of SR_sigmaR, SR_regime, SR_autocorr, and impl_error parameters. Changing the first 3 stock recruitment related parameters would conflict with changes in recruitment deviations. Since implementation error is not a parameter specified in the control file and SSMSE does not rely on the implementation error parameter created through the forecasting file, including the implementation error in all did not seem appropriate. In this case, we just want to apply the change to the SizeSel_P_3_Fishery(1) parameter of the model. The second item is scen, which contains a vector of information about how to apply the changes within and across scenarios. The first value is an option to specify how the change should be applied among scenarios, either randomize (use a different value for each iteration of each scenario) or replicate (use the same set of values across scenarios for the same number iteration, but each value will be different across iterations within the same scenario). Note that the same values will be applied across iterations and scenarios if there isnt any stochasticity in the values being drawn (e.g., standard deviation set at 0), regardless if randomize or replicate is chosen. In the example above, there is no stochasticity, so specifying randomize or replicate does not matter. Following the first value are the names of the scenarios to apply the change to. If the change should be applied to all of the scenarios, all can be used in place of naming every scenario. In this example, The change will only be applied to the scenario named scen2, so the input for scen is c(\"randomize\", \"scen2\"). The pattern is a vector of character inputs. The first value should be either model_change, meaning that SSMSE will calculate the change values, or custom which allows the user to directly put in the values that they want in the model. In this case, model_change is used. A second input can also be added when the model_change pattern is used, which specifies the distribution to pull from. This could be normal or lognormal, but if no input is provided, a normal distribution will be used for sampling. The fourth item, named input, is a dataframe, which contains different column name values depending on if the model_change pattern is used or if the custom pattern is used. For the model_change options, a dataframe in input specifies a change for the parameter of a distribution. Because we are using the model change option, we have the following columns: first_yr_averaging: The first year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_averaging: The last year to average historical values from the model, if using for the change. This should be NA if historical averaging will not be used. last_yr_orig_val: The last year of the future deviations with the original model value. This value will not be changed, but the following years value will be. first_yr_final_val: The first year where the final value as specified will be reached. If no future changes are made, the final value will continue forward in the OM through all projected years. For step changes, the first_yr_final_val is just 1 year after the last_yr_orig_val. However, if a more gradual adjustment toward the final value is desired, this could be any number of years after the original value. ts_param: The sampling parameter to modify. Options are mean, ar_1_phi (if using autocorrelation), sd, and cv. If not included in the dataframe, mean is assumed to be the same as in the previous model year, sd is assumed to be 0, and we assume no autocorrelation (as specified through an autoregressive AR1 process). Phi should be between -1 and 1 (this creates a stationary process) and if phi = 1, this creates a random walk (as random walk is a special case of an AR1 model). Note that both sd and cv cannot be specified within the same input dataframe. method: How to apply the change relative to the historical (if specified) or previous years value (if first_yr_averaging and last_yr_averaging for that row are NA). Options are multiplicative, additive, and absolute. This example uses absolute, meaning that the number in value is directly used. value. The value of the parameter change. This may be NA if historical averaging is used. 4.5.2 Example of future_om_list denoting a gradual change Suppose we wanted to change the value of SizeSel_P_3_Fishery(1) in scen2 over 4 years after year 102 to arrive at a value of 4.5 in year 106. We can do this by using my_future_om_list, but changing the value in the first row of first_yr_final_val to 106: my_future_om_list[[1]][["input"]][1, "first_yr_final_val"] <- 106 my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 106 mean absolute 4.5 4.5.3 Example of future_om_list with a random deviations Suppose we now wanted the value of SizeSel_P_3_Fishery(1) to change randomly according to a normal distribution with a standard deviation of 0.1 around a mean of 4.5 from 104 onwards. This can be done by adding a line specifying a change in standard deviation (which for now, has been assumed to be 0) to the data frame: new_vals <- data.frame(first_yr_averaging = NA, last_yr_averaging = NA, last_yr_orig_val = 103, first_yr_final_val = 104, ts_param = "sd", method = "absolute", value = 0.1) my_future_om_list[[1]][["input"]] <- rbind(my_future_om_list[[1]][["input"]], new_vals) my_future_om_list[[1]] $pars [1] "SizeSel_P_3_Fishery(1)" $scen [1] "replicate" "scen2" $pattern [1] "model_change" $input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 NA NA 102 106 mean absolute 4.5 2 NA NA 103 104 sd absolute 0.1 Note that the last_yr_orig_val and first_yr_final_val are different than the line for the mean, which is allowed. 4.5.4 Example of using historical values for determining parameter values This example applies random annual deviations to all parameters for scenarios scen2 and scen3. future_om_list_2 <- vector(mode = "list", length = 1) future_om_list_2 <- lapply(future_om_list_2, function (x) x <- vector(mode = "list", length = 4)) names(future_om_list_2[[1]]) <- c("pars", "scen", "pattern", "input") future_om_list_2[[1]][["pars"]] <- "all" future_om_list_2[[1]][["scen"]] <- c("randomize", "scen2", "scen3") future_om_list_2[[1]][["pattern"]] <- "model_change" # defaults to using normal dist future_om_list_2[[1]][["input"]] <- data.frame(first_yr_averaging = c(1, 1), last_yr_averaging = c(100, 100), last_yr_orig_val = c(100, 100), first_yr_final_val = c(101, 101), ts_param = c("cv", "mean"), method = c("absolute", "multiplier"), value = c(0.1, 1)) Note that the choice of year range for historical values does not matter unless the parameter is already time-varying in the original operating model or has become time varying through a previous change. Otherwise, the base model parameter will be applied. If no historical years are included, then the base parameter value will be the basis of comparison for relative changes (i.e., method = multiplier or additive). 4.5.5 Example using custom pattern instead of model_change custom_future_om_list <- create_future_om_list(example_type = "custom", list_length = 1) custom_future_om_list [[1]] [[1]]$pars [1] "impl_error" [[1]]$scen [1] "randomize" "all" [[1]]$pattern [1] "custom" [[1]]$input par scen iter yr value 1 impl_error scen1 1 101 1.05 2 impl_error scen1 2 101 1.05 3 impl_error scen1 3 101 1.05 4 impl_error scen1 4 101 1.05 5 impl_error scen1 5 101 1.05 6 impl_error scen1 1 102 1.05 7 impl_error scen1 2 102 1.05 8 impl_error scen1 3 102 1.05 9 impl_error scen1 4 102 1.05 10 impl_error scen1 5 102 1.05 11 impl_error scen1 1 103 1.05 12 impl_error scen1 2 103 1.05 13 impl_error scen1 3 103 1.05 14 impl_error scen1 4 103 1.05 15 impl_error scen1 5 103 1.05 16 impl_error scen1 1 104 1.05 17 impl_error scen1 2 104 1.05 18 impl_error scen1 3 104 1.05 19 impl_error scen1 4 104 1.05 20 impl_error scen1 5 104 1.05 21 impl_error scen1 1 105 1.05 22 impl_error scen1 2 105 1.05 23 impl_error scen1 3 105 1.05 24 impl_error scen1 4 105 1.05 25 impl_error scen1 5 105 1.05 26 impl_error scen1 1 106 1.05 27 impl_error scen1 2 106 1.05 28 impl_error scen1 3 106 1.05 29 impl_error scen1 4 106 1.05 30 impl_error scen1 5 106 1.05 31 impl_error scen2 1 101 1.10 32 impl_error scen2 2 101 1.10 33 impl_error scen2 3 101 1.10 34 impl_error scen2 4 101 1.10 35 impl_error scen2 5 101 1.10 36 impl_error scen2 1 102 1.10 37 impl_error scen2 2 102 1.10 38 impl_error scen2 3 102 1.10 39 impl_error scen2 4 102 1.10 40 impl_error scen2 5 102 1.10 41 impl_error scen2 1 103 1.10 42 impl_error scen2 2 103 1.10 43 impl_error scen2 3 103 1.10 44 impl_error scen2 4 103 1.10 45 impl_error scen2 5 103 1.10 46 impl_error scen2 1 104 1.10 47 impl_error scen2 2 104 1.10 48 impl_error scen2 3 104 1.10 49 impl_error scen2 4 104 1.10 50 impl_error scen2 5 104 1.10 51 impl_error scen2 1 105 1.10 52 impl_error scen2 2 105 1.10 53 impl_error scen2 3 105 1.10 54 impl_error scen2 4 105 1.10 55 impl_error scen2 5 105 1.10 56 impl_error scen2 1 106 1.10 57 impl_error scen2 2 106 1.10 58 impl_error scen2 3 106 1.10 59 impl_error scen2 4 106 1.10 60 impl_error scen2 5 106 1.10 61 impl_error scen3 1 101 1.10 62 impl_error scen3 2 101 1.10 63 impl_error scen3 3 101 1.10 64 impl_error scen3 4 101 1.10 65 impl_error scen3 5 101 1.10 66 impl_error scen3 1 102 1.10 67 impl_error scen3 2 102 1.10 68 impl_error scen3 3 102 1.10 69 impl_error scen3 4 102 1.10 70 impl_error scen3 5 102 1.10 71 impl_error scen3 1 103 1.10 72 impl_error scen3 2 103 1.10 73 impl_error scen3 3 103 1.10 74 impl_error scen3 4 103 1.10 75 impl_error scen3 5 103 1.10 76 impl_error scen3 1 104 1.10 77 impl_error scen3 2 104 1.10 78 impl_error scen3 3 104 1.10 79 impl_error scen3 4 104 1.10 80 impl_error scen3 5 104 1.10 81 impl_error scen3 1 105 1.10 82 impl_error scen3 2 105 1.10 83 impl_error scen3 3 105 1.10 84 impl_error scen3 4 105 1.10 85 impl_error scen3 5 105 1.10 86 impl_error scen3 1 106 1.10 87 impl_error scen3 2 106 1.10 88 impl_error scen3 3 106 1.10 89 impl_error scen3 4 106 1.10 90 impl_error scen3 5 106 1.10 The inputs required for pattern = custom are similar to using pattern = model_change, but there is no need to specify a distribution as the second vector input of pattern and the column names in input are different. Also that the change is randomized, which indicates that a value for each scenario and each iteration are necessary, whereas if the first value of scen was replicate, separate values for each scenario should not be specified, but rather a single value for all should be used. The columns in the dataframe are: par: the parameter name or all if it should be applied to all parameters in the pars input scen: which scenario the value should apply to, or all if replicate is used as the scenario input iter: which number iteration the value should apply to. These are absolute iteration numbers. yr: the year the value applies to. These should be after the original end year of the OM. value: the value to apply to the model 4.5.6 How is the operating modified to accomodate changes as specified in the future_OM_list? All changes are made by converting the parameter(s) with changes to be time varing by using additive annual parameter deviations. Because this is an operating model, Stock Synthesis is not run with estimation, so to get the changes into the OM, values (drawn or calculated in model_changes or specified by the user using custom) are directly input as parameter deviations into the ss.par file. If an OM already contains time varying parameters, these parameters will also be converted to additive parameter deviations before applying the changes specified in the future_OM_list. 4.5.7 Example of specifying recruitment deviations This shows some code for putting recruitment deviations with a mean of 0 and the same standard deviation as the historical recruitment deviations in years 1 to 100: template_mod_change <- create_future_om_list(list_length = 1) rec_dev_specify <- template_mod_change[[1]] rec_dev_specify$pars <- "rec_devs" rec_dev_specify$scen <- c("replicate", "all") rec_dev_specify$input$first_yr_averaging <- 1 rec_dev_specify$input$last_yr_averaging <- 100 rec_dev_specify$input$last_yr_orig_val <- 100 rec_dev_specify$input$first_yr_final_val <- 101 rec_dev_specify$input$ts_param <- "sd" rec_dev_specify$input$value <- NA rec_dev_list <- list(rec_dev_specify) rec_dev_list [[1]] [[1]]$pars [1] "rec_devs" [[1]]$scen [1] "replicate" "all" [[1]]$pattern [1] "model_change" [[1]]$input first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param method value 1 1 100 100 101 sd absolute NA "],["helper.html", "5 Helper functions in SSMSE 5.1 Creating multiple operating models with develop_OMs() 5.2 Set up sampling for a scenario with create_sample_struct() 5.3 Get examples of the future_om_list input with create_future_om_list()", " 5 Helper functions in SSMSE run_SSMSE() requires some detailed inputs that are time consuming to make. In the interest of automating routines as much as possible, helper functions are available in SSMSE. 5.1 Creating multiple operating models with develop_OMs() This function allows users to change a parameter in a Stock Synthesis OM and, if desired, refit the operating model to data. Typically, users will want to create multiple operating models to use in different scenarios in order to account for uncertainties in parameter values, such as biological parameters relating to natural mortality or growth. 5.2 Set up sampling for a scenario with create_sample_struct() run_SSMSE() requires the input sample_struct, which outlines the future sampling structure to use to generate data sets. This object could be difficult to create manually and users may want to just continue sampling as previously done in an existing SS model. create_sample_struct() will use the time patterns in sampling for different data types found in an SS data file and extend it forward in time. If no pattern is found, the function will return NAs and provide a warning to the user. If any NAs are returned (except to indicate that an entire data type is missing), the user will need to remove or fill in the NA value before usign the list as an input to run_SSMSE(). See use of create_sample_struct() in simple example. 5.3 Get examples of the future_om_list input with create_future_om_list() Users can modify the future structure of their operating model through the future_om_list input to run_SSMSE. The structure and names of this object matter, so examples are provided by calling create_future_om_list(). See more details about the specification of future_om_list objects. "],["output.html", "6 Output and Plots 6.1 Summarizing output 6.2 Checking estimation model convergence 6.3 Calculating performance metrics", " 6 Output and Plots 6.1 Summarizing output Output is summarized using SSMSE_summary_all(): summary_list <- SSMSE_summary_all(dir = "path/to/scenario/dir", scenarios = c("sample_low", "sample_high"), run_parallel = TRUE) Relying on ss3sim::get_results_all(), this function creates: For each scenario, 3 scenario level .csv files For all scenarios, 2 cross-scenario .csv files named by default to SSMSE_tsand SSMSE_scalar. For all scenarios, the function returns a list object containing data frames of timeseries (ts), scalar, and derived quantities (dq) summaries. Note that run_parallel = TRUE is only faster than run_parallel FALSE when there is more than once scenario and none of the scenario-level .csv files have been created yet. By default, if a user doesnt specify scenarios, all scenarios in dir will be summarized. 6.2 Checking estimation model convergence One of the first checks to do if using an estimation model after running an MSE analysis is to check that the estimation model has converged. A number of checks could be done, but a basic one is checking the gradients for the estimation model, which are added to the SSMSE_scalar summary sheet. 6.3 Calculating performance metrics Typically, a suite of performance metrics are used in MSE. Punt et al. (2016) recommends at least using metrics related to: average catch variation in catch over time population size "],["plotting-output-and-performance-metrics.html", "7 Plotting output and performance metrics", " 7 Plotting output and performance metrics Currently, it has been left up to the user to plot summaries and performance metrics, as the potential options for plots and performance metrics users may desire are extensive. There are currently diagnostic plots to examine how sampled data compares to the operating model values and data sets used to condition the operating models. plot_index_sampling() creates a diagnostic plot for indices, while plot_comp_sampling() creates diagnostic plots for composition data. "],["glossary-of-mse-terms.html", "8 Glossary of MSE terms", " 8 Glossary of MSE terms This glossary is based off of Rademeyer et al. (2007) and Punt et al. (2016). Assessment error - Error that occurs during the process of conducting an assessment, specifically error which inform the catch control rule that is being evaluated using the MSE: management advice for any system is based on uncertain data. Consequently, the data that inform catch control rules need to be generated in a manner which is as realistic as possible. Uncertainty arises when the model used for conducting assessments and providing management advice differs from the operating model, or the data are too noisy to estimate all key parameters reliably (Punt et al. 2016). Assessment model (AM) - A fitted and well scrutinized Stock Synthesis stock assessment model. The AM is input for SSMSE so that it can build the operating model. Bootstrapped dataset - After running an SS model (and if the user turns on the option in the starter file), the output file data.ss_new contains a dataset that has the same form as the input data.ss file, but instead of the input values, contains a sampled dataset. This is derived from using the expected values given the parameter estimates and model structure (see glossary entry on Expected values), uncertainty either estimated or input in the model, and an assumed distribution for sampling. As many bootstrapped data sets as the user desires can be output from an SS model. These bootstrap datasets are the third or greater set of values in the data.ss_new file. Estimation model (EM) - This refers to the model used within the MSE procedure to represent the stock assessment process. Note that an estimation model proxy could also be used to represent the stock assessment process. Expected values - After running an SS model (and if the user turns on teh option in the starter file), the output file data.ss_new contains a dataset that has the same form as the input data.ss file, but instead of the input values, contains the expected values given the parameter estimates and model structure. This expected values dataset is the second set of values in the data.ss_new file. Implementation error - Also called implementation uncertainty or outcome uncertainty. Broadly, this includes error of implementing the management action(s), which is often not done perfectly. Definition from Punt et al. (2016): The most obvious form of this type of uncertainty is when catches are not the same as the TACs typically more is taken or the decision-makers do not implement the TACs suggested by the management strategy. However, there are many other sources of outcome uncertainty, such as that associated with catch limits set for recreational fisheries and regulating discards. Index - An index of abundance. In SS files, this is sometimes used interchangably with CPUE. Plural, indices. Management strategy - Following Punt et al. (2016), there are 2 types: model-based management strategies and empirical management strategies. Ideally, SSMSE will allow for users to use either of these management strategies. Model-based management strategies include conducting a stock assessment and using output for determining harvest control rules. Empirical management strategies do not involve conducting a stock assessment but rather setting regulations from data (although summarization is possible). In some cases, management strategies are computationally intensive/time consuming to formally include in the context of an MSE; to speed up the process, it is commen to use management strategy proxies, typically an assumed error distribution about the operating model values. Management strategies are also known as management procedures. Model uncertainty - Definition from Punt et al. (2016): the form of relationships within an operating model will always be subject to uncertainty. The simplest type of model uncertainty involves, for example, whether the stockrecruitment relationship is BevertonHolt or Ricker, whether a fixed value for a model parameter is correct, or whether fishery selectivity is asymptotic or dome-shaped. However, there are other more complicated types of model structure uncertainty such as how many stocks are present in the area modelled, the error structure of the data used for assessment purposes, the impact of future climate change on biological relationships such as the stockrecruitment function, and ecosystem impacts on biological and fishery processes. Observation error - Error that results from not observing the true dynamics of the system. See also bootstrapped dataset. Operating model (OM) - This is a Stock Synthesis model that defines the assumed true dynamics of the population and its associated fisheries for the purposes of management strategy evaluation. Most often more than one operating model is necessary in order to adequately characterize the uncertainty in the true dynamics of the system. The Stock Synthesis OM may also define how sampling from the true dynamics is done, as SS produces expected values and, if desired, bootstrapped data sets based on sampling assumptions implicit in the model. For more information, see the glossary entries on Expected values and bootstrapped dataset Parameter uncertainty - Definition from Punt et al. (2016): many operating models are fit to the data available, but the values estimated for the parameters of those operating models (e.g. fishery selectivity-at-age, the parameters of the stockrecruitment relationship and historical deviations in recruitment about the stockrecruitment relationship) are subject to error. Performance statistics - Definition from Rademeyer et al. (2007): Statistics that summarize different aspects of the results of a simulation trial used to evaluate how well a specific [management strategy] achieves some or all of the general objectives for management for a particular scenario. Performance statistics usually fall into one of three categories: catch-related, stability related, or risk related. Process uncertainty - Definition from Punt et al. (2016): variation (usually assumed to be random, though sometimes incorporating autocorrelation) in parameters often considered fixed in stock assessments such as natural mortality, future recruitment about a stockrecruitment relationship and selectivity. Stock Synthesis (SS, SS3) - A integrative, general population dynamics modeling program used to assess the effects of fishing on population. Available at the Stock Synthesis website Uncertainty - Incorporating uncertainty into an MSE procedure is extremely important. There are several potential sources of uncertainty, which we have divided as done in Punt et al. (2016) in addition to adding observation error. For more details, see the glossary entries on Process Uncertainty, Parameter uncertainty, Model uncertainty, Assessment error, Implementation error, and observation error. Punt et al. (2016) suggest that MSE should at least consider process uncertainty (particularly deviations from the stock recruitment relationship), parameter uncertainty (particularly which relates to productivity and stock size), and observation error. Note that Rademeyer et al. (2007) divides error into estimation error, implementation error, observation error and process error, which could be used instead. These may be more natural divisions in sources of error. "],["404.html", "Page not found", " Page not found The page you requested cannot be found (perhaps it was moved or renamed). You may want to try searching to find the page's new location, or use the table of contents to find the page you are looking for. "]] diff --git a/docs/manual/simple.html b/docs/manual/simple.html index 9eb82f9b..9454b02d 100644 --- a/docs/manual/simple.html +++ b/docs/manual/simple.html @@ -24,7 +24,7 @@ - + @@ -239,7 +239,6 @@

    2.1 Setup R workspace folders

    # Create a folder for the output in the working directory.
 run_SSMSE_dir <- file.path("run_SSMSE-ex")
 dir.create(run_SSMSE_dir)
-
Warning in dir.create(run_SSMSE_dir): 'run_SSMSE-ex' already exists

2.2 Create the operating models (OMs)

@@ -247,39 +246,37 @@

2.2 Create the operating models (

2.2.1 Specify alternative values for h

The cod model with h = 0.65 (as in scenario 1) is included as external package data in SSMSE. However, we will need to modify it to use as an operating model with h = 1 (as in scenario 2). Note in this case that refit_OM is false, so the model is not being refit, just run through without fitting. To condition the new model on the same data as the input model, refit_OM should be TRUE.

First, we identify where the base cod model is stored, modify it such that the steepness parameter is 1, and save the modified cod OM for scenario 2 in a new folder in the run_SSMSE_dir directory.

-
cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE")
-# develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir
-# specified
-develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep",
-            par_vals = 1, refit_OMs = FALSE, hess = FALSE)
-
Warning in file.rename(from = file.path(out_dir, basename(OM_in_dir)), to = tmp_mod_path): cannot rename file 'run_SSMSE-
-ex/cod' to 'run_SSMSE-ex/cod_SR_BH_steep_1', reason 'Access is denied'
-
# OM model for scenario 2
-cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1")
+
cod_mod_path <- system.file("extdata", "models", "cod", package = "SSMSE")
+# develop_OMs will save a model called "cod_SR_BH_steep_1" in the out_dir
+# specified
+develop_OMs(OM_name = "cod", out_dir = run_SSMSE_dir, par_name = "SR_BH_steep",
+            par_vals = 1, refit_OMs = FALSE, hess = FALSE)
+# OM model for scenario 2
+cod_1_path <- file.path(run_SSMSE_dir, "cod_SR_BH_steep_1")

2.3 Adding process error through recruitment deviations and time-varying selectivity

Recruitment deviations, implementation error, and changes in parameters in the projection period of the OM can be added through the future_om_list input to run_SSMSE.

First, we’ll set up the list to add recruitment deviations in the projection period. The same recruitment deviation patterns are used across scenarios, but different patterns are use across iterations in the same scenario. We also want these deviations to have the same standard deviations as the historical deviations with 0 mean (the assumed default).

-
# Start from a list created by a helper function
-template_mod_change <- create_future_om_list() 
-# add recruitment deviations
-rec_dev_specify <- template_mod_change[[1]]
-rec_dev_specify$pars <- "rec_devs" # apply change to rec devs
-rec_dev_specify$scen <- c("replicate", "all")
-# using 1 to 100 means the sd or mean will be calculated by taking the sd across years
-# from 1 to 100
-rec_dev_specify$input$first_yr_averaging <- 1
-rec_dev_specify$input$last_yr_averaging <- 100
-# The following 2 lines suggest that this change is immediately applied in year
-# 101, with no transitory period for using sd 0 to the new sd.
-rec_dev_specify$input$last_yr_orig_val <- 100
-rec_dev_specify$input$first_yr_final_val <- 101
-rec_dev_specify$input$ts_param <- "sd" # this change is for the sd
-# no input value needed since it will be calclated from the historical rec devs.
-rec_dev_specify$input$value <- NA
-rec_dev_specify
+
# Start from a list created by a helper function
+template_mod_change <- create_future_om_list() 
+# add recruitment deviations
+rec_dev_specify <- template_mod_change[[1]]
+rec_dev_specify$pars <- "rec_devs" # apply change to rec devs
+rec_dev_specify$scen <- c("replicate", "all")
+# using 1 to 100 means the sd or mean will be calculated by taking the sd across years
+# from 1 to 100
+rec_dev_specify$input$first_yr_averaging <- 1
+rec_dev_specify$input$last_yr_averaging <- 100
+# The following 2 lines suggest that this change is immediately applied in year
+# 101, with no transitory period for using sd 0 to the new sd.
+rec_dev_specify$input$last_yr_orig_val <- 100
+rec_dev_specify$input$first_yr_final_val <- 101
+rec_dev_specify$input$ts_param <- "sd" # this change is for the sd
+# no input value needed since it will be calclated from the historical rec devs.
+rec_dev_specify$input$value <- NA
+rec_dev_specify
$pars
 [1] "rec_devs"
 
@@ -293,19 +290,19 @@ 
2.3 Adding process error through
   first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param   method value
 1                  1               100              100                101       sd absolute    NA

Next, suppose we want to allow selectivity to vary annually for 1 selectivity parameter of the fishery throughout the projection period. The following specifies that the value for selectivity varies randomly around the base value with a sd of 0.2.

-
# put together the change for selectivity (random values around the orig val, with
-# an sd of 0.2)
-mod_change_sel <- template_mod_change[[1]]
-mod_change_sel$scen[2] <- "all" # apply to all scenarios
-# The following 2 lines suggest that this change is immediately applied in year
-# 101, with no transitory period for using sd 0 to the new sd.
-# historical values are NA in this case, because they are not used to determine
-# the sd to use.
-mod_change_sel$input$last_yr_orig_val <- 100
-mod_change_sel$input$first_yr_final_val <- 101
-mod_change_sel$input$ts_param <- "sd" # this change is for the sd
-mod_change_sel$input$value <- 0.2 # se to use in the projection period
-mod_change_sel
+
# put together the change for selectivity (random values around the orig val, with
+# an sd of 0.2)
+mod_change_sel <- template_mod_change[[1]]
+mod_change_sel$scen[2] <- "all" # apply to all scenarios
+# The following 2 lines suggest that this change is immediately applied in year
+# 101, with no transitory period for using sd 0 to the new sd.
+# historical values are NA in this case, because they are not used to determine
+# the sd to use.
+mod_change_sel$input$last_yr_orig_val <- 100
+mod_change_sel$input$first_yr_final_val <- 101
+mod_change_sel$input$ts_param <- "sd" # this change is for the sd
+mod_change_sel$input$value <- 0.2 # se to use in the projection period
+mod_change_sel
$pars
 [1] "SizeSel_P_3_Fishery(1)"
 
@@ -319,15 +316,16 @@ 
2.3 Adding process error through
   first_yr_averaging last_yr_averaging last_yr_orig_val first_yr_final_val ts_param   method value
 1                 NA                NA              100                101       sd absolute   0.2

Finally, add these two changes together into an object to pass to run_SSMSE

-
future_om_list_recdevs_sel <- list(rec_dev_specify, 
-                                   mod_change_sel)
+
future_om_list_recdevs_sel <- list(rec_dev_specify, 
+                                   mod_change_sel)

2.3.1 Add observation error through sampling from OM

The argument sample_struct specifies the structure for sampling from the OM (and passing to the EM). The function create_sample_struct can be used to construct a simple sampling structure consistent with an input data file:

-
datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE")
-sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning
-
Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this dataframe.
-
sample_struct_1_scen
+
datfile <- system.file("extdata", "models", "cod", "ss3.dat", package = "SSMSE")
+sample_struct_1_scen <- create_sample_struct(dat = datfile, nyrs = 6) # note warning
+
Warning in FUN(X[[i]], ...): Pattern not found for lencomp: FltSvy 1, Seas 1. Returning NA for Yr in this
+dataframe.
+
sample_struct_1_scen
$catch
    Yr Seas FltSvy    SE
 1 101    1      1 0.005
@@ -358,9 +356,9 @@ 
2.3.1 Add observation error throu
 
Note that length comp (lencomp) includes an NA value for year. This is because
 no consistent pattern was identified, so the user must define their own input.
 In this case, we will remove sampling length comps all together:

-
sample_struct_1_scen$lencomp <- NULL # don't use length sampling

+
sample_struct_1_scen$lencomp <- NULL # don't use length sampling

 
The same sampling structure will be used for both scenarios, which we define in a list below:

-
sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen)

+
sample_struct_list_all <- list("h-ctl" = sample_struct_1_scen, "h-1" = sample_struct_1_scen)
@@ -372,23 +370,23 @@

2.4 Examine the management proced
  • Put this projected catch (without implementation error, in the case of this example) back into the OM. Extend the OM forward in time to get the true values for the population.

    • Let’s take a look at step 2 in the management procedure, which is implemented using the forecasting module in SS. We will examine the forecast file for the estimation model to better understand how catches will be forecasted from the assessment. We will use the same management procedure in both of these scenarios, although for a full MSE analysis, it is likely that multiple management procedures would be compared.

    -
    fore <- r4ss::SS_readforecast(
-  system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"),
-  verbose = FALSE)
-fore$Forecast
    +
    fore <- r4ss::SS_readforecast(
+  system.file("extdata", "models", "cod", "forecast.ss", package = "SSMSE"),
+  verbose = FALSE)
+fore$Forecast 
    [1] 3
    -
    fore$Btarget
    +
    fore$Btarget
    [1] 0.4

    fore$Forecast = 3 means our forecasts from the assessment will use fishing mortality (F) to attmpt to achieve a relative (to unfished) spawning stock biomass. Based on fore$Btarget, the relative biomass target is 40% of unfished spawning stock biomass. Note also that the control rule fore$BforconstantF and fore$BfornoF values are set low to make it unlikely that they will be used (these parameters are used for a ramp harvest control rule, which we do not want to use here):

    -
    fore$BforconstantF
    +
    fore$BforconstantF
    [1] 0.03
    -
    fore$BfornoF
    +
    fore$BfornoF
    [1] 0.01

    Futhermore, fore$Flimitfraction is set to 1 so that the forecasted catch is set equal to the overfishing limit (for simplicity):

    -
    fore$Flimitfraction
    +
    fore$Flimitfraction
    [1] 1

    Note that the number of forecast years is 1:

    -
    fore$Nforecastyrs
    +
    fore$Nforecastyrs
    [1] 1

    However, an assessment will be conducted every 3 years and thus 3 years of projections is required. SSMSE will automatically modify this value in the estimation model to the appropriate number of forecasting years.

    More information on using the forecast module in SS to forecast catches is available in the Stock Synthesis users manual.

    @@ -397,24 +395,24 @@

    2.4 Examine the management proced

    2.5 Run SSMSE

    Now, we create a directory to store our results, and use run_SSMSE to run the MSE analysis loop (note this will take some time to run, ~ 20 min):

    -
    run_res_path <- file.path(run_SSMSE_dir, "results")
-dir.create(run_res_path)
-res <- run_SSMSE(
-    scen_name_vec = c("h-ctl", "h-1"),# name of the scenario
-    out_dir_scen_vec = run_res_path, # directory in which to run the scenario
-    iter_vec = c(5,5), # run with 5 iterations each
-    OM_name_vec = NULL, # specify directories instead
-    OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files
-    EM_name_vec = c("cod", "cod"), # cod is included in package data
-    MS_vec = c("EM","EM"), # The management strategy is specified in the EM
-    nyrs_vec = c(6, 6), # Years to project OM forward
-    nyrs_assess_vec = c(3, 3), # Years between assessments
-    future_om_list = future_om_list_recdevs_sel,
-    run_parallel = TRUE, # Run iterations in parallel
-    sample_struct_list = sample_struct_list_all, # How to sample data for running the EM.
-    sample_struct_hist_list = NULL, # because this is null, will just use sampling
-    # as in the current OM data file for the historical period.
-    seed = 12345) #Set a fixed integer seed that allows replication
    +
    run_res_path <- file.path(run_SSMSE_dir, "results")
+dir.create(run_res_path)
+res <- run_SSMSE(
+    scen_name_vec = c("h-ctl", "h-1"),# name of the scenario
+    out_dir_scen_vec = run_res_path, # directory in which to run the scenario
+    iter_vec = c(5,5), # run with 5 iterations each
+    OM_name_vec = NULL, # specify directories instead
+    OM_in_dir_vec = c(cod_mod_path, normalizePath(cod_1_path)), # OM files
+    EM_name_vec = c("cod", "cod"), # cod is included in package data
+    MS_vec = c("EM","EM"), # The management strategy is specified in the EM
+    nyrs_vec = c(6, 6), # Years to project OM forward
+    nyrs_assess_vec = c(3, 3), # Years between assessments
+    future_om_list = future_om_list_recdevs_sel,
+    run_parallel = TRUE, # Run iterations in parallel
+    sample_struct_list = sample_struct_list_all, # How to sample data for running the EM.
+    sample_struct_hist_list = NULL, # because this is null, will just use sampling
+    # as in the current OM data file for the historical period.
+    seed = 12345) #Set a fixed integer seed that allows replication

    See ?run_SSMSE for more details on function arguments. In a real MSE analysis, running 100+ iterations to reflect the full range of uncertainty (given observation and process errors) in the results would be preferred. However, we are only running 5 iterations per scenario in this demonstration to reduce computing time.

    @@ -431,15 +429,15 @@

    2.7 Performance metrics

    2.8 Summarize results

    The function SSMSE_summary_all can be used to summarize the model results in a list of 3 dataframes, one for scalar outputs (named scalar), one for timeseries outputs (ts), one for derived quantities (dq). This function also creates summary csv files in the folder where the results are stored.

    -
    # Summarize 1 iteration of output
-summary <- SSMSE_summary_all(run_res_path)
-## Extracting results from 2 scenarios
-## Starting h-1 with 5 iterations
-## Starting h-ctl with 5 iterations
    +
    # Summarize 1 iteration of output
+summary <- SSMSE_summary_all(run_res_path)
+## Extracting results from 2 scenarios
+## Starting h-1 with 5 iterations
+## Starting h-ctl with 5 iterations

    Plotting and data manipulation can then be done with these summaries. For example, SSB over time by model can be plotted. The models include the Operating Model (cod_OM), Estimation model (EM) for the historical period of years 0-100 (cod_EM_init), and the EM run with last year of data in year 103 (cod_EM_103). The operating models are shown in blue or black (depending on the scenario), and the estimation model runs are shown in orange, and the scenarios are shown on different subplots:

    -
    library(ggplot2) # use install.packages("ggplot2") to install package if needed
-library(tidyr) # use install.packages("tidyr") to install package if needed
-library(dplyr) # use install.packages("dplyr") to install package if needed
    +
    library(ggplot2) # use install.packages("ggplot2") to install package if needed
+library(tidyr) # use install.packages("tidyr") to install package if needed
+library(dplyr) # use install.packages("dplyr") to install package if needed
    
 Attaching package: 'dplyr'
    The following objects are masked from 'package:stats':
@@ -452,88 +450,88 @@ 
    2.8 Summarize results
    
 
    
 
    2.9 Simple Convergence Check
    
 
    Check there are no params on bounds or SSB that is way too small or way too large
    
-
    check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) {
-  require(dplyr) # note: not the best way to do this
-  if(any(!is.na(summary$scalar$params_on_bound))) {
-    warning("Params on bounds")
-  } else {
-    message("No params on bounds")
-  }
-  summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM")
-  calc_SSB <- summary$ts %>%
-    filter(year >= min_yr & year <= max_yr) %>% 
-    select(iteration, scenario, year, model_run, model_type, SpawnBio)
-  OM_vals <- calc_SSB %>% 
-              filter(model_type == "OM") %>% 
-              rename(SpawnBio_OM = SpawnBio ) %>% 
-              select(iteration, scenario, year, SpawnBio_OM)
-  EM_vals <- calc_SSB %>% 
-               filter(model_type == "EM") %>% 
-               rename(SpawnBio_EM = SpawnBio) %>% 
-               select(iteration, scenario, year, model_run, SpawnBio_EM)
-  bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% 
-                  mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM)
-  filter_SSB <- bind_vals %>% 
-    filter(SSB_ratio > 2 | SSB_ratio < 0.5)
-  if(nrow(filter_SSB) > 0 ) {
-    warning("Some large/small SSBs relative to OM")
-  } else {
-    message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM")
-  }
-  return_val <- bind_vals
-}
-values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5)
    
+
    check_convergence <- function(summary, min_yr = 101, max_yr = 120, n_EMs = 5) {
+  require(dplyr) # note: not the best way to do this
+  if(any(!is.na(summary$scalar$params_on_bound))) {
+    warning("Params on bounds")
+  } else {
+    message("No params on bounds")
+  }
+  summary$ts$model_type <- ifelse(grepl("_EM_", summary$ts$model_run), "EM", "OM")
+  calc_SSB <- summary$ts %>%
+    filter(year >= min_yr & year <= max_yr) %>% 
+    select(iteration, scenario, year, model_run, model_type, SpawnBio)
+  OM_vals <- calc_SSB %>% 
+              filter(model_type == "OM") %>% 
+              rename(SpawnBio_OM = SpawnBio ) %>% 
+              select(iteration, scenario, year, SpawnBio_OM)
+  EM_vals <- calc_SSB %>% 
+               filter(model_type == "EM") %>% 
+               rename(SpawnBio_EM = SpawnBio) %>% 
+               select(iteration, scenario, year, model_run, SpawnBio_EM)
+  bind_vals <- full_join(EM_vals, OM_vals, by = c("iteration", "scenario", "year")) %>% 
+                  mutate(SSB_ratio = SpawnBio_EM/SpawnBio_OM)
+  filter_SSB <- bind_vals %>% 
+    filter(SSB_ratio > 2 | SSB_ratio < 0.5)
+  if(nrow(filter_SSB) > 0 ) {
+    warning("Some large/small SSBs relative to OM")
+  } else {
+    message("All SSBs in EM are no greater than double and no less than half SSB vals in the OM")
+  }
+  return_val <- bind_vals
+}
+values <- check_convergence(summary = summary, min_yr = 101, max_yr = 106, n_EMs = 5)
    
 
    No params on bounds
    
 
    All SSBs in EM are no greater than double and no less than half SSB vals in the OM
    
 
    
 
    
 
    2.10 Plot Spawning Stock Biomass (SSB)
    
 
    This plot shows that SSB estimated does not match perfectly with the operating model. A similar plot could be made for any parameter of interest.
    
-
    # plot SSB by year and model run
-ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), 
-                ggplot2::aes(x = year, y = SpawnBio)) +
-  ggplot2::geom_vline(xintercept = 100, color = "gray") +
-  ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+
-  ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) +
-  ggplot2::scale_linetype_manual(values = rep("solid", 50)) +
-  ggplot2::guides(linetype = FALSE) +
-  ggplot2::facet_wrap(. ~ scenario) +
-  ggplot2::theme_classic()
    
+
    # plot SSB by year and model run
+ggplot2::ggplot(data = subset(summary$ts, model_run %in% c("cod_OM", "cod_SR_BH_steep_1_OM", "cod_EM_103")), 
+                ggplot2::aes(x = year, y = SpawnBio)) +
+  ggplot2::geom_vline(xintercept = 100, color = "gray") +
+  ggplot2::geom_line(ggplot2::aes(linetype = as.character(iteration), color = model_run))+
+  ggplot2::scale_color_manual(values = c("#D65F00", "black", "blue")) +
+  ggplot2::scale_linetype_manual(values = rep("solid", 50)) +
+  ggplot2::guides(linetype = FALSE) +
+  ggplot2::facet_wrap(. ~ scenario) +
+  ggplot2::theme_classic()
    
 
    Warning: `guides(<scale> = FALSE)` is deprecated. Please use `guides(<scale> = "none")` instead.
    
 
    
 
    Now, we calculate and plot the performance metric, which is average spawning stock biomass (SSB) from years 104 to 106.
    
-
    # get_SSB_avg calculates the SSB in each year for each
-# iteration of the operating model, then takes the average over the years from
-# min_yr, to max_year. It uses the summary object as input to do these
-# calculations.
-get_SSB_avg <- function(summary, min_yr, max_yr) {
-  OM_vals <- unique(summary$ts$model_run)
-  OM_vals <- grep("_OM$", OM_vals, value = TRUE)
-  SSB_yr <- summary$ts %>% 
-          filter(year >= min_yr & year <= max_yr) %>% 
-          filter(model_run %in% OM_vals) %>% 
-          select(iteration, scenario, year, SpawnBio) %>% 
-          group_by(iteration, scenario) %>% 
-          summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% 
-          ungroup()
-  SSB_yr
-}
-avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106)
-
-# function to summarize data in plot
-data_summary <- function(x) {
-  m <- mean(x)
-  ymin <- m - sd(x)
-  ymax <- m + sd(x)
-  return(c(y = m, ymin = ymin, ymax = ymax))
-}
-# Now, plot the average relative spawning stock biomass for years 104 - 106
-ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) +
-  stat_summary(fun.data = data_summary, 
-               position = position_dodge(width = 0.9), color = "blue") +
-  labs(title = "Long-term average  SSB\n(years 104-106)", 
-       x = "Scenario", y = "SSB") +
-  theme_classic()
    
+
    # get_SSB_avg calculates the SSB in each year for each
+# iteration of the operating model, then takes the average over the years from
+# min_yr, to max_year. It uses the summary object as input to do these
+# calculations.
+get_SSB_avg <- function(summary, min_yr, max_yr) {
+  OM_vals <- unique(summary$ts$model_run)
+  OM_vals <- grep("_OM$", OM_vals, value = TRUE)
+  SSB_yr <- summary$ts %>% 
+          filter(year >= min_yr & year <= max_yr) %>% 
+          filter(model_run %in% OM_vals) %>% 
+          select(iteration, scenario, year, SpawnBio) %>% 
+          group_by(iteration, scenario) %>% 
+          summarize(avg_SSB = mean(SpawnBio), .groups = "keep") %>% 
+          ungroup()
+  SSB_yr
+}
+avg_SSB <- get_SSB_avg(summary, min_yr = 104, max_yr = 106)
+
+# function to summarize data in plot
+data_summary <- function(x) {
+  m <- mean(x)
+  ymin <- m - sd(x)
+  ymax <- m + sd(x)
+  return(c(y = m, ymin = ymin, ymax = ymax))
+}
+# Now, plot the average relative spawning stock biomass for years 104 - 106
+ggplot(data = avg_SSB, aes(x = scenario, y = avg_SSB)) +
+  stat_summary(fun.data = data_summary, 
+               position = position_dodge(width = 0.9), color = "blue") +
+  labs(title = "Long-term average  SSB\n(years 104-106)", 
+       x = "Scenario", y = "SSB") +
+  theme_classic()
    
 
    
 
    From the above plot, we see differences in the average SSb between the 2 scenarios.
    
 
    
@@ -544,7 +542,7 @@ 
    2.11 Example MSE Results
    
 
    
 
    2.12 Delete the files
    
 
    If you wish to delete the files created from this example, you can use:
    
-
    unlink(run_SSMSE_dir, recursive = TRUE)
    
+
    unlink(run_SSMSE_dir, recursive = TRUE)

    From 25b61599d7ee80849361655a6bffdb10044c3bed Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Thu, 7 Oct 2021 08:26:35 -0400 Subject: [PATCH 09/10] fix: export create_scen_list function Used in tests --- NAMESPACE | 1 + R/utils.R | 1 + 2 files changed, 2 insertions(+) diff --git a/NAMESPACE b/NAMESPACE index 5367b450..02be978c 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -3,6 +3,7 @@ export(SSMSE_summary_all) export(create_future_om_list) export(create_sample_struct) +export(create_scen_list) export(develop_OMs) export(get_bin) export(parse_MS) diff --git a/R/utils.R b/R/utils.R index 3f25c799..021291a3 100644 --- a/R/utils.R +++ b/R/utils.R @@ -66,6 +66,7 @@ #' expected value with the reference value identifying the lag in years for availability #' of the index (Ref_years: vector length n indices). #' @author Kathryn Doering +#' @export #' @examples #' scen_list <- create_scen_list( #' scen_name_vec = c("scen 1", "scen_2"), From 16cb9ab3c0bc4df885d8449a9a22129e35d48749 Mon Sep 17 00:00:00 2001 From: Kathryn Doering Date: Thu, 7 Oct 2021 09:14:22 -0400 Subject: [PATCH 10/10] fix: example errors --- NAMESPACE | 1 + R/utils.R | 9 +++++---- man/get_input_value.Rd | 8 ++++---- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/NAMESPACE b/NAMESPACE index 02be978c..e778823e 100644 --- a/NAMESPACE +++ b/NAMESPACE @@ -12,6 +12,7 @@ export(plot_index_sampling) export(run_EM) export(run_SSMSE) export(run_ss_model) +export(set_MSE_seeds) import(ggplot2) import(parallel) import(r4ss) diff --git a/R/utils.R b/R/utils.R index 021291a3..fc308d90 100644 --- a/R/utils.R +++ b/R/utils.R @@ -329,14 +329,14 @@ clean_init_mod_files <- function(OM_out_dir, EM_out_dir = NULL, MS = "EM", #' "value" = c(2, 2, 2, 3, 3), #' "se_log" = 0.2 #' ) -#' get_input_value( +#' SSMSE:::get_input_value( #' data = dfr, method = "most_common_value", colname = "se_log", #' group = "value" #' ) -#' get_input_value(data = dfr, method = "most_common_value", colname = "value") -#' get_input_value(data = dfr, method = "only_value", colname = "se_log") +#' SSMSE:::get_input_value(data = dfr, method = "most_common_value", colname = "value") +#' SSMSE:::get_input_value(data = dfr, method = "only_value", colname = "se_log") #' # generates an error: -#' # get_input_value(data = dfr, method = "only_value", colname = "value") +#' # SSMSE:::get_input_value(data = dfr, method = "only_value", colname = "value") get_input_value <- function(data, method = "most_common_value", colname, @@ -699,6 +699,7 @@ combine_cols <- function(dat_list, list_item, colnames) { #' Can be 1) NULL (default); 2) An integer vector of length 1, length 1+length(scen_name_vec), or length 1 + length(scen_name_vec)+sum(iter_vec); 3) A list with 3 components the same as teh output of set_MSE_seeds #' @param iter_vec The number of iterations per scenario. A vector of integers #' in the same order as scen_name_vec. +#' @export #' @returns A list of length 3 with 1) the global seed value; 2) the scenario seed values; and 3) the iteration seed values. #' @examples #' seeds <- set_MSE_seeds(seed = seq(10, 80, by = 10), iter_vec = c(2, 3)) diff --git a/man/get_input_value.Rd b/man/get_input_value.Rd index 3cf16edb..3a8746f6 100644 --- a/man/get_input_value.Rd +++ b/man/get_input_value.Rd @@ -40,14 +40,14 @@ dfr <- data.frame( "value" = c(2, 2, 2, 3, 3), "se_log" = 0.2 ) -get_input_value( +SSMSE:::get_input_value( data = dfr, method = "most_common_value", colname = "se_log", group = "value" ) -get_input_value(data = dfr, method = "most_common_value", colname = "value") -get_input_value(data = dfr, method = "only_value", colname = "se_log") +SSMSE:::get_input_value(data = dfr, method = "most_common_value", colname = "value") +SSMSE:::get_input_value(data = dfr, method = "only_value", colname = "se_log") # generates an error: -# get_input_value(data = dfr, method = "only_value", colname = "value") +# SSMSE:::get_input_value(data = dfr, method = "only_value", colname = "value") } \author{ Kathryn Doering