cwrc_migration_batch.drush.inc

<?php

/**
 * @file
 * Custom CWRC drush scripts.
 */

/**
 * Implements hook_drush_command().
 */
function cwrc_migration_batch_drush_command() {
  $items = array();

  $items['cwrc_migration_batch_ingest_biography_writing'] = array(
    'description' => 'Ingest objects CWRC',
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_biography_writing path_to_directory name_of_mods_directory name_of_orlando_directory collection_pid cModel_pid mods_to_dc_xsl default_XACML_policy_stream',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_ingest_entities'] = array(
    'description' => 'Ingest objects CWRC',
     'options' => array(
      'input_dir' => 'path to directory of source material',
      'collection_pid' => 'cModel pid assigments',
      'cModel_pid' => 'cModel pid assigments',
      'namespace' => 'pid namespace',
      'content_datastream_dsid' => 'content datastream id',
      'content_datastream_label' => 'content datastream name',
      'entity_to_dc_xslt_path' => 'convert entity to DC',
      "wf_category" => "workflow category",
      'wf_stamp' => 'workflow stamp',
      'wf_status' => 'workflow status',
      "wf_note" => "workflow note",
    ),
   'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_entities path_to_directory name_of_default_workflow_file collection_pid cModel_pid content_datastream_dsid content_datastream_label entity_to_dc_xslt_path',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_ingest_cwrc_documents'] = array(
    'description' => 'Ingest objects - CWRC Documents',
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_cwrc_documents path_to_directory name_of_default_workflow_file collection_pid cModel_pid content_datastream_dsid content_datastream_label content_to_mods_xslt_path mods_to_dc_xslt_path',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_ingest_events_documentation'] = array(
    'description' => 'Ingest objects CWRC',
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_events_documentation path_to_directory name_of_mods_directory name_of_orlando_directory collection_pid cModel_pid mods_to_dc_xsl default_XACML_policy_stream',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_ingest_bibliography'] = array(
    'description' => 'Ingest objects CWRC',
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_bibliography path_to_directory name_of_mods_directory name_of_legacy_directory collection_pid cModel_pid mods_to_dc_xsl default_XACML_policy_stream',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_export_book'] = array(
    'description' => 'Export book objects objects CWRC',
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_export_book path_to_list_of_book_PIDs destination_path',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_reindex_list'] = array(
    'description' => 'Solr index objects',
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_reindex_list path_to_list_of_book_PIDs',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  # 2019-12-19: events only? 
  $items['cwrc_migration_batch_ingest_orlando_biography_writing'] = array(
    'description' => 'Ingest Orlando Biography/Writing/event objects into CWRC',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'entity_mapping_file' => 'path to the JSON PID to standard name entity mapping file used to add URIs to bio/writing/event',
      'xslt_to_v2' => 'convert legacy bio/writ docs to version 2 (convert dates, utf-8)',
      'xslt_to_mods' => 'path to XSLT to convert biography/writing/event to mods',
      'xslt_mods_to_dc' => 'path to XSLT to convert mods to DC',
      'collection_pid_pub_c' => 'add object to the given collection if PUC-C workflow',
      'collection_pid_non_pub_c' => 'add object to the given collection if no PUB-C workflow',
      'cModel_pid' => 'cModel pid assigments',
      'schema_object' => 'schema object pid assigments',
      'XACML_policy' => 'path to the default XACML policy',
      'namespace' => 'pid namespace',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_orlando_biography_writing ',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  # 2019-12-19: combine bio/writ 
  $items['cwrc_migration_batch_ingest_orlando_combine_biography_writing'] = array(
    'description' => 'Ingest Orlando Biography/Writing objects into CWRC',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'entity_mapping_file' => 'path to the JSON PID to standard name entity mapping file used to add URIs to bio/writing/event',
      'xslt_to_v2' => 'convert legacy bio/writ docs to version 2 (convert dates, utf-8)',
      'xslt_to_v3' => 'update combined documents to version 3 (header)',
      'xslt_to_mods' => 'path to XSLT to convert biography/writing/event to mods',
      'xslt_mods_to_dc' => 'path to XSLT to convert mods to DC',
      'collection_pid_pub_c' => 'add object to the given collection if PUC-C workflow',
      'collection_pid_non_pub_c' => 'add object to the given collection if no PUB-C workflow',
      'cModel_pid' => 'cModel pid assigments',
      'schema_object' => 'schema object pid assigments',
      'XACML_policy' => 'path to the default XACML policy',
      'namespace' => 'pid namespace',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_orlando_combine_biography_writing ',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  # 2019-12-19: name/org/bibl 
  $items['cwrc_migration_batch_ingest_orlando_entities'] = array(
    'description' => 'Ingest Orlando Entity objects into CWRC',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'entity_mapping_file' => 'path to the JSON PID to standard name entity mapping file used to add URIs to bio/writing/event',
      'xslt_to_v2' => 'convert legacy Orlando docs to version 2 (convert dates, utf-8)',
      'xslt_to_dc' => 'path to XSLT to convert mods to DC',
      'main_ds_id' => 'Id of the main datastream (where content XML stored)',
      'collection_pid_pub_c' => 'add object to the given collection if PUC-C workflow',
      'collection_pid_non_pub_c' => 'add object to the given collection if no PUB-C workflow',
      'cModel_pid' => 'cModel pid assigments',
      'XACML_policy' => 'path to the default XACML policy',
      'namespace' => 'pid namespace',
      'supplemental_filename' => 'supplemental content used to augment content',
      'add_pub_c_workflow' => 'add an additional workflow activity - pub-c',
      'entity_type' => 'define the entity type (based on XML element name)',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_orlando_entities',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  # 2020-10-23: bibl - reingest bibl reuse existing pids - DON'T use without entity mapping file
  $items['cwrc_migration_batch_ingest_orlando_entities_reuse_pids'] = array(
    'description' => 'Ingest Orlando Entity objects into CWRC reusing PIDs',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'entity_mapping_file' => 'path to the JSON PID to standard name entity mapping file used to add URIs to bio/writing/event',
      'xslt_to_v2' => 'convert legacy Orlando docs to version 2 (convert dates, utf-8)',
      'xslt_to_dc' => 'path to XSLT to convert mods to DC',
      'main_ds_id' => 'Id of the main datastream (where content XML stored)',
      'collection_pid_pub_c' => 'add object to the given collection if PUC-C workflow',
      'collection_pid_non_pub_c' => 'add object to the given collection if no PUB-C workflow',
      'cModel_pid' => 'cModel pid assigments',
      'XACML_policy' => 'path to the default XACML policy',
      'namespace' => 'pid namespace',
      'supplemental_filename' => 'supplemental content used to augment content',
      'add_pub_c_workflow' => 'add an additional workflow activity - pub-c',
      'entity_type' => 'define the entity type (based on XML element name)',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_orlando_entities_reuse_pids',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_ingest_ceww_entries'] = array(
    'description' => 'Ingest CEWW Entry objects into CWRC',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'xslt_to_mods' => 'path to XSLT to convert entriy to mods',
      'xslt_mods_to_dc' => 'path to XSLT to convert mods to DC',
      'main_ds_id' => 'Id of the main datastream (where content XML stored)',
      'collection_pid' => 'add object to the given collection if PUC-C workflow',
      'cModel_pid' => 'cModel pid assigments',
      'schema_object' => 'schema object pid assigments',
      'XACML_policy' => 'path to the default XACML policy',
      'namespace' => 'pid namespace',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_ceww_entries',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_ceww_add_subject'] = array(
    'description' => 'CEWW Entry: update to add subject',
    'options' => array(
      'pid_file' => 'PIDs to modify',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ceww_add_subject',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_ingest_generic'] = array(
    'description' => 'Ingest objects into CWRC',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'xslt_to_mods' => 'path to XSLT to convert entriy to mods',
      'xslt_to_dc' => 'path to XSLT to convert mods to DC',
      'main_ds_id' => 'Id of the main datastream (where content XML stored)',
      'collection_pid' => 'add object to the given collection if PUC-C workflow',
      'cModel_pid' => 'cModel pid assigments',
      'schema_object' => 'schema object pid assigments',
      'XACML_policy' => 'path to the default XACML policy',
      'namespace' => 'pid namespace',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_ingest_generic',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_validate'] = array(
    'description' => 'A naive attempt to validate cwrc:documentCModel objects against a schema',
    'options' => array(
      'collection_pid' => 'pid of the collection',
      'ds_id' => 'datastream id containing the content to schema validate',
      'override_rng_schema' => 'schema to use if the schema can not be determined from the xml-model processing instruction',
      'validation_url' => 'url of the validation service: https://validator.services.cwrc.ca/validate.html',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_validate',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_verify_orlando'] = array(
    'description' => 'A naive attempt to verify the Orlando batch ingest',
    'options' => array(
      'legacy_dir' => 'path to directory of legacy source material',
      'entity_mapping_file' => 'path to the JSON PID to standard name entity mapping file used to add URIs to bio/writing/event',
      'main_ds_id' => 'Id of the main datastream (where content XML stored)',
      'item_type' => 'Type of the input: EVENT, ENTRY, NAME, ORGNAME, BIBCIT',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_verify_orlando',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_thumbnail_entries'] = array(
    'description' => 'Add a custom thumbnail to the specified entries',
    'options' => array(
      'pid_file' => 'PIDs to modify thumbnails',
      'tn_path' => 'path to the thumbnail image',
      'tn_label' => 'thumbnail image label',
      'ds_id' => 'datastream to add the thumbnail',
      'ds_mimetype' => 'thumbnail mimetype',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_thumbnail_entries',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );


  $items['cwrc_migration_batch_retrieve_content'] = array(
    'description' => 'Given a PID retrieve the content of a specific datastream and store in a given directory',
    'options' => array(
      'pid_file' => 'PIDs to modify',
      'ds_id' => 'id of the datastream (where content XML stored)',
      'dest_path' => 'destination directory',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_retrieve_content',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_export_by_date'] = array(
    'description' => 'Given a PID and date, retrieve the content of a specific datastream and store in a given directory',
    'options' => array(
      'pid_file' => 'list PIDs to process',
      'ds_id' => 'id of the datastream (where content XML stored)',
      'dest_dir' => 'destination directory',
      'date' => 'destination directory',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_export_by_date --pid_file=/tmp/z --ds_id=CWRC --dest_dir=/tmp/zz --date="2020-09-01"',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_apply_xslt'] = array(
    'description' => 'Apply XSLT to each object in the list and add a workflowstamp',
    'options' => array(
      'pid_file' => 'PIDs to modify',
      'ds_id' => 'id of the datastream (where content XML stored)',
      'xslt_path' => 'path to XSLT to convert entriy to mods',
      "wf_category" => "workflow category",
      'wf_stamp' => 'workflow stamp',
      'wf_status' => 'workflow status',
      "wf_note" => "workflow note",
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_apply_xslt',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_retrieve_members_of_collection'] = array(
    'description' => 'Given the PID of a collection, return a list of members',
    'options' => array(
      'collection_pid' => 'PID of the collection of retrieve members',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_retrieve_members_of_collection',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_retrieve_item_summary'] = array(
    'description' => 'list of items and their associated membership',
    'options' => array(
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_retrieve_item_summary',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_retrieve_orphan_summary'] = array(
    'description' => 'list of orphaned items.',
    'options' => array(
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_retrieve_orphan_summary',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_delete'] = array(
    'description' => 'Delete a list of objects',
    'options' => array(
      'pid_file' => 'PIDs to delete',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_delete',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );

  $items['cwrc_migration_batch_test'] = array(
    'description' => 'Test a list of objects',
    'options' => array(
      'pid_file' => 'list PIDs to process',
    ),
    'examples' => array(
      'drush -u 1 cwrc_migration_batch_test',
    ),
    'bootstrap' => DRUSH_BOOTSTRAP_MAX,
  );



  return $items;
}

/**
 * CWRC migration batch - migration_init 
 *
 * @param string $connection
 *   tuque connection object
 */
function migration_init (
    &$connection
  )
{
  
  // https://groups.google.com/forum/#!topic/islandora-dev/1KyBnj7DjCQ
  global $user;
  drush_print($user->name);


  // Include modules.
  module_load_include('inc', 'islandora', 'includes/utilities');
  module_load_include('inc', 'cwrc_migration_batch', 'includes/workflow');
  module_load_include('inc', 'cwrc_migration_batch', 'includes/helpers');

  ini_set('max_execution_time', 3000);


  // Get the tuque connection.
  $connection = islandora_get_tuque_connection();
  $connection->connection->username = $user->name;
  $connection->connection->password = $user->pass;
}

/**
 * CWRC setup
 *
 **/
function rels_setup (
    $connection 
    , $cModel_pid
    , $collection_pid
    )
{
  // Content Model.  Create the content model.
  if ( !islandora_object_load($cModel_pid) )
  {
    create_cModel($cModel_pid, $connection);
    drush_print("content model added - $cModel_pid");
  }
  else {
    drush_print("content model already exists - $cModel_pid");
  }

  // Collection.  Create the collection object.
  if ( !islandora_object_load($collection_pid) )
  {
    create_collection($collection_pid, $connection);
    drush_print("Collection added - $collection_pid");
  }
  else
  {
    drush_print("collection already exists - $collection_pid");
  }
}




/** 
 * CWRC migration batch - entities 
 */
function drush_cwrc_migration_batch_ingest_entities() 
{
  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');

  // path to the input document files
  $input_dir = drush_get_option(
    'input_dir'
    , '/tmp/entities_ingest'
    );

  $XACML_policy = drush_get_option(
    'XACML_policy'
    , '/tmp/xacml_policy.xml'
    );

  // pid of the collection to add the object to 
  $collection_pid = drush_get_option(
    'collection_pid'
    , 'cwrc:delete_me'
  );

  // pid of the cModel the object belongs to 
  $cModel_pid = drush_get_option(
    'cModel_pid'
    , 'cwrc'
  );

  $namespace = drush_get_option(
    'namespace'
    , 'cwrc'
  );

  // id of the content datastream (dsid)
  $content_datastream_dsid = drush_get_option(
    'content_datastream_dsid'
    , 'X'
  );

  // label of the content datastream
  $content_datastream_label = drush_get_option(
    'content_datastream_label'
    , 'X'
  );

  // path to the cwrc_entities to DC XSLT file
  $entity_to_dc_xslt_path = drush_get_option(
    'entity_to_dc_xslt_path'
    , 'xslt/entity_to_dc_TEI_support.xsl'
  );
  
  $wf_category = drush_get_option('wf_category' , 'machine_processed');
  $wf_stamp = drush_get_option('wf_stamp' , 'orlando:SUB');
  $wf_status = drush_get_option('wf_status' , 'c');
  $wf_note = drush_get_option('wf_note' , 'Apply XSLT: $xslt');

  drush_print("CWRC Entities Migration batch");

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // Collection.  Create the collection object.
  rels_setup ( $connection, $cModel_pid, $collection_pid);

  // setup XSL 
  // Get the path to the content_to_dc.xsl file.
  // ../cwrc-entities/xslt/entity_to_dc.xml
  $dc_xslt
    = drupal_get_path('module', 'cwrc_entities') . '/' . $entity_to_dc_xslt_path;
  drush_print("XSLT DC: " . $dc_xslt);
  dc_xslt_setup($dc_xslt, $xslt_proc);

  // Set source directory for the content to ingest
  drush_print("Source dir: " . $input_dir);
  $cwrc_directory = $input_dir;

  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  $src_dir_handle = opendir($cwrc_directory);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier($namespace, TRUE);

    // load XML doc
    // Load cwrc xml from a file.
    $tmp = $cwrc_directory . $file_name;
    $cwrc_xml = file_get_contents($tmp);

    $cwrc_dom = new DOMDocument();
    $cwrc_dom->load($tmp);

    // build the DC content
    // Convert entity to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    $xslt_proc->setParameter( "", "PID_PARAM", $pid );
    $dc_xml = $xslt_proc->transformToXML($cwrc_dom);

    //print ("zzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");

    // We need to prepare a new object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );

    // create content datastream
    $cwrc_ds = $object->constructDatastream($content_datastream_dsid, 'M');
    $cwrc_ds->label = $content_datastream_label;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($cwrc_xml);
    $object->ingestDatastream($cwrc_ds);

    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // add a workflow stamp indicating that the object was ingested
    if (!$object['WORKFLOW']) {
      $workflow = CWRCWorkflowAPI::fromTemplate($object['WORKFLOW']);
    }
    else {
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
    }
    $workflow->addWorkflowStep(
      $object->id,
      new WorkflowItem(WorkflowConst::WORKFLOW, array()),
      new WorkflowItem(
        WorkflowConst::ACTIVITY,
        array(
          "category" => $wf_category,
          'stamp' => $wf_stamp,
          'status' => $wf_status,
          "note" => $wf_note,
          )
        )
      );
    CWRCWorkflowAPI::updateDatastream($workflow, $object);

    // Store the object.
    try {
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");
    }
    catch (Exception $e) {
      drush_print("pid=$pid title=\"$title\" was not added.");
      $file_error_count++;
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}

/** 
 * CWRC migration batch - bibliography Orlando
 * assumes Orlando bibl have been converted into MODS
 * workflow is pulled from the legacy Orlando format stored in subdir
 *
 * @param string $path_to_file
 *   path to directory where migrated files are located.
 * @param string $mods_subdir
 *   sub directory where the MODS records reside
 * @param string $legacy_subdir
 *   sub directory where the legacy orlando records reside
 * @param string $collection_pid
 *   pid of the collection to add the object to 
 * @param string $cModel_pid
 *   pid of the cModel the object belongs to 
 * @param string $dc_xslt
 *   location of the DC XSLT transform 
 * @param string $policy
 *   location of the Fedora XACML policy 
 */
function drush_cwrc_migration_batch_ingest_bibliography (
    $path_to_file
    , $mods_subdir
    , $legacy_subdir 
    , $collection_pid
    , $cModel_pid
    , $dc_xslt
    , $policy 
  ) 
{
  drush_print("CWRC Bibliography Migration batch");

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // Content Model.  Create the content model.
  // Collection.  Create the collection object.
  rels_setup ( $connection, $cModel_pid, $collection_pid);

  // setup XSL 
  // Get the path to the content_to_dc.xsl file.
  $dc_xslt
      = drupal_get_path('module', 'cwrc_migration_batch') . '/' . $dc_xslt;
  dc_xslt_setup($dc_xslt, $xslt_proc);

  // Set source directory for the legacy Orlando content
  $legacy_dir = $path_to_file . "/" . $legacy_subdir . "/"; 

  // Set source directory for the content to ingest
  $cwrc_directory = $path_to_file . "/" . $mods_subdir . "/";

  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier("cwrc", TRUE);

    // load XML doc
    // Load cwrc xml from a file.
    $tmp = $cwrc_directory . $file_name;
    $cwrc_xml = file_get_contents($tmp);

    $cwrc_dom = new DOMDocument();
    $cwrc_dom->load($tmp);

    // load legacy XML doc
    // Load cwrc xml from a file.
    $tmp = $legacy_dir . $file_name;
    $legacy_xml = file_get_contents($tmp);

    
    // build the DC content
    // Convert entity to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    $xslt_proc->setParameter( "", "PID_PARAM", $pid );
    $dc_xml = $xslt_proc->transformToXML($cwrc_dom);

    //print ("zzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");


    // We need to prepare a new object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    //$cwrc_ds = $object->constructDatastream('CWRC-CONTENT', 'M');
    $cwrc_ds = $object->constructDatastream('MODS', 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($cwrc_xml);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($legacy_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      create_DS_POLICY(
        $policy 
        , $object
      );
    }

    // Store the object.
    try {
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);


}


/** 
 * CWRC migration batch - event / documentation Orlando
 * assumes Orlando bibl have been converted into MODS
 * workflow is pulled from the legacy Orlando format stored in subdir
 *
 * @param string $path_to_file
 *   path to directory where migrated files are located.
 * @param string $mods_subdir
 *   sub directory where the MODS records reside
 * @param string $legacy_subdir
 *   sub directory where the legacy orlando records reside
 * @param string $collection_pid
 *   pid of the collection to add the object to 
 * @param string $cModel_pid
 *   pid of the cModel the object belongs to 
 * @param string $dc_xslt
 *   location of the DC XSLT transform 
 * @param string $policy
 *   location of the Fedora XACML policy 
 */
function drush_cwrc_migration_batch_ingest_events_documentation (
    $path_to_file
    , $mods_subdir
    , $legacy_subdir 
    , $collection_pid
    , $cModel_pid
    , $dc_xslt
    , $policy 
  ) 
{
  drush_print("CWRC Event/Documentation Migration batch");

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // Content Model.  Create the content model.
  // Collection.  Create the collection object.
  rels_setup ( $connection, $cModel_pid, $collection_pid);

  // setup XSL 
  // Get the path to the content_to_dc.xsl file.
  $dc_xslt
      = drupal_get_path('module', 'cwrc_migration_batch') . '/' . $dc_xslt;
  dc_xslt_setup($dc_xslt, $xslt_proc);

  // Set source directory for the legacy Orlando content
  $legacy_dir = $path_to_file . "/" . $legacy_subdir . "/"; 

  // Set source directory for the content to ingest
  $mods_dir = $path_to_file . "/" . $mods_subdir . "/";

  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier("cwrc", TRUE);

    // load MODS doc
    // Load MODS xml from a file.
    $tmp = $mods_dir. $file_name;
    $mods_xml = file_get_contents($tmp);

    $mods_dom = new DOMDocument();
    $mods_dom->load($tmp);

    // load legacy XML doc
    // Load cwrc xml from a file.
    $tmp = $legacy_dir . $file_name;
    $legacy_xml = file_get_contents($tmp);

    
    // build the DC content
    // Convert MODS to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    $xslt_proc->setParameter( "", "PID_PARAM", $pid );
    $dc_xml = $xslt_proc->transformToXML($mods_dom);

    //print ("zzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");


    // We need to prepare a new object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    $cwrc_ds = $object->constructDatastream('CWRC-CONTENT', 'M');
    $cwrc_ds->label = 'CWRC Event ' . $pid;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_xml);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create DC datastream
    create_DS_MODS($mods_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($legacy_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      create_DS_POLICY(
        $policy 
        , $object
      );
    }

    // Store the object.
    try {
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}



/** 
 * CWRC migration batch - biography / writing  Orlando
 * assumes Orlando bibl have been converted into MODS
 * workflow is pulled from the legacy Orlando format stored in subdir
 * the separate biography and writing docs for a person are
 * linked via RELS-EXT
 *
 * @param string $path_to_file
 *   path to directory where migrated files are located.
 * @param string $mods_subdir
 *   sub directory where the MODS records reside
 * @param string $legacy_subdir
 *   sub directory where the legacy orlando records reside
 * @param string $collection_pid
 *   pid of the collection to add the object to 
 * @param string $cModel_pid
 *   pid of the cModel the object belongs to 
 * @param string $dc_xslt
 *   location of the DC XSLT transform 
 * @param string $policy
 *   location of the Fedora XACML policy 
 */
function drush_cwrc_migration_batch_ingest_biography_writing (
    $path_to_file
    , $mods_subdir
    , $legacy_subdir 
    , $collection_pid
    , $cModel_pid
    , $dc_xslt
    , $policy 
  ) 
{
  drush_print("CWRC Biography / Writing Migration batch");

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // Content Model.  Create the content model.
  // Collection.  Create the collection object.
  rels_setup ( $connection, $cModel_pid, $collection_pid);

  // setup XSL 
  // Get the path to the content_to_dc.xsl file.
  $dc_xslt
      = drupal_get_path('module', 'cwrc_migration_batch') . '/' . $dc_xslt;
  dc_xslt_setup($dc_xslt, $xslt_proc);

  // Set source directory for the legacy Orlando content
  $legacy_dir = $path_to_file . "/" . $legacy_subdir . "/"; 

  // Set source directory for the content to ingest
  $mods_dir = $path_to_file . "/" . $mods_subdir . "/";

  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier("cwrc", TRUE);

    // Strip out the .xml from the file name and store in the variable.
    preg_match('/[a-z0-9-_]*/', $file_name, $object_name);
    // Strip out file name to get type -b(biography) or -w (writable).
    preg_match('/-[a-z]+/', $file_name, $object_type);

    $object_name = $object_name[0];
    $object_type = $object_type[0];


    $pid = "cwrc:$object_name";

    // If the object already exists then continue to next file.
    if (islandora_object_load($pid)) {
      drush_print("object $object_name is already in fedora");
      continue;
    }

    // load MODS doc
    // Load MODS xml from a file.
    $tmp = $mods_dir . $file_name;
    $mods_xml = file_get_contents($tmp);

    $mods_dom = new DOMDocument();
    $mods_dom->load($tmp);

    // load legacy XML doc
    // Load cwrc xml from a file.
    $tmp = $legacy_dir . $file_name;
    $legacy_xml = file_get_contents($tmp);

    
    // build the DC content
    // Convert MODS to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    $xslt_proc->setParameter( "", "PID_PARAM", $pid );
    $dc_xml = $xslt_proc->transformToXML($mods_dom);

    //print ("zzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");


    // We need to prepare a new object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    $cwrc_ds = $object->constructDatastream('CWRC', 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_xml);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create MODS datastream
    create_DS_MODS($mods_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($legacy_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      create_DS_POLICY( $policy , $object);
    }


    // create relationships
    $object->relationships->add(FEDORA_RELS_EXT_URI, 'originalFile', $file_name);

    // We need to get the object_type and use the other type to set the
    // relationship.
    if ($object_type == '-b') {
      $other_object_type = '-w';
    }
    else {
      $other_object_type = '-b';
    }

    // Replace the file name.
    $other_pid = preg_replace('/-[a-z]+/', $other_object_type, $object_name);
    // Set the relationship to the other object type.
    $object->relationships->add(FEDORA_RELS_EXT_URI, 'isDependentOf', "cwrc:$other_pid");


    // Store the object.
    try {
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}

/** 
 * CWRC migration batch - CWRC documents compatible with CWRC-Writer 
 *
 * @param string $path_to_file
 *   path to directory where migrated files are located.
 * @param string $collection_pid
 *   pid of the collection to add the object to 
 * @param string $cModel_pid
 *   pid of the cModel the object belongs to 
 * @param string $content_datastream_dsid
 *   id of the content datastream (dsid)
 * @param string $content_datastream_label
 *   label of the content datastream
 * @param string $entity_to_dc_xslt_path
 *   path to the cwrc_entities to DC XSLT file
 * @param string $schema_pid
 *   PID of the Object to reference for schema and css information 
 */
function drush_cwrc_migration_batch_ingest_cwrc_documents(
    $path_to_file
    , $workflow_file
    , $collection_pid
    , $cModel_pid
    , $content_datastream_dsid
    , $content_datastream_label
    , $content_to_mods_xslt_path
    , $mods_to_dc_xslt_path
    , $schema_pid
  ) 
{

  
  drush_print("CWRC documents compatible with CWRC-Writer Migration batch");

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // Content Model.  Create the content model.
  // Collection.  Create the collection object.
  rels_setup ( $connection, $cModel_pid, $collection_pid);

  // setup XSL 
  // Get the path to the mods_to_dc.xsl file.
  $dc_xslt
    = drupal_get_path('module', 'cwrc_migration_batch') .'/'. $mods_to_dc_xslt_path;
  drush_print("XSLT DC: " . $dc_xslt);
  dc_xslt_setup($dc_xslt, $xslt_dc_proc);

  // Get the path to the mods_to_dc.xsl file.
  $mods_xslt
    = drupal_get_path('module', 'cwrc_migration_batch') .'/'. $content_to_mods_xslt_path;
  drush_print("XSLT MODS: " . $mods_xslt);
  dc_xslt_setup($mods_xslt, $xslt_mods_proc);

  // setup the default workflow
  // get the contents of the default workflow for an entity
  $path_to_workflow_file
      = drupal_get_path('module', 'cwrc_migration_batch') .'/'. $workflow_file
      ;
  drush_print("Workflow: " . $path_to_workflow_file);
  $workflow_xml = file_get_contents($path_to_workflow_file);


  // Set source directory for the content to ingest
  $cwrc_directory = $path_to_file;

  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  $src_dir_handle = opendir($cwrc_directory);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;
    $content_datastream_label = $file_name;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier("cwrc", TRUE);

    // load XML doc
    // Load cwrc xml from a file.
    $tmp = $cwrc_directory . $file_name;
    $cwrc_xml = file_get_contents($tmp);
    
    //print ("zzz " . $cwrc_xml . "zzzzzzzzzzzzzz\n");

    $cwrc_dom = new DOMDocument();
    $cwrc_dom->loadXML($cwrc_xml);

    //print_r ("zzz " . $cwrc_dom . "zzzzzzzzzzzzzz\n");

    // build the MODS 
    // Convert content to MODS.
    // Transform to MODS xml.
    // Apply the stylesheet.
    $xslt_mods_proc->setParameter( "", "PID_PARAM", $pid );
    $mods_xml = $xslt_mods_proc->transformToXML($cwrc_dom);

    //print ("zzz " . $mods_xml . "zzzzzzzzzzzzzz\n");

    $mods_dom = new DOMDocument();
    $mods_dom->loadXML($mods_xml);
 
    // build the DC content
    // Convert MODS to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    $xslt_dc_proc->setParameter( "", "PID_PARAM", $pid );
    $dc_xml = $xslt_dc_proc->transformToXML($mods_dom);

    //print ("zzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");


    // We need to prepare a new object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    $cwrc_ds = $object->constructDatastream($content_datastream_dsid, 'M');
    $cwrc_ds->label = $content_datastream_label;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($cwrc_xml);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create DC datastream
    create_DS_MODS($mods_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow($workflow_xml, $object);

    // create relationships
    $object->relationships->add(CWRC_RELS_EXT_URI, 'hasSchema', $schema_pid);

    // Create the Policy datastream.
//    if ( $policy != null && $policy!="NONE" ) 
//    {
//      create_DS_POLICY(
//        $policy 
//        , $object
//      );
//    }

    // Store the object.
    try {
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");
    }
    catch (Exception $e) {
      drush_print("pid=$pid title=\"$title\" was not added.");
      $file_error_count++;
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
  // unsure how to properly set policy - come combo of POLICY and RELS-EXT 
  // datastreams 
  drush_print("\n Next Steps: set permissions on the Collection and objects");

}

/**
 * CWRC export batch - book objects
 *
 * $path_to_list_of_book_PIDs
 *  path to the list of book PIDs to export
 * $destination_path
 *  path to save the results
 */
function drush_cwrc_migration_batch_export_book (
  $path_to_list_of_book_PIDs
  , $destination_path
  )
{
  drush_print("CWRC Export Book Batch");

  // Include modules.
  module_load_include('inc', 'islandora_paged_content', 'includes/utilities');

  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // open the input file for reading
  $handle_read = fopen("$path_to_list_of_book_PIDs", "r");

  $handle_write = null;

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  $line = null;
  $tmp = null;
  while ($handle_read && ($line = fgets($handle_read)) !== false)
  {
    $pid = trim($line);

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("[".$pid."]");


    // export the object.
    try {
      $book_object = islandora_object_load($pid);
      $current_pages = islandora_paged_content_get_pages($book_object);
      $handle_write = fopen($destination_path . "/" . $book_object->label."-".$pid, 'w+');
      $file_count++;
      foreach ($current_pages as $page)
      {
        $tmp = null;
        $tmp = islandora_object_load($page['pid']);
        fwrite($handle_write, $tmp['OCR']->content);
        drush_print("page={$page['page']} | label={$page['label']} | pid={$page['pid']} was added. #{$file_count} file exported.");
      }
      fclose($handle_write);
      drush_print("Book pid=$pid was exported. #{$file_count} file exported.");
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid was not added.");
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files exported: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}


/**
 * CWRC reindex list of objects (by PID)
 *
 * $path_to_list_of_book_PIDs
 *  path to the list of book PIDs to export
 */
function drush_cwrc_migration_batch_reindex_list(
  $path_to_list_of_book_PIDs
  )
{
  drush_print("CWRC reindex list ");

  // Include modules.

  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // open the input file for reading
  $handle_read = fopen("$path_to_list_of_book_PIDs", "r");

  $handle_write = null;

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  $line = null;
  $tmp = null;
  while ($handle_read && ($line = fgets($handle_read)) !== false)
  {
    $pid = trim($line);

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("[".$pid."]");


    // export the object.
    try {
      $obj = islandora_object_load($pid);
      $file_count++;
      drush_print("Book pid=$pid was exported. #{$file_count} file exported.");
      drush_print($obj['WORKFLOW']->content);
      $obj['WORKFLOW']->setContentFromString($obj['WORKFLOW']->content);
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid was not added.");
      drush_print($e->getMessage());
    }

    usleep(5000);
  }
  drush_print('Migration Complete');
  drush_print('Number of files: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}




/** 
 * CWRC migration batch - biography / writing / Events Orlando version 2 - 2016-11-29
 * assumes MODS and DC generated on the fly 
 * workflow is pulled from the legacy Orlando format stored in subdir
 * the separate biography and writing docs for a person are
 * linked via MODS entity reference
 *
 */
function drush_cwrc_migration_batch_ingest_orlando_biography_writing () 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');

  $main_ds_id = 'CWRC';

  drush_print("Orlando Biography / Writing / Event migration batch version 2.0");

  // path to the input document files 
  $legacy_dir = drush_get_option('legacy_dir','/tmp/orlando/bio_writ');

  // path to entity mapping file
  $entity_mapping_file = drush_get_option(
    'entity_mapping_file'
    , '/tmp/orlando/entity_mapping.json'
    );

  // 
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_to_v2 = drush_get_option(
    'xslt_to_v2'
    , $ext_home . 'transforms/orlando_legacy_to_v2_bio_writing.xsl'
  );
  $xslt_to_mods = drush_get_option(
    'xslt_to_mods'
    , $ext_home . 'transforms/orlando_combined_bio_writing_TO_mods.xsl'
  );
  $xslt_mods_to_dc = drush_get_option(
    'xslt_mods_to_dc'
    , libraries_get_path('islandora_cwrc_xslt_library') . '/xslt/mods_to_dc.xsl'
  );
  
  $collection_pid_pub_c = drush_get_option(
    'collection_pid_pub_c'
    , 'cwrc:fc271c04-7fd1-46b4-9462-339dbe917c46'
  );
  $collection_pid_non_pub_c = drush_get_option(
    'collection_pid_non_pub_c'
    , 'cwrc:bf8ba2b7-dbed-440b-9000-88860be1bcc9'
  );
  $cModel_pid = drush_get_option(
    'cModel_pid' , 'cwrc:documentCModel'
  );
  $schemaObject = drush_get_option(
    'schema_object', 'cwrc:biographySchema'
  );
  $policy = drush_get_option(
    'XACML_policy', $ext_home . 'xml/orlando_default_XACML_Policy_Stream.xml'
  );
  $namespace = drush_get_option(
    'namespace', 'orlando'
  );


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  
  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  //xslt_setup($xslt_bio_writ_to_v2, $bio_writ_to_v2_xslt_proc);
  //xslt_setup($xslt_bio_writ_to_mods, $bio_writ_to_mods_xslt_proc);
  //xslt_setup($xslt_mods_to_dc, $mods_to_dc_xslt_proc);
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $to_v2_xslt_proc = $saxonProc->newXsltProcessor();
  $to_mods_xslt_proc = $saxonProc->newXsltProcessor();
  $mods_to_dc_xslt_proc = $saxonProc->newXsltProcessor();

  $to_v2_xslt_proc->compileFromFile($xslt_to_v2);
  $to_mods_xslt_proc->compileFromFile($xslt_to_mods);
  $mods_to_dc_xslt_proc->compileFromFile($xslt_mods_to_dc);

  // Initialize entity mapping: a JSON file ofPID to standard name 
  // entity mapping file used to add URI's to bio/writing/event',
  try {
    drush_print("Entity mapping file: " . $entity_mapping_file);
    entity_mapping_load($entity_mapping, $entity_mapping_file);
    drush_print("person: " . count($entity_mapping["NAME"]));
    drush_print("org: " . count($entity_mapping["ORGNAME"]));
    drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
    // write the entity mapping file
    entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);
  } catch (Exception $e) {
    drush_print($e);
    exit();
  }

  // xpath to determine if PUB-C Orlando document - Biography/Writing/Event
  $xpathStrPub = "/*/ORLANDOHEADER/REVISIONDESC/RESPONSIBILITY[@WORKSTATUS='PUB' and @WORKVALUE='C']";
  //$xpathStrBio = "/BIOGRAPHY";
  //$xpathStrWriting = "/WRITING";

  // Set source directory for the legacy Orlando content
  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  drush_print("Input: " . $legacy_dir);
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier($namespace, TRUE);

    // add ID and PID to the entity mapping
    // ToDo: assumption - use UTF-8 version
    $legacy_id = $file_name;
    if ( isset($pid) && isset($legacy_id) ) {
      entity_mapping_add($entity_mapping, $legacy_id, $pid, "EVENT");
      drush_print("Entity ID [$legacy_id] added with value [$pid]");
    }
    else {
      drush_print("Entity ID [$legacy_id] not found");
    }

    // Strip out the .xml from the file name and store in the variable.
    //preg_match('/[a-z0-9-_]*/', $file_name, $object_name);
    // Strip out file name to get type -b(biography) or -w (writable).
    //preg_match('/-[a-z]?/', $file_name, $object_type);
    //$object_name = $object_name[0];
    //$object_type = $object_type[0];
    //$pid = "cwrc:$object_name";

    // If the object already exists then continue to next file.
    if (islandora_object_load($pid)) {
      drush_print("object $object_name is already in fedora");
      continue;
    }
    
    $content_datastream_label = preg_replace('/(\.sgm)$/','.xml', $file_name); 


    // Load xml from a file.
    $legacy_content = file_get_contents($legacy_dir.$file_name);
    //drush_print($legacy_content);

    //libxml_disable_entity_loader(false);
    //$content_dom = new DOMDocument();
    //$content_dom->load($legacy_dir.$file_name);
    //$content_dom->loadXML($legacy_content);

    // build the version 2.0 of the biography / writing document
    $to_v2_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($legacy_content)
    );
    //$bio_writ_to_v2_xslt_proc->setSourceFromXdmValue($content_dom);
    $content_v2_xml = $to_v2_xslt_proc->transformToString();
    $content_v2_xml = str_replace(
        '<!DOCTYPE ORLANDO [ <!ENTITY % character_entities SYSTEM "http://cwrc.ca/schemas/character_entities.dtd"> %character_entities; ]>\n' 
        , ""
        , $content_v2_xml
      );

    $content_uri_xml = cwrc_migration_batch_add_entity_refs_content($content_v2_xml, $main_ds_id, $entity_mapping);

    // Build MODS XML.
    //$content_dom = null;
    //$content_dom = new DOMDocument();
    //$content_dom->loadXML($v2_content);
    //$mods_xml = $bio_writ_to_mods_xslt_proc->transformToString($content_dom);
    $to_mods_xslt_proc->setParameter(
      "param_original_id", $saxonProc->createAtomicValue($file_name) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_editor_brown_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['NAME']['Brown, Susan I.'])) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_editor_clements_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['NAME']['Clements, Patricia'])) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_editor_grundy_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['NAME']['Grundy, Isobel'])) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_publisher_cup_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['ORGNAME']['Cambridge University Press'])) 
      );
    $to_mods_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_uri_xml)
      );
    $mods_xml = $to_mods_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $mods_xml);

    //$mods_dom = new DOMDocument();
    //$mods_dom->load($mods_xml);

    // build the DC content
    // Convert MODS to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    //$mods_to_dc_xslt_proc->setParameter( "", "PID_PARAM", $pid );
    //$dc_xml = $mods_to_dc_xslt_proc->transformToString($mods_dom);
    $mods_to_dc_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($mods_xml)
    );
    $dc_xml = $mods_to_dc_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");

    // determine collection based on workflow
    $content_dom = new DOMDocument();
    $content_dom->loadXML($content_v2_xml);
    $xpathObj = new DOMXpath($content_dom); 
    $pubCRes = $xpathObj->query($xpathStrPub);
    $collection_pid = $collection_pid_non_pub_c;
    if (!is_null($pubCRes) and $pubCRes->length>0)
    {
      $collection_pid = $collection_pid_pub_c;
      drush_print("pub-c");
    } 

    //determine the schema for the object
    //$schema_obj = null;
    //$tmpBioSchema = $xpathObj->query($xpathStrBio);
    //if (!is_null($tmpBioSchema))
    //{
      //$schema_obj = islandora_object_load($bioSchemaObject);
    //}
    //$tmpWritingSchema = $xpathObj->query($xpathStrWriting);
    //if (!is_null($tmpWritingSchema))
    //{
      //$schema_obj = islandora_object_load($writingSchemaObject);
    //}
    $schema_obj = islandora_object_load($schemaObject);


    // prepare a new Fedora object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    // with the legacy content
    $cwrc_ds = $object->constructDatastream($main_ds_id, 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create MODS datastream
    create_DS_MODS($mods_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($content_uri_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      //create_DS_POLICY($policy, $object);
      $policy_ds
          = new IslandoraXacml(
              $object
              , file_get_contents($policy)
              );
      $policy_ds->writeBackToFedora();
    }

    islandora_cwrc_writer_set_document_schema($object, $schema_obj);

    try {
      // Store the object.
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Initial ingest into the CWRC repository.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      // update content datastream with version 2.0 of the content
      // UTF-8 and ISO 8601 dates
      $object[$main_ds_id]->setContentFromString($content_v2_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Conversion to version 2.0 of the Orlando format (UTF-8 and ISO 8601 dates).",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      //update URI's refences in entities
      if ($object[$main_ds_id])
      {
        $object[$main_ds_id]->setContentFromString($content_uri_xml);
        cwrc_migration_batch_add_entity_refs_workflow($object, $main_ds_id);
      }
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  
  // write the entity mapping file
  entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}

/** 
 * CWRC migration batch - combine biography / writing Orlando version 3 - 2019-12-19
 * assumes MODS and DC generated on the fly 
 * workflow is pulled from the legacy Orlando format stored in subdir
 * the separate biography and writing docs for a person are
 * linked via MODS entity reference
 *
 */
function drush_cwrc_migration_batch_ingest_orlando_combine_biography_writing () 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');

  $main_ds_id = 'CWRC';

  drush_print("Orlando Biography / Writing / Event migration batch version 2.0");

  // path to the input document files 
  $legacy_dir = drush_get_option('legacy_dir','/tmp/orlando/bio_writ');

  // path to entity mapping file
  $entity_mapping_file = drush_get_option(
    'entity_mapping_file'
    , '/tmp/orlando/entity_mapping.json'
    );

  // UTF-8 plus legacy dates 
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_to_v2 = drush_get_option(
    'xslt_to_v2'
    , $ext_home . 'transforms/orlando_legacy_to_v2_bio_writing.xsl'
  );

  // merge bio/writing doc headers
  $xslt_to_v3 = drush_get_option(
    'xslt_to_v3'
    , $ext_home . 'transforms/bioANDwritingTOentry.xsl'
  );

  $xslt_to_mods = drush_get_option(
    'xslt_to_mods'
    , $ext_home . 'transforms/orlando_combined_bio_writing_TO_mods.xsl'
  );
  $xslt_mods_to_dc = drush_get_option(
    'xslt_mods_to_dc'
    , libraries_get_path('islandora_cwrc_xslt_library') . '/xslt/mods_to_dc.xsl'
  );
  
  $collection_pid_pub_c = drush_get_option(
    'collection_pid_pub_c'
    , 'cwrc:fc271c04-7fd1-46b4-9462-339dbe917c46'
  );
  $collection_pid_non_pub_c = drush_get_option(
    'collection_pid_non_pub_c'
    , 'cwrc:bf8ba2b7-dbed-440b-9000-88860be1bcc9'
  );
  $cModel_pid = drush_get_option(
    'cModel_pid' , 'cwrc:documentCModel'
  );
  $schemaObject = drush_get_option(
    'schema_object', 'cwrc:biographySchema'
  );
  $policy = drush_get_option(
    'XACML_policy', $ext_home . 'xml/orlando_default_XACML_Policy_Stream.xml'
  );
  $namespace = drush_get_option(
    'namespace', 'orlando'
  );


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  //xslt_setup($xslt_bio_writ_to_v2, $bio_writ_to_v2_xslt_proc);
  //xslt_setup($xslt_bio_writ_to_mods, $bio_writ_to_mods_xslt_proc);
  //xslt_setup($xslt_mods_to_dc, $mods_to_dc_xslt_proc);
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $to_v2_xslt_proc = $saxonProc->newXsltProcessor();
  $to_v3_xslt_proc = $saxonProc->newXsltProcessor();
  $to_mods_xslt_proc = $saxonProc->newXsltProcessor();
  $mods_to_dc_xslt_proc = $saxonProc->newXsltProcessor();

  $to_v2_xslt_proc->compileFromFile($xslt_to_v2);
  $to_v3_xslt_proc->compileFromFile($xslt_to_v3);
  $to_mods_xslt_proc->compileFromFile($xslt_to_mods);
  $mods_to_dc_xslt_proc->compileFromFile($xslt_mods_to_dc);

  // Initialize entity mapping: a JSON file ofPID to standard name 
  // entity mapping file used to add URI's to bio/writing/event',
  try {
    drush_print("Entity mapping file: " . $entity_mapping_file);
    entity_mapping_load($entity_mapping, $entity_mapping_file);
    drush_print("person: " . count($entity_mapping["NAME"]));
    drush_print("org: " . count($entity_mapping["ORGNAME"]));
    drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
    // write the entity mapping file
    entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);
  } catch (Exception $e) {
    drush_print($e);
    exit();
  }

  // xpath to determine if PUB-C Orlando document - Biography/Writing/Event
  $xpathStrPub = "/*/ORLANDOHEADER/REVISIONDESC/RESPONSIBILITY[@WORKSTATUS='PUB' and @WORKVALUE='C']";
  //$xpathStrBio = "/BIOGRAPHY";
  //$xpathStrWriting = "/WRITING";

  // Set source directory for the legacy Orlando content
  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  drush_print("Input legacy dir: " . $legacy_dir);
  $legacy_bio = $legacy_dir . "/biography/";
  $legacy_writ = $legacy_dir . "/writing/";
  $legacy_sccs = $legacy_dir . "/sccs_archive/";

  // build hash map of author ids with bio/writing doc attached
  // entries = array(
  //                "austja" => array(
  //                        "bio_xml" => ""
  //                        "writing_xml" => ""
  //                        "bio_sccs" => ""
  //                        "writing_sccs" => ""
  //                        )
  //                )
  $entries = array();
  build_hash_map($entries, $legacy_writ, "writ_xml", "writ_sccs");
  build_hash_map($entries, $legacy_bio, "bio_xml", "bio_sccs");

  // Counter for the number of files to migrate.
  $file_count = 0; 
  $file_error_count = 0;
  foreach ($entries as $key => $entry) {

    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("Entry: " . $key);

    // generate an id for the object
    // $pid = $connection->repository->getnextIdentifier($namespace, TRUE);
    // Use $key as the PID for example, orlando:austja 
    $pid = $namespace . ":" . $key; 

    // add ID and PID to the entity mapping
    // ToDo: assumption - use UTF-8 version
    $legacy_id = $key;
    if ( isset($pid) && isset($legacy_id) ) {
      entity_mapping_add($entity_mapping, $legacy_id, $pid, "ENTRY");
      drush_print("Entity ID [$legacy_id] added with value [$pid]");
    }
    else {
      drush_print("Entity ID [$legacy_id] not found");
    }

    // Strip out the .xml from the file name and store in the variable.
    //preg_match('/[a-z0-9-_]*/', $file_name, $object_name);
    // Strip out file name to get type -b(biography) or -w (writable).
    //preg_match('/-[a-z]?/', $file_name, $object_type);
    //$object_name = $object_name[0];
    //$object_type = $object_type[0];
    //$pid = "cwrc:$object_name";

    // If the object already exists then continue to next file.
    if (islandora_object_load($pid)) {
      drush_print("object $pid - $key is already in fedora");
      continue;
    }

    // get legacy files
    $legacy_writing_content = null;
    if (isset($entry['writ_xml'])) {
        $legacy_writing_content = file_get_contents($legacy_dir."/writing/".$entry['writ_xml']);
    }
    $legacy_biography_content = null;
    if (isset($entry['bio_xml'])) {
        $legacy_biography_content = file_get_contents($legacy_dir."/biography/".$entry['bio_xml']);
    }

    // build combined doc with both biography and writing orlando documents
    // <ORLANDO><BIOGRAPHY/><WRITING></ORLANDO>
    $tmp_doctype = "<!DOCTYPE ORLANDO [ <!ENTITY % character_entities SYSTEM 'http://cwrc.ca/schemas/character_entities.dtd'> %character_entities; ]>\n";
    $legacy_combined =
        "<?xml version='1.0' encoding='UTF-8'?>\n"
        . "<?xml-model href='https://cwrc.ca/schemas/orlando_entry.rng' type='application/xml' schematypens='http://relaxng.org/ns/structure/1.0'?>\n"
        . "<?xml-stylesheet type='text/css' href='https://cwrc.ca/templates/css/orlando.css'?>\n"
        . $tmp_doctype
        . "<ENTRY ID='".$key."'>\n"
        . ((isset($entry['bio_xml'])) ? preg_replace("/^.*(<BIOGRAPHY.+$)/s",'$1', $legacy_biography_content) : "")
        . ((isset($entry['writ_xml'])) ? preg_replace('/^.*(<WRITING.+$)/s','$1', $legacy_writing_content) : "")
        . "</ENTRY>"
        ;

        
    //drush_print($legacy_content);


    // build the version 2.0 of the biography / writing document
    // UTF-8 + Legacy Dates + remove DOCTYPE
    $to_v2_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($legacy_combined)
    );
    //$bio_writ_to_v2_xslt_proc->setSourceFromXdmValue($content_dom);
    $content_v2_xml = $to_v2_xslt_proc->transformToString();
    $content_v2_xml = str_replace($tmp_doctype , "" , $content_v2_xml);

    //drush_print($content_v2_xml);

    // build the version 3.0 of the biography / writing document
    // * merge bio/writing doc headers
    $to_v3_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_v2_xml)
    );
    $content_v3_xml = $to_v3_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $content_v3_xml);
    
    $content_uri_xml = cwrc_migration_batch_add_entity_refs_content($content_v3_xml, $main_ds_id, $entity_mapping);

    // Build MODS XML.
    $to_mods_xslt_proc->setParameter(
      "param_original_id", $saxonProc->createAtomicValue($key) 
      );
    if (isset($entry['bio_xml'])) {
        $to_mods_xslt_proc->setParameter(
            "param_original_bio_filename", $saxonProc->createAtomicValue($entry['bio_xml']) 
            );
    }
    if (isset($entry['writ_xml'])) {
        $to_mods_xslt_proc->setParameter(
            "param_original_writ_filename", $saxonProc->createAtomicValue($entry['writ_xml']) 
            );
    }
    $to_mods_xslt_proc->setParameter(
      "param_editor_brown_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['NAME']['Brown, Susan I.'])) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_editor_clements_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['NAME']['Clements, Patricia'])) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_editor_grundy_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['NAME']['Grundy, Isobel'])) 
      );
    $to_mods_xslt_proc->setParameter(
      "param_publisher_cup_value_uri"
      , $saxonProc->createAtomicValue(build_uri_from_pid($entity_mapping['ORGNAME']['Cambridge University Press'])) 
      );
    $to_mods_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_uri_xml)
      );
    $mods_xml = $to_mods_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $mods_xml);


    // build the DC content
    // Convert MODS to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    //$mods_to_dc_xslt_proc->setParameter( "", "PID_PARAM", $pid );
    //$dc_xml = $mods_to_dc_xslt_proc->transformToString($mods_dom);
    $mods_to_dc_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($mods_xml)
    );
    $dc_xml = $mods_to_dc_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");

    // TODO: update 2019-12-20 based on new schema
    // determine collection PUB-C versus non-PUB-C based on workflow
    $content_dom = new DOMDocument();
    $content_dom->loadXML($content_v3_xml);
    $xpathObj = new DOMXpath($content_dom); 
    $pubCRes = $xpathObj->query($xpathStrPub);
    $collection_pid = $collection_pid_non_pub_c;
    if (!is_null($pubCRes) and $pubCRes->length>0)
    {
      $collection_pid = $collection_pid_pub_c;
      drush_print("pub-c");
    } 

    //determine the schema for the object
    //$schema_obj = null;
    //$tmpBioSchema = $xpathObj->query($xpathStrBio);
    //if (!is_null($tmpBioSchema))
    //{
      //$schema_obj = islandora_object_load($bioSchemaObject);
    //}
    //$tmpWritingSchema = $xpathObj->query($xpathStrWriting);
    //if (!is_null($tmpWritingSchema))
    //{
      //$schema_obj = islandora_object_load($writingSchemaObject);
    //}
    $schema_obj = islandora_object_load($schemaObject);


    // prepare a new Fedora object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );

    // common label root
    //$content_datastream_label = preg_replace('/(\.sgm)$/','.xml', $file_name); 
    $content_datastream_label = $key;

    // create LEGACY_WRITING XML datastream from Doc Archive contents <=2019
    if (isset($entry['writ_xml'])) {
        $cwrc_ds = $object->constructDatastream('LEGACY_WRITING_XML', 'M');
        $cwrc_ds->label = $content_datastream_label."-w.xml";
        $cwrc_ds->mimeType = 'text/xml';
        $cwrc_ds->setContentFromString($legacy_writing_content);
        $object->ingestDatastream($cwrc_ds);
    }

    // create LEGACY_WRITING SCCS version datastream from Doc Archive contents <=2019
    if (isset($entry['writ_sccs'])) {
        //$legacy_writing_sccs = file_get_contents($legacy_dir."/sccs/".$entry['writ_sccs']);
        $legacy_writing_sccs = $legacy_sccs.$entry['writ_sccs'];
        $cwrc_ds = $object->constructDatastream('LEGACY_WRITING_SCCS', 'M');
        $cwrc_ds->label = "s.".$content_datastream_label."-w.sgm";
        $cwrc_ds->mimeType = 'application/octet-stream';
        $cwrc_ds->setContentFromFile($legacy_writing_sccs, FALSE);
        //$cwrc_ds->setContentFromString($legacy_writing_sccs);
        $object->ingestDatastream($cwrc_ds);
        $legacy_writing_sccs = null;
    }

    // create LEGACY_BIOGRAPHY XML datastream from Doc Archive contents <=2019
    if (isset($entry['bio_xml'])) {
        $cwrc_ds = $object->constructDatastream('LEGACY_BIOGRAPHY_XML', 'M');
        $cwrc_ds->label = $content_datastream_label."-b.xml";
        $cwrc_ds->mimeType = 'text/xml';
        $cwrc_ds->setContentFromString($legacy_biography_content);
        $object->ingestDatastream($cwrc_ds);
    }

    // create LEGACY_BIOGRAPHY SCCS version datastream from Doc Archive contents <=2019
    if (isset($entry['bio_sccs'])) {
        //$legacy_writing_sccs = file_get_contents($legacy_dir."/sccs/".$entry['bio_sccs']);
        $legacy_bio_sccs = $legacy_sccs.$entry['bio_sccs'];
        $cwrc_ds = $object->constructDatastream('LEGACY_BIOGRAPHY_SCCS', 'M');
        $cwrc_ds->label = "s.".$content_datastream_label."-b.sgm";
        $cwrc_ds->mimeType = 'application/octet-stream';
        $cwrc_ds->setContentFromFile($legacy_bio_sccs, FALSE);
        //$cwrc_ds->setContentFromString($legacy_writing_sccs);
        $object->ingestDatastream($cwrc_ds);
        $legacy_biography_sccs = null;
    } 

    // create combined content datastream
    // with the legacy content
    $cwrc_ds = $object->constructDatastream('CWRC', 'M');
    $cwrc_ds->label = $content_datastream_label.".xml";
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_combined);
    $object->ingestDatastream($cwrc_ds);
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create MODS datastream
    create_DS_MODS($mods_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($content_v3_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      //create_DS_POLICY($policy, $object);
      $policy_ds
          = new IslandoraXacml(
              $object
              , file_get_contents($policy)
              );
      $policy_ds->writeBackToFedora();
    }

    islandora_cwrc_writer_set_document_schema($object, $schema_obj);

    try {
      // Store the object.
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Initial ingest into the CWRC repository.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      // update content datastream with version 2.0 of the content
      // UTF-8 and ISO 8601 dates
      $object['CWRC']->setContentFromString($content_v2_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Conversion to version 2.0 of the Orlando format (UTF-8 and ISO 8601 dates).",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      // update content datastream with version 3.0 of the content
      // * merge bio/writing doc headers
      $object['CWRC']->setContentFromString($content_v3_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Conversion to version 3.0 of the Orlando format (rearrange elements).",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      //update URI's refences in entities
      if ($object[$main_ds_id])
      {
        $object[$main_ds_id]->setContentFromString($content_uri_xml);
        cwrc_migration_batch_add_entity_refs_workflow($object, $main_ds_id);
      }

    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }

  // write the entity mapping file
  entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}


/** 
 * CWRC migration batch - Entities to Orlando version 2 - 2016-11-29 - 2019-12-19
 * workflow is pulled from the legacy Orlando format stored in subdir
 *
 */
function drush_cwrc_migration_batch_ingest_orlando_entities() 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("Orlando entity migration batch version 2.0");

  // path to the input document files 
  $legacy_dir = drush_get_option(
    'legacy_dir'
    , '/tmp/orlando/bio_writ'
    );

  // path to entity mapping file
  $entity_mapping_file = drush_get_option(
    'entity_mapping_file'
    , '/tmp/orlando/entity_mapping.json'
    );

  // 
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_to_clean = drush_get_option(
    'xslt_to_clean'
    , $ext_home . 'transforms/orlando_legacy_to_v2_bio_writing.xsl'
  );
  $xslt_to_v2 = drush_get_option(
    'xslt_to_v2'
    , $ext_home . ''
  );
  $xslt_to_dc = drush_get_option(
    'xslt_to_dc'
    , libraries_get_path('islandora_cwrc_xslt_library') . '/xslt/mods_to_dc.xsl'
  );
  
  $main_ds_id = drush_get_option(
    'main_ds_id'
    , 'MODS'
  );
  $entity_type = drush_get_option(
    'entity_type'
    , 'BIBCIT'
  );
  $collection_pid_pub_c = drush_get_option(
    'collection_pid_pub_c'
    , 'cwrc:fc271c04-7fd1-46b4-9462-339dbe917c46'
  );
  $collection_pid_non_pub_c = drush_get_option(
    'collection_pid_non_pub_c'
    , 'cwrc:bf8ba2b7-dbed-440b-9000-88860be1bcc9'
  );
  $cModel_pid = drush_get_option(
    'cModel_pid' , 'cwrc:documentCModel'
  );
  $policy = drush_get_option(
    'XACML_policy', $ext_home . 'xml/orlando_default_XACML_Policy_Stream.xml'
  );
  $namespace = drush_get_option(
    'namespace', 'orlando'
  );
  $supplemental_filename = drush_get_option(
    'supplemental_filename', ''
  );
  $add_pub_c_workflow = drush_get_option(
    'add_pub_c_workflow', 'no'
  );


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  
  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $to_clean_xslt_proc = $saxonProc->newXsltProcessor();
  $to_v2_xslt_proc = $saxonProc->newXsltProcessor();
  $to_dc_xslt_proc = $saxonProc->newXsltProcessor();

  $to_clean_xslt_proc->compileFromFile($xslt_to_clean);
  $to_v2_xslt_proc->compileFromFile($xslt_to_v2);
  $to_dc_xslt_proc->compileFromFile($xslt_to_dc);

  // Initialize entity mapping: a JSON file ofPID to standard name 
  // entity mapping file used to add URI's to bio/writing/event',
  try {
    drush_print("Entity mapping file: " . $entity_mapping_file);
    entity_mapping_load($entity_mapping, $entity_mapping_file);
    drush_print("person: " . count($entity_mapping["NAME"]));
    drush_print("org: " . count($entity_mapping["ORGNAME"]));
    drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
    // write the entity mapping file
    entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);
  } catch (Exception $e) {
    drush_print($e);
    exit();
  }

  // xpath to determine if PUB-C Orlando document -  BIBL Legacy XPath
  $xpathStrPub = "//RESPONSIBILITY[@WORKSTATUS='PUB' and @WORKVALUE='C']";

  // Set source directory for the legacy Orlando content
  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  drush_print("Input: " . $legacy_dir);
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier($namespace, TRUE);

    // If the object already exists then continue to next file.
    if (islandora_object_load($pid)) {
      drush_print("object $pid is already in fedora");
      continue;
    }
    
    $content_datastream_label = preg_replace('/(\.sgm)$/','.xml', $file_name); 

    // Load xml from a file.
    $legacy_content = file_get_contents($legacy_dir.$file_name);

    // build the UTF-8 version with conversion of Orlando dates
    $to_clean_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($legacy_content)
    );
    $clean_xml = $to_clean_xslt_proc->transformToString();

    // add ID and PID to the entity mapping
    // ToDo: assumption - use UTF-8 version
    $legacy_id = entity_mapping_extract_id($clean_xml);
    if ( isset($pid) && isset($legacy_id) ) {
      entity_mapping_add($entity_mapping, $legacy_id, $pid, $entity_type);
      drush_print("Entity ID [$legacy_id] added with value [$pid]");
    }
    else {
      drush_print("Entity ID [$legacy_id] not found");
    }

    if ($main_ds_id=='MODS') {
      // update URI's refences in bibliographic entities
      // requires UTF-8 
      $content_uri_xml = cwrc_migration_batch_add_entity_refs_content($clean_xml, $main_ds_id, $entity_mapping);
    }
    else {
      // for other entity 
      $content_uri_xml = $clean_xml;
    }

    // build the version 2.0 
    $to_v2_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_uri_xml)
    );
    if ($supplemental_filename)
    {
      $to_v2_xslt_proc->setParameter(
        "supplemental_filename"
         , $saxonProc->createAtomicValue($supplemental_filename) 
      );
    }
    $content_v2_xml = $to_v2_xslt_proc->transformToString();

    //drush_print($content_v2_xml);

    // delete doctype
    $content_v2_xml = str_replace(
        '<!DOCTYPE ORLANDO [ <!ENTITY % character_entities SYSTEM "http://cwrc.ca/schemas/character_entities.dtd"> %character_entities; ]>\n' 
        , ""
        , $content_v2_xml
      );

    // build DC 
    $to_dc_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_v2_xml)
    );
    $to_dc_xslt_proc->setParameter(
        "PID_PARAM"
         , $saxonProc->createAtomicValue($pid) 
      );
    $dc_xml = $to_dc_xslt_proc->transformToString();

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    // determine collection based on workflow
    $legacy_dom = new DOMDocument();
    $legacy_dom->resolveExternals = true;
    $legacy_dom->loadXML($legacy_content);
    $xpathObj = new DOMXpath($legacy_dom); 
    $pubCRes = $xpathObj->query($xpathStrPub);
    $collection_pid = $collection_pid_non_pub_c;
    if (!is_null($pubCRes) and $pubCRes->length>0)
    {
      $collection_pid = $collection_pid_pub_c;
      drush_print("pub-c");
    } 

    // prepare a new Fedora object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    // with the legacy content
    $cwrc_ds = $object->constructDatastream($main_ds_id, 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($cwrc_ds);
 
    // create LEGACY_ENTITY datastream from Doc Archive contents <=2019
    $legacy_ds = $object->constructDatastream('LEGACY_'.$main_ds_id, 'M');
    $legacy_ds->label = "legacy_" . $file_name;
    $legacy_ds->mimeType = 'text/xml';
    $legacy_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($legacy_ds);
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($clean_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      //create_DS_POLICY($policy, $object);
      $policy_ds
          = new IslandoraXacml(
              $object
              , file_get_contents($policy)
              );
      $policy_ds->writeBackToFedora();
    }

    try {
      // Store the object.
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Initial ingest into the CWRC repository.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      // update content datastream: UTF-8 and ISO 8601 dates
      $object[$main_ds_id]->setContentFromString($clean_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "UTF-8 and ISO 8601 dates.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      //update URI's refences in entities
      if ($main_ds_id == 'MODS')
      {
        $object[$main_ds_id]->setContentFromString($content_uri_xml);
        cwrc_migration_batch_add_entity_refs_workflow($object, $main_ds_id);
      }

      // update content datastream with version 2.0 of the content
      // UTF-8 and ISO 8601 dates (from previous steps)
      $object[$main_ds_id]->setContentFromString($content_v2_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Conversion to version 2.0 of the Orlando format.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      if ($add_pub_c_workflow==='yes')
      {
        // ToDo: assume all name and org are pub-c as input authority lists don't contain workflow
        // add a workflow stamp indicating that the object was ingested
        $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
        $workflow->addWorkflowStep(
          $object->id, 
          new WorkflowItem(WorkflowConst::WORKFLOW, array()),
          new WorkflowItem(
            WorkflowConst::ACTIVITY,
            array(
              "category" => "machine_processed",
              'stamp' => 'orlando:PUB',
              'status' => 'c',
              "note" => "Update workflow to PUB-C.",
              )
            )
          );
        CWRCWorkflowAPI::updateDatastream($workflow, $object);
      }
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }

  drush_print("person: " . count($entity_mapping["NAME"]));
  drush_print("org: " . count($entity_mapping["ORGNAME"]));
  drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
  entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}

/** 
 * CWRC migration batch - Entities to Orlando version 3 - 2020-10-23 - reuse PIDs 
 * workflow is pulled from the legacy Orlando format stored in subdir
 *
 */
function drush_cwrc_migration_batch_ingest_orlando_entities_reuse_pids() 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("Orlando entity migration batch version 2.0");

  // path to the input document files 
  $legacy_dir = drush_get_option(
    'legacy_dir'
    , '/tmp/orlando/bio_writ'
    );

  // path to entity mapping file
  $entity_mapping_file = drush_get_option(
    'entity_mapping_file'
    , '/tmp/orlando/entity_mapping.json'
    );

  // 
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_to_clean = drush_get_option(
    'xslt_to_clean'
    , $ext_home . 'transforms/orlando_legacy_to_v2_bio_writing.xsl'
  );
  $xslt_to_v2 = drush_get_option(
    'xslt_to_v2'
    , $ext_home . ''
  );
  $xslt_to_dc = drush_get_option(
    'xslt_to_dc'
    , libraries_get_path('islandora_cwrc_xslt_library') . '/xslt/mods_to_dc.xsl'
  );
  
  $main_ds_id = drush_get_option(
    'main_ds_id'
    , 'MODS'
  );
  $entity_type = drush_get_option(
    'entity_type'
    , 'BIBCIT'
  );
  $collection_pid_pub_c = drush_get_option(
    'collection_pid_pub_c'
    , 'cwrc:fc271c04-7fd1-46b4-9462-339dbe917c46'
  );
  $collection_pid_non_pub_c = drush_get_option(
    'collection_pid_non_pub_c'
    , 'cwrc:bf8ba2b7-dbed-440b-9000-88860be1bcc9'
  );
  $cModel_pid = drush_get_option(
    'cModel_pid' , 'cwrc:documentCModel'
  );
  $policy = drush_get_option(
    'XACML_policy', $ext_home . 'xml/orlando_default_XACML_Policy_Stream.xml'
  );
  $namespace = drush_get_option(
    'namespace', 'orlando'
  );
  $supplemental_filename = drush_get_option(
    'supplemental_filename', ''
  );
  $add_pub_c_workflow = drush_get_option(
    'add_pub_c_workflow', 'no'
  );


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  
  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $to_clean_xslt_proc = $saxonProc->newXsltProcessor();
  $to_v2_xslt_proc = $saxonProc->newXsltProcessor();
  $to_dc_xslt_proc = $saxonProc->newXsltProcessor();

  $to_clean_xslt_proc->compileFromFile($xslt_to_clean);
  $to_v2_xslt_proc->compileFromFile($xslt_to_v2);
  $to_dc_xslt_proc->compileFromFile($xslt_to_dc);

  // Initialize entity mapping: a JSON file ofPID to standard name 
  // entity mapping file used to add URI's to bio/writing/event',
  try {
    drush_print("Entity mapping file: " . $entity_mapping_file);
    entity_mapping_load($entity_mapping, $entity_mapping_file);
    drush_print("person: " . count($entity_mapping["NAME"]));
    drush_print("org: " . count($entity_mapping["ORGNAME"]));
    drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
    // write the entity mapping file
    entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);
  } catch (Exception $e) {
    drush_print($e);
    exit();
  }

  // xpath to determine if PUB-C Orlando document -  BIBL Legacy XPath
  $xpathStrPub = "//RESPONSIBILITY[@WORKSTATUS='PUB' and @WORKVALUE='C']";

  // Set source directory for the legacy Orlando content
  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  drush_print("Input: " . $legacy_dir);
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;
    $legacy_id = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }
   
    $content_datastream_label = preg_replace('/(\.sgm)$/','.xml', $file_name); 

    // Load xml from a file.
    $legacy_content = file_get_contents($legacy_dir.$file_name);

    // build the UTF-8 version with conversion of Orlando dates
    $to_clean_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($legacy_content)
    );
    $clean_xml = $to_clean_xslt_proc->transformToString();

    $legacy_id = entity_mapping_extract_id($clean_xml);
    
    // get id for object from entity mapping
    $pid =  entity_mapping_lookup($entity_mapping, $legacy_id, "BIBCIT");
    if (!isset($pid) or empty($pid)) {
      drush_print("ERROR: $legacy_id not found in entity_mapping");
      continue;
    }

    // If the object already exists then continue to next file.
    if ( $object = islandora_object_load($pid)) {
      drush_print("INFO: object legacy[$legacy_id] with pid [$pid] is already in fedora - DELETING");
      $object->repository->purgeObject($pid);
    }
 
    // add ID and PID to the entity mapping
    // ToDo: assumption - use UTF-8 version
    if ( isset($pid) && isset($legacy_id) ) {
      entity_mapping_add($entity_mapping, $legacy_id, $pid, $entity_type);
      drush_print("Entity ID [$legacy_id] added with value [$pid]");
    }
    else {
      drush_print("Entity ID [$legacy_id] not found");
    }

    if ($main_ds_id=='MODS') {
      // update URI's refences in bibliographic entities
      // requires UTF-8 
      $content_uri_xml = cwrc_migration_batch_add_entity_refs_content($clean_xml, $main_ds_id, $entity_mapping);
    }
    else {
      // for other entity 
      $content_uri_xml = $clean_xml;
    }

    // build the version 2.0 
    $to_v2_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_uri_xml)
    );
    if ($supplemental_filename)
    {
      $to_v2_xslt_proc->setParameter(
        "supplemental_filename"
         , $saxonProc->createAtomicValue($supplemental_filename) 
      );
    }
    $content_v2_xml = $to_v2_xslt_proc->transformToString();

    //drush_print($content_v2_xml);

    // delete doctype
    $content_v2_xml = str_replace(
        '<!DOCTYPE ORLANDO [ <!ENTITY % character_entities SYSTEM "http://cwrc.ca/schemas/character_entities.dtd"> %character_entities; ]>\n' 
        , ""
        , $content_v2_xml
      );

    // build DC 
    $to_dc_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($content_v2_xml)
    );
    $to_dc_xslt_proc->setParameter(
        "PID_PARAM"
         , $saxonProc->createAtomicValue($pid) 
      );
    $dc_xml = $to_dc_xslt_proc->transformToString();

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    // determine collection based on workflow
    $legacy_dom = new DOMDocument();
    $legacy_dom->resolveExternals = true;
    $legacy_dom->loadXML($legacy_content);
    $xpathObj = new DOMXpath($legacy_dom); 
    $pubCRes = $xpathObj->query($xpathStrPub);
    $collection_pid = $collection_pid_non_pub_c;
    if (!is_null($pubCRes) and $pubCRes->length>0)
    {
      $collection_pid = $collection_pid_pub_c;
      drush_print("pub-c");
    } 

    // prepare a new Fedora object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    // with the legacy content
    $cwrc_ds = $object->constructDatastream($main_ds_id, 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($cwrc_ds);
 
    // create LEGACY_ENTITY datastream from Doc Archive contents <=2019
    $legacy_ds = $object->constructDatastream('LEGACY_'.$main_ds_id, 'M');
    $legacy_ds->label = "legacy_" . $file_name;
    $legacy_ds->mimeType = 'text/xml';
    $legacy_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($legacy_ds);
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // Workflow Datastream.
    // We want to get all of the responsibilitie tags.
    // For each responsibility create a workflow element.
    // Create the MODS datastream.
    create_DS_workflow_from_Source($clean_xml, $object);

    // Create the Policy datastream.
    if ( $policy != null && $policy!="NONE" ) 
    {
      //create_DS_POLICY($policy, $object);
      $policy_ds
          = new IslandoraXacml(
              $object
              , file_get_contents($policy)
              );
      $policy_ds->writeBackToFedora();
    }

    try {
      // Store the object.
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Initial ingest into the CWRC repository.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      // update content datastream: UTF-8 and ISO 8601 dates
      $object[$main_ds_id]->setContentFromString($clean_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "UTF-8 and ISO 8601 dates.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      //update URI's refences in entities
      if ($main_ds_id == 'MODS')
      {
        $object[$main_ds_id]->setContentFromString($content_uri_xml);
        cwrc_migration_batch_add_entity_refs_workflow($object, $main_ds_id);
      }

      // update content datastream with version 2.0 of the content
      // UTF-8 and ISO 8601 dates (from previous steps)
      $object[$main_ds_id]->setContentFromString($content_v2_xml);

      // add a workflow stamp indicating that the object was ingested
      $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "machine_processed",
            'stamp' => 'orlando:SUB',
            'status' => 'c',
            "note" => "Conversion to version 2.0 of the Orlando format.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

      if ($add_pub_c_workflow==='yes')
      {
        // ToDo: assume all name and org are pub-c as input authority lists don't contain workflow
        // add a workflow stamp indicating that the object was ingested
        $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
        $workflow->addWorkflowStep(
          $object->id, 
          new WorkflowItem(WorkflowConst::WORKFLOW, array()),
          new WorkflowItem(
            WorkflowConst::ACTIVITY,
            array(
              "category" => "machine_processed",
              'stamp' => 'orlando:PUB',
              'status' => 'c',
              "note" => "Update workflow to PUB-C.",
              )
            )
          );
        CWRCWorkflowAPI::updateDatastream($workflow, $object);
      }
    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }

  drush_print("person: " . count($entity_mapping["NAME"]));
  drush_print("org: " . count($entity_mapping["ORGNAME"]));
  drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
  entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}



/** 
 * CWRC migration batch - CEWW - 2018-05-17
 * assumes MODS and DC generated on the fly 
 * workflow is pulled from a file 
 *
 */
function drush_cwrc_migration_batch_ingest_ceww_entries() 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("CEWW entries ingest");

  // path to the input document files 
  $legacy_dir = drush_get_option('legacy_dir','/tmp/ceww');

  // 
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_to_mods = drush_get_option(
    'xslt_to_mods'
    , $ext_home . 'transforms/ceww_entry2mods.xsl'
  );
  $xslt_mods_to_dc = drush_get_option(
    'xslt_mods_to_dc'
    , libraries_get_path('islandora_cwrc_xslt_library') . '/xslt/mods_to_dc.xsl'
  );
  $main_ds_id = drush_get_option(
    'main_ds_id'
    , 'CWRC'
  );
  $collection_pid = drush_get_option(
    'collection_pid'
    , 'ceww:ee081346-e5ba-4b95-a90a-33d127ea85c8'
  );
  $cModel_pid = drush_get_option(
    'cModel_pid' , 'cwrc:documentCModel'
  );
  $schema_object = drush_get_option(
    'schema_object', 'cwrc:fe081346-e5ba-4b95-a90a-33d127ea85c8'
  );
  $policy = drush_get_option(
    'XACML_policy', $ext_home . 'xml/ceww_default_XACML_Policy_Stream.xml'
  );
  $namespace = drush_get_option(
    'namespace', 'ceww'
  );


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  
  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  //xslt_setup($xslt_bio_writ_to_v2, $bio_writ_to_v2_xslt_proc);
  //xslt_setup($xslt_bio_writ_to_mods, $bio_writ_to_mods_xslt_proc);
  //xslt_setup($xslt_mods_to_dc, $mods_to_dc_xslt_proc);
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $to_mods_xslt_proc = $saxonProc->newXsltProcessor();
  $mods_to_dc_xslt_proc = $saxonProc->newXsltProcessor();

  $to_mods_xslt_proc->compileFromFile($xslt_to_mods);
  $mods_to_dc_xslt_proc->compileFromFile($xslt_mods_to_dc);

  //determine the schema for the object
  drush_print("Schema: $schema_object");
  $schema_obj = islandora_object_load($schema_object);
  if (empty($schema_obj)) {
    drush_print("Schema missing - terminating");
    exit;
  }

  // Set source directory for the legacy Orlando content
  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  drush_print("Input: " . $legacy_dir);
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier($namespace, TRUE);

    // Strip out the .xml from the file name and store in the variable.
    //preg_match('/[a-z0-9-_]*/', $file_name, $object_name);
    // Strip out file name to get type -b(biography) or -w (writable).
    //preg_match('/-[a-z]?/', $file_name, $object_type);
    //$object_name = $object_name[0];
    //$object_type = $object_type[0];
    //$pid = "cwrc:$object_name";

    // If the object already exists then continue to next file.
    if (islandora_object_load($pid)) {
      drush_print("object $object_name is already in fedora");
      continue;
    }
    
    $content_datastream_label = preg_replace('/(\.sgm)$/','.xml', $file_name); 


    // Load xml from a file.
    $legacy_content = file_get_contents($legacy_dir.$file_name);

    // Build MODS XML.
    $to_mods_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($legacy_content)
      );
    $mods_xml = $to_mods_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $mods_xml);

    //$mods_dom = new DOMDocument();
    //$mods_dom->load($mods_xml);

    // build the DC content
    // Convert MODS to DC.
    // Transform to DC xml.
    // Apply the stylesheet.
    //$mods_to_dc_xslt_proc->setParameter( "", "PID_PARAM", $pid );
    //$dc_xml = $mods_to_dc_xslt_proc->transformToString($mods_dom);
    $mods_to_dc_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($mods_xml)
    );
    $dc_xml = $mods_to_dc_xslt_proc->transformToString();

    //print ("zzzzzzzzzzzzzzzzzzzz " . $dc_xml);

    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");



    // prepare a new Fedora object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    // with the legacy content
    $cwrc_ds = $object->constructDatastream($main_ds_id, 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create MODS datastream
    create_DS_MODS($mods_xml, $object);

    islandora_cwrc_writer_set_document_schema($object, $schema_obj);

    try {
      // Store the object.
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");

      // add a workflow stamp indicating that the object was ingested
      if (!$object['WORKFLOW']) {
        $workflow = CWRCWorkflowAPI::fromTemplate($object['WORKFLOW']);
      }
      else {
        $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      }
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "published",
            'stamp' => 'cwrc:cre',
            'status' => 'c',
            "note" => "Initial ingest into the CWRC repository.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}


/** 
 * CWRC migration batch - Generic - 2018-07-31
 * assumes MODS and DC generated on the fly 
 * MODS optional
 *
 */
function drush_cwrc_migration_batch_ingest_generic() 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("Generic ingest");

  // path to the input document files 
  $legacy_dir = drush_get_option('legacy_dir','/tmp/ceww');

  // if null then no MODS datastream 
  $xslt_to_mods = drush_get_option(
    'xslt_to_mods'
    , ''
  );
  $xslt_to_dc = drush_get_option(
    'xslt_to_dc'
    , libraries_get_path('islandora_cwrc_xslt_library') . '/xslt/mods_to_dc.xsl'
  );
  $main_ds_id = drush_get_option(
    'main_ds_id'
    , 'CWRC'
  );
  $collection_pid = drush_get_option(
    'collection_pid'
    , 'ceww:ee081346-e5ba-4b95-a90a-33d127ea85c8'
  );
  $cModel_pid = drush_get_option(
    'cModel_pid' , 'cwrc:documentCModel'
  );
  $schema_object = drush_get_option(
    'schema_object', ''
  );
  $policy = drush_get_option(
    'XACML_policy', $ext_home . 'xml/ceww_default_XACML_Policy_Stream.xml'
  );
  $namespace = drush_get_option(
    'namespace', 'ceww'
  );


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  
  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $to_mods_xslt_proc = $saxonProc->newXsltProcessor();
  $to_dc_xslt_proc = $saxonProc->newXsltProcessor();

  $is_mods = false;
  if (empty($xslt_to_mods) || @xslt_to_mods == '' ) {
    drush_print("No MODS");
    $is_mods = false;
  } else {
    drush_print("To MODS XSLT: " . $xslt_to_mods);
    $to_mods_xslt_proc->compileFromFile($xslt_to_mods);
    $is_mods = true;
  }
  drush_print("To DC XSLT: " . $xslt_to_dc);
  $to_dc_xslt_proc->compileFromFile($xslt_to_dc);

  //determine the schema for the object
  drush_print("Schema: $schema_object");
  if (!empty($schema_object) || $schema_object !== '') {
    $schema_obj = islandora_object_load($schema_object);
    if (empty($schema_obj)) {
      drush_print("Schema missing - terminating");
      exit;
    }
  }
  else {
    drush_print("Schema empty - no schema association");
  }

  // Set source directory for the legacy Orlando content
  // Open the source direcotry and iterate through every file in the 
  // directory - one file maps to one Fedora 
  drush_print("Input: " . $legacy_dir);
  $src_dir_handle = opendir($legacy_dir);

  drush_print("DS main: " . $main_ds_id);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle))) 
  {
    $pid = null;

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($file_name);
    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }

    // generate an id for the object
    $pid = $connection->repository->getnextIdentifier($namespace, TRUE);

    // Strip out the .xml from the file name and store in the variable.
    //preg_match('/[a-z0-9-_]*/', $file_name, $object_name);
    // Strip out file name to get type -b(biography) or -w (writable).
    //preg_match('/-[a-z]?/', $file_name, $object_type);
    //$object_name = $object_name[0];
    //$object_type = $object_type[0];
    //$pid = "cwrc:$object_name";

    // If the object already exists then continue to next file.
    if (islandora_object_load($pid)) {
      drush_print("object $object_name is already in fedora");
      continue;
    }
    
    $content_datastream_label = preg_replace('/(\.sgm)$/','.xml', $file_name); 

    // Load xml from a file.
    $legacy_content = file_get_contents($legacy_dir.$file_name);

    // Build MODS XML.
    if ($is_mods) {
      $to_mods_xslt_proc->setSourceFromXdmValue(
        $saxonProc->parseXmlFromString($legacy_content)
        );
      $mods_xml = $to_mods_xslt_proc->transformToString();

      //print ("zzzzzzzzzzzzzzzzzzzz " . $mods_xml);

      //$mods_dom = new DOMDocument();
      //$mods_dom->load($mods_xml);

      // build the DC content
      // Convert MODS to DC.
      // Transform to DC xml.
      // Apply the stylesheet.
      //$dc_xml = $mods_to_dc_xslt_proc->transformToString($mods_dom);
      $to_dc_xslt_proc->setParameter( "PID_PARAM", $saxonProc->createAtomicValue($pid) );
      $to_dc_xslt_proc->setSourceFromXdmValue(
        $saxonProc->parseXmlFromString($mods_xml)
      );
      $dc_xml = $to_dc_xslt_proc->transformToString();
    } else {
      $to_dc_xslt_proc->setParameter( "PID_PARAM", $saxonProc->createAtomicValue($pid) );
      $to_dc_xslt_proc->setSourceFromXdmValue(
        $saxonProc->parseXmlFromString($legacy_content)
      );
      $dc_xml = $to_dc_xslt_proc->transformToString();
      //print ("zzzzzzzzzzzzzzzzzzzz " . $dc_xml);
    }


    // set object title
    $title = get_dcTitle_from_dc($dc_xml);

    //print("dc:title extracted - \"$title\" \n");



    // prepare a new Fedora object.
    $object
      = islandora_prepare_new_object(
        $pid 
        , $title
        , array()
        , array($cModel_pid)
        , array(
          array(
            'relationship' => 'isMemberOfCollection'
            , 'pid' => "$collection_pid"
          )
        )
      );


    // create content datastream
    // with the legacy content
    $cwrc_ds = $object->constructDatastream($main_ds_id, 'M');
    $cwrc_ds->label = $file_name;
    $cwrc_ds->mimeType = 'text/xml';
    $cwrc_ds->setContentFromString($legacy_content);
    $object->ingestDatastream($cwrc_ds);
 
    
    // create DC datastream
    create_DS_DC($dc_xml, $object);

    // create MODS datastream
    if ($is_mods) {
      create_DS_MODS($mods_xml, $object);
    }

    if (!empty($schema_object) || $schema_object !== '') {
      islandora_cwrc_writer_set_document_schema($object, $schema_obj);
    }

    try {
      // Store the object.
      islandora_add_object($object);
      $file_count++;
      drush_print("pid=$pid title=\"$title\" was added. #{$file_count} file migrated.");

      // add a workflow stamp indicating that the object was ingested
      if (!$object['WORKFLOW']) {
        $workflow = CWRCWorkflowAPI::fromTemplate($object['WORKFLOW']);
      }
      else {
        $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
      }
      $workflow->addWorkflowStep(
        $object->id, 
        new WorkflowItem(WorkflowConst::WORKFLOW, array()),
        new WorkflowItem(
          WorkflowConst::ACTIVITY,
          array(
            "category" => "published",
            'stamp' => 'cwrc:cre',
            'status' => 'c',
            "note" => "Initial ingest into the CWRC repository.",
            )
          )
        );
      CWRCWorkflowAPI::updateDatastream($workflow, $object);

    }
    catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}




/** 
 * CWRC migration batch - validate cwrc:documentCModel objects - 2016-11-29
 * run an XML schema validation service against the specificed datastream
 * and schema 
 *
 */
function drush_cwrc_migration_batch_validate() 
{
  module_load_include('inc', 'cwrc_migration_batch', 'includes/helpers');

  drush_print("Validation:");

  // collection PID to define set of input documents 
  $collection_pid = drush_get_option('collection_pid' ,'');

  // the datastream to use with the validation test 
  $ds_id = drush_get_option('ds_id', 'CWRC');

  // the default schema to use if the xml-model processing instruction
  // lookup fails
  $override_rng_schema = drush_get_option(
    'override_rng_schema'
    , '' // default empty unless set
  );

  $validation_url = drush_get_option(
    'validation_url'
    , 'http://localhost:8080/validator/validate.html'
  );

  //get all members of the specified collection 
  $query = <<<EOT
    PREFIX fre: <info:fedora/fedora-system:def/relations-external#>
    PREFIX fm: <info:fedora/fedora-system:def/model#>
    SELECT ?obj
    FROM <#ri>
    WHERE {
      {
        ?obj fre:isMemberOfCollection ?sub
      }
      UNION
      {
        ?obj fre:isMemberOf ?sub
      }
      ?obj fm:state fm:Active .
      ?sub fm:state fm:Active
      FILTER(sameTerm(?sub, <info:fedora/$collection_pid>))
    }
EOT;

  drush_print($query);
  drush_print("Results:");

  $tuque = islandora_get_tuque_connection();
  $repo = $tuque->repository;
  $results = $repo->ri->sparqlQuery($query);

  // foreach member of the collection, run the validation test  
  if (!empty($results)) 
  { 
    foreach ($results as $result) 
    {
      $pid = $result['obj']['value'];

      //drush_print($pid);

      //lookup the object
      $object = islandora_object_load($pid);
      if (!$object) {
        drush_print("Error: object not found $pid");
        continue;
      }

      //contains datastream ID
      $ds = $object[$ds_id];
      if (!$ds)
      {
        drush_print("Error: datastream [$ds_id] not found $pid");
        continue;
      }

      $content = $ds->content;
      //drush_print($content);


      if (!empty($override_rng_schema)) {
        // if override then use
        $schema = $override_rng_schema;
      } else if ($ds_id === 'ENTRY' and !empty($override_rng_schema)) {
        // special handling for the CWRC-Writer enabled modules - DS ID = "ENTRY" use attached schema object
        $schema = determine_schema_url_from_rels_ext($object);
      }
      else {
        // otherwise use the schema at the top of the content
        $schema = determine_schema_url_from_xml($content, $override_rng_schema);
      }

      $validation_res = validate_content($content, $schema, $validation_url);
      drush_print("INFO: [$pid] using [$schema] - result: [$validation_res]");

      // update URI's refences in entities
      // if ($object['CWRC'])
      // {
      //   cwrc_migration_batch_add_entity_refs(object, "CWRC");
      // }

    }
  }
}


/** 
 * Orlando verify batch 
 * compare the files for migration to the results after migration 
 *
 */
function drush_cwrc_migration_batch_verify_orlando() 
{
  module_load_include('inc', 'cwrc_migration_batch', 'includes/helpers');

  drush_print("Verification:");

  // path to the input document files 
  $legacy_dir = drush_get_option(
    'legacy_dir'
    , '/tmp/orlando/bio_writ'
    );

  // path to entity mapping file
  $entity_mapping_file = drush_get_option(
    'entity_mapping_file'
    , '/tmp/orlando/entity_mapping.json'
    );

  $entity_type = drush_get_option(
    'main_ds_id'
    , 'CWRC'
  );
  
  $item_type = drush_get_option(
    'item_type'
    , 'EVENT'
  );

  // convert to UTF-8
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_to_clean = drush_get_option(
    'xslt_to_clean'
    , $ext_home . 'transforms/orlando_legacy_to_v2_bio_writing.xsl'
  );

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Initialize entity mapping: a JSON file ofPID to standard name
  // entity mapping file used to add URI's to bio/writing/event',
  try {
    drush_print("Entity mapping file: " . $entity_mapping_file);
    entity_mapping_load($entity_mapping, $entity_mapping_file);
    drush_print("person: " . count($entity_mapping["NAME"]));
    drush_print("org: " . count($entity_mapping["ORGNAME"]));
    drush_print("bibl: " . count($entity_mapping["BIBCIT"]));
    drush_print("event: " . count($entity_mapping["EVENT"]));
    drush_print("entry: " . count($entity_mapping["ENTRY"]));
    // write the entity mapping file
    entity_mapping_write_to_file($entity_mapping, $entity_mapping_file);
  } catch (Exception $e) {
    drush_print($e);
    exit();
  }

  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());

  $to_clean_xslt_proc = $saxonProc->newXsltProcessor();
  $to_clean_xslt_proc->compileFromFile($xslt_to_clean);

  // Set source directory for the legacy Orlando content
  // Open the source directory and iterate through every file in the
  // directory - one file maps to one Fedora
  drush_print("Input: " . $legacy_dir);
  $src_dir_handle = opendir($legacy_dir);

  // Counter for the number of files to migrate.
  $file_count = 0;
  $file_error_count = 0;
  while (FALSE !== ($file_name = readdir($src_dir_handle)))
  {
    drush_print($file_name);

    // If the file is a directory then continue to next file.
    if (is_dir($file_name)) {
      continue;
    }    

    
    $legacy_content = file_get_contents($legacy_dir.$file_name);
    // build the UTF-8 version with conversion of Orlando dates
    $to_clean_xslt_proc->setSourceFromXdmValue(
      $saxonProc->parseXmlFromString($legacy_content)
      );
    $clean_xml = $to_clean_xslt_proc->transformToString();

    $json_status = 'fail';
    $fedora_status = 'fail';
    $pid = cwrc_migration_batch_lookup_pid($entity_mapping, $item_type, $file_name, $clean_contents);
    if (isset($pid) or !empty($pid)) {
      $json_status = "success";
      // if not in the json file then fedora test will also fail
      $fedora_status = cwrc_migration_batch_verify_fedora($item_type, $pid, $file_name);
    } 

    drush_print("Verify [$file_name] in json mapping: [$json_status] with pid: [$pid] in fedora: [$fedora_status]");
  }

}

function cwrc_migration_batch_lookup_pid($entity_mapping, $item_type, $file_name, $xml) 
{
  $item_id = null;
  switch ($item_type) {
    case 'EVENT':
      $item_id = $file_name;
      break;
    case 'ENTRY':
      $item_id = preg_replace('/^([a-zA-Z_0-9]{6,6})-[wb]\.xml/','$1', $file_name);
      break;
    case 'NAME':
    case 'ORGNAME':
    case 'BIBCIT':
      $item_id = entity_mapping_extract_id ($xml);
      break;
  }    
  return $entity_mapping[$item_type][$item_id];
}

function cwrc_migration_batch_verify_fedora($item_type, $pid, $file_name) 
{
  $status = fail;
  //$status = islandora_object_load($pid) ? "success" : "fail";
  $obj = islandora_object_load($pid);
  if ($item_type === 'ENTRY') {
    $entry_type = preg_replace('/^[a-zA-Z_0-9]{6,6}-([wb])\.xml/','$1', $file_name);
    $xml_match = ($entry_type === 'w') ? '/<WRITING/' : '/<BIOGRAPHY/';
    
    if (!empty($obj)
      and !empty($obj['CWRC'])
      and !empty($obj['WORKFLOW'])
      and !empty($obj['MODS'])
      and preg_match($xml_match, $obj['CWRC']->content)
      ) {
        $status = 'success';
      }
  } else if ($item_type === 'EVENT') {
    if (!empty($obj)
      and !empty($obj['CWRC'])
      and !empty($obj['WORKFLOW'])
      and !empty($obj['MODS'])
      and preg_match('<EVENT', $obj['CWRC']->content)
      ) {
        $status = 'success';
      }
  } else if ($item_type === 'NAME') {
    if (!empty($obj)
      and !empty($obj['NAME'])
      and !empty($obj['WORKFLOW'])
      ) {
        $status = 'success';
      }
  } else if ($item_type === 'ORGNAME') {
    if (!empty($obj)
      and !empty($obj['ORGNAME'])
      and !empty($obj['WORKFLOW'])
      ) {
        $status = 'success';
      }
  } else if ($item_type === 'BIBCIT') {
    if (!empty($obj)
      and !empty($obj['MODS'])
      and !empty($obj['WORKFLOW'])
      ) {
        $status = 'success';
      }
  }

  return $status;
}


function cwrc_migration_batch_add_entity_refs_content($content, $ds_id, $entity_mapping) 
{

  // add entity references to elements in content
  if ($ds_id != "MODS") {
    $content = add_entity_references_via_element(
      $content
      , "info:fedora/cwrc:person-entityCModel"
      , "STANDARD"
      , "REF"
      , null
      , ISLANDORA_ORLANDO_MIGRATE_PERSON_ORG
      , "NAME"
      , $entity_mapping
    );
  }
  $content = add_entity_references_via_element(
    $content
    , "info:fedora/cwrc:person-entityCModel"
    , "NAME"
    , "REF"
    , "STANDARD"
    , ISLANDORA_ORLANDO_MIGRATE_PERSON_ORG
    , "NAME"
    , $entity_mapping
  );
  $content = add_entity_references_via_element(
    $content
    , "info:fedora/cwrc:organization-entityCModel"
    , "ORGNAME"
    , "REF"
    , "STANDARD"
    , ISLANDORA_ORLANDO_MIGRATE_PERSON_ORG
    , "ORGNAME"
    , $entity_mapping
  );
  if ($ds_id != "MODS") {
    $content = add_entity_references_via_element(
      $content
      , "info:fedora/cwrc:citationCModel"
      , "TEXTSCOPE"
      , "REF"
      , "DBREF"
      , ISLANDORA_ORLANDO_MIGRATE_BIBL
      , "BIBCIT"
      , $entity_mapping
    );
    $content = add_entity_references_via_element(
      $content
      , "info:fedora/cwrc:citationCModel"
      , "BIBCIT"
      , "REF"
      , "DBREF"
      , ISLANDORA_ORLANDO_MIGRATE_BIBL
      , "BIBCIT" 
      , $entity_mapping
    );
  }

  // update MODS with a URI
  // XQuery XMLDatabase version 
  // if ($ds_id == "MODS") {
  // $content = add_entity_references_via_xpath(
    // $content
    // , "info:fedora/cwrc:person-entityCModel"
    // , "/mods:mods/mods:subject/mods:name[@type=\"personal\"]"
    // , "valueURI"
    // , ""
    // , "orlando_migration/orlando_entity_lookup_person_org.xq"
    // , "NAME" 
    // , $entity_mapping
    // );
  // }


  return $content;

}

/** 
 * CWRC migration batch - add @REF attributes to the core entity elements
 * (name/org/bibcit/textscope/standard/etc.) 
 *
 */
function cwrc_migration_batch_add_entity_refs_workflow($object, $ds_id) 
{

  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');
  module_load_include('inc', 'islandora_cwrc_basexdb', 'includes/xmldb_endpoints');

  // add a workflow stamp indicating that the object was ingested
  $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
  $workflow->addWorkflowStep(
    $object->id,
    new WorkflowItem(WorkflowConst::WORKFLOW, array()),
    new WorkflowItem(
      WorkflowConst::ACTIVITY,
      array(
        "category" => "machine_processed",
        'stamp' => 'orlando:SUB',
        'status' => 'c',
        "note" => "Add URIs to entities references in $ds_id content.",
        )
      )
    );
  CWRCWorkflowAPI::updateDatastream($workflow, $object);
}


/** 
 * CWRC migration batch - add @REF attributes to the core entity elements
 * (name/org/bibcit/textscope/standard/etc.) 
 *
 */
function cwrc_migration_batch_add_entity_refs($object, $ds_id, $entity_mapping) 
{

  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');
  module_load_include('inc', 'islandora_cwrc_basexdb', 'includes/xmldb_endpoints');


  //contains datastream ID
  $ds = $object[$ds_id];
  if (!$ds)
  {
    drush_print("Error: datastream [$ds_id] not found $pid");
  }
  else {
    $content = $ds->content;

    $content = cwrc_migration_batch_add_entity_refs_content($content, $ds_id, $entity_mapping);

    // update object
    $ds->setContentFromString($content);
  }

  // add a workflow stamp indicating that the object was ingested
  $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
  $workflow->addWorkflowStep(
    $object->id,
    new WorkflowItem(WorkflowConst::WORKFLOW, array()),
    new WorkflowItem(
      WorkflowConst::ACTIVITY,
      array(
        "category" => "machine_processed",
        'stamp' => 'orlando:PUB',
        'status' => 'c',
        "note" => "Add URIs to entities.",
        )
      )
    );
  CWRCWorkflowAPI::updateDatastream($workflow, $object);
}


/** 
 * CWRC migration batch - CEWW - 2018-05-29
 * update subject 
 *
 */
function drush_cwrc_migration_batch_ceww_add_subject() 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("CEWW: Add subject");


  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");

  $src_dsid = "CWRC";
  $dst_dsid = "MODS";

  // Counter for the number of files.
  $file_count = 0;
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print($pid);

    $cwrc_dom = new DOMDocument;
    $mods_dom = new DOMDocument;
    $subject_dom = new DOMDocument;

    try {
      $object = islandora_object_load($pid);

      if (!empty($object[$src_dsid]) and !empty($object[$dst_dsid])) {

        $mods_dom->loadXML($object[$dst_dsid]->content);
        $cwrc_dom->loadXML($object[$src_dsid]->content);

        $mods_xpath = new DOMXpath($mods_dom);
        $cwrc_xpath = new DOMXpath($cwrc_dom);

        $mods_subject = $mods_xpath->query("/mods:mods/mods:subject");

        if ( $mods_subject->length == 0) {
          $name_ref = $cwrc_xpath->query("(/CWRC/ENTRY//HEADING)[1]/NAME/@REF")->item(0)->nodeValue;
          $name_str = $cwrc_xpath->query("(/CWRC/ENTRY//HEADING)[1]/NAME")->item(0)->nodeValue;

          drush_print("Subject: $name_str - $name_ref");

          $subject_str = <<<XML
                  <subject>
                    <name valueURI="$name_ref">
                      <namePart>$name_str</namePart>
                    </name>
                  </subject>
XML;

          //drush_print("Add str: \n $subject_str");
          $subject_dom->loadXML($subject_str);
          $subject_dom = $mods_dom->importNode($subject_dom->documentElement,TRUE); 
          $mods_dom->documentElement->appendChild($subject_dom); 
          $object[$dst_dsid]->content = $mods_dom->saveXML();
          $file_count++;
        }
        else {
          // skip is exists
          drush_print("Skipping " . $pid . " - contains subject");
          next;
        }
      }
      else {
        // skip is exists
        drush_print("Skipping " . $pid . ". ");
        next;
      }

    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid \"$name_str\" was not updated.");
      drush_print($e->getMessage());
    }

  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}


/** 
 * CWRC migration batch - CEWW - 2018-05-29
 * add Thumbnail
 *
 */
function drush_cwrc_migration_batch_thumbnail_entries() 
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("Add thumbnail");


  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // path to the thumbnail to add to object specified by PID
  $tn_path = drush_get_option(
    'tn_path'
  );
  
  // label
  $tn_label = drush_get_option(
    'tn_label'
    , 'TN'
  );

  // object datastream to modify
  $ds_id = drush_get_option(
    'ds_id'
    , 'TN'
  );

  // mimetype
  $ds_mimetype = drush_get_option(
    'ds_mimetype',
    file_get_mimetype($tn_path)
    );

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  drush_print("Mimetype: $ds_mimetype");
  drush_print("TN File: $tn_path");
  try {
    $file_handle = fopen($tn_path, "r");
    fclose($file_handle);
  } catch (Exception $e) {
    drush_print($e->getMessage());
    exit;
  }

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");

  // Counter for the number of files.
  $file_count = 0;
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("PID: $pid");

    try {
      $object = islandora_object_load($pid);

      if (empty($object[$ds_id])) {
        $ds = $object->constructDatastream($ds_id, 'M');
        $ds->setContentFromFile($tn_path, FALSE);
        $ds->label = $tn_label;
        $ds->mimetype = $ds_mimetype;
        $object->ingestDatastream($ds);
        $file_count++;
      }
      else {
        // skip is exists
        drush_print("Skipping " . $pid . " as datastream exists");
        next;
      }

    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}


/** 
 * CWRC migration batch - retrieve content of specified item 
 *
 */
function drush_cwrc_migration_batch_retrieve_content()
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("Apply XSLT to list of objects and add a workflow stamp");
  

  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // Datastream ID 
  $ds_id = drush_get_option('ds_id', 'CWRC');

  // Destination directory 
  $dest_dir = drush_get_option('dest_dir', '/tmp/'.'z_'.uniqid(rand(), true));


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");


  // Destination path 
  $dest_dir = rtrim($dest_dir, '/\\');
  drush_print("Destination directory: $dest_dir");
  if (!file_exists($dest_dir)) {
    mkdir($dest_dir, 0744, true);
  }


  // Counter for the number of files to migrate.
  $file_count = 0; 
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("PID: $pid");

    try {

      $object = islandora_object_load($pid);
      if (!empty($object[$ds_id]->content)) {

        $ds_content = $object[$ds_id]->content;
        $fh = fopen($dest_dir.'/'.$pid.'_'.$ds_id, 'w+');
        fwrite($fh, $ds_content);
        fclose($fh);

        $file_count++;
      } else {
        drush_print("Skipping " . $pid . " as datastream doesn't exist.");
        $file_error_count++;
        next;
      }

    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid DS=\"$ds_id\" was not updated.");
      drush_print($e->getMessage());
    }
  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}

/** 
 * CWRC migration batch - retrieve content of specified item at a specific date
 *
 */
function drush_cwrc_migration_batch_export_by_date()
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // Datastream ID 
  $ds_id = drush_get_option('ds_id', 'CWRC');

  // Destination directory 
  $dest_dir = drush_get_option('dest_dir', '/tmp/'.'z_'.uniqid(rand(), true));

  // find before date 
  $date_before = drush_get_option('date', '2020-09-12');


  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");


  // Destination path 
  $dest_dir = rtrim($dest_dir, '/\\');
  drush_print("Destination directory: $dest_dir");
  if (!file_exists($dest_dir)) {
    mkdir($dest_dir, 0744, true);
  }


  // Counter for the number of files to migrate.
  $file_count = 0; 
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("PID: $pid");

    try {

      $object = islandora_object_load($pid);
      if (!empty($object[$ds_id]->content)) {

        foreach ($object[$ds_id] as $version => $datastream_version) {
          // drush_print($datastream_version->createdDate->format(DATE_RFC850));

          if ($datastream_version->createdDate < $date_before) {
            drush_print($datastream_version->createdDate);
            $ds_content = $datastream_version->content;
            $fh = fopen($dest_dir.'/'.$pid.'_'.$ds_id.'_'.$datastream_version->createdDate, 'w+');
            fwrite($fh, $ds_content);
            fclose($fh);
          }
        }

        $file_count++;
      } else {
        drush_print("Skipping " . $pid . " as datastream doesn't exist.");
        $file_error_count++;
        next;
      }

    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid DS=\"$ds_id\" was not updated.");
      drush_print($e->getMessage());
    }
  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}


/** 
 * CWRC migration batch - apply specified XSLT to the specified set of PIDs 
 *
 */
function drush_cwrc_migration_batch_apply_xslt()
{
  module_load_include('inc', 'islandora_cwrc_writer', 'includes/utilities');

  module_load_include('inc', 'islandora_workflow_rest', 'includes/cwrc_workflow');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/basic_enum');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/workflow_item');
  module_load_include('inc', 'islandora_workflow_rest', 'includes/utilities');


  drush_print("Apply XSLT to list of objects and add a workflow stamp");
  

  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // Datastream ID 
  $ds_id = drush_get_option('ds_id', 'CWRC');

  // XSLT 
  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';
  $xslt_path = drush_get_option(
    'xslt_path'
    , $ext_home . 'transforms/orlando_gender.xsl'
  );

  $wf_category = drush_get_option('wf_category' , 'machine_processed');
  $wf_stamp = drush_get_option('wf_stamp' , 'orlando:SUB');
  $wf_status = drush_get_option('wf_status' , 'c');
  $wf_note = drush_get_option('wf_note' , 'Apply XSLT: $xslt');

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // setup XSL 
  // http://www.saxonica.com/saxon-c/index.xml#debug
  
  $saxonProc = new Saxon\SaxonProcessor();
  drush_print("XSLT Version: " . $saxonProc->version());
  
  $xslt_proc = $saxonProc->newXsltProcessor();
  $xslt_proc->compileFromFile($xslt_path);

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");

  // Counter for the number of files to migrate.
  $file_count = 0; 
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("PID: $pid");

    try {

      $object = islandora_object_load($pid);
      if (!empty($object[$ds_id]->content)) {
        // process XSLT
        $ds_content = $object[$ds_id]->content;
        $xslt_proc->setSourceFromXdmValue(
          $saxonProc->parseXmlFromString($ds_content)
        );
        $ds_content = $xslt_proc->transformToString();
        $object[$ds_id]->setContentFromString($ds_content);

        // add a workflow stamp indicating that the object was ingested
        $workflow = CWRCWorkflowAPI::fromDatastream($object['WORKFLOW']);
        $workflow->addWorkflowStep(
          $object->id,
          new WorkflowItem(WorkflowConst::WORKFLOW, array()),
          new WorkflowItem(
            WorkflowConst::ACTIVITY,
            array(
              "category" => $wf_category,
              'stamp' => $wf_stamp,
              'status' => $wf_status,
              "note" => $wf_note,
              )
            )
          );
        CWRCWorkflowAPI::updateDatastream($workflow, $object);
        $file_count++;
      } else {
        drush_print("Skipping " . $pid . " as datastream doesn't exist.");
        $file_error_count++;
        next;
      }

    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid DS=\"$ds_id\" was not updated.");
      drush_print($e->getMessage());
    }
  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of files migrated: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);
}


/** 
 * CWRC migration batch - list PIDs of all items within a collection 
 *
 */
function drush_cwrc_migration_batch_retrieve_members_of_collection()
{

  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  // Collection PID
  $collection_pid = drush_get_option(
    'collection_pid', 'islandora:root'
  );

  drush_print("Members of collection: $collection_pid");

  $query = <<<EOT
    PREFIX fre: <info:fedora/fedora-system:def/relations-external#>
    PREFIX fm: <info:fedora/fedora-system:def/model#>
    SELECT ?obj
    FROM <#ri>
    WHERE {
      {
        ?obj fre:isMemberOfCollection ?sub
      }
      UNION
      {
        ?obj fre:isMemberOf ?sub
      }
      ?obj fm:state fm:Active .
      ?sub fm:state fm:Active
      FILTER(sameTerm(?sub, <info:fedora/$collection_pid>))
    }
EOT;

  drush_print($query);

  // Counter for the number of files.
  $count = 0;
  $error_count = 0;

  try
  {
    $connection = islandora_get_tuque_connection();
    $results = $connection->repository->ri->sparqlQuery($query);
    foreach ($results as $result)
    {
      $count++;
      drush_print($result['obj']['value']);
    }
  }
  catch (Exception $e) {
     // print exception
     print $e->getMessage();
     $error_count++;
  }

  drush_print('Number of objects: ' . $count);
  drush_print('Number of errors: ' . $error_count);
}

/**
 * CWRC migration batch - list PIDs of all items
 *
 */
function drush_cwrc_migration_batch_retrieve_item_summary()
{

  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  $query = <<<EOT
    PREFIX fre: <info:fedora/fedora-system:def/relations-external#>
    PREFIX fm: <info:fedora/fedora-system:def/model#>
    SELECT ?obj ?collection ?member_of ?model ?label
    FROM <#ri>
    WHERE {
      {
        ?obj fre:isMemberOfCollection ?collection
      }
      UNION
      {
        ?obj fre:isMemberOf ?member_of
      }
      ?obj fm:hasModel ?model .
      ?obj fm:label ?label .
      ?obj fm:state fm:Active
      FILTER(! sameTerm(?model, <info:fedora/fedora-system:FedoraObject-3.0>))
    }
EOT;

  drush_print($query);

  // Counter for the number of files.
  $count = 0;
  $error_count = 0;

  try
  {
    $connection = islandora_get_tuque_connection();
    $results = $connection->repository->ri->sparqlQuery($query);
    drush_print($result);
    foreach ($results as $result)
    {
      $count++;
      drush_print(
        $result['obj']['value']
        .",".$result['collection']['value']
        .",".$result['member_of']['value']
        .",".$result['model']['value']
        .","."\"".str_replace('"', '""', $result['label']['value'])."\"" 
        );
    }
  }
  catch (Exception $e) {
     // print exception
     drush_print($e->getMessage());
     $error_count++;
  }

  drush_print('Number of objects: ' . $count);
  drush_print('Number of errors: ' . $error_count);
}

/**
 * CWRC migration batch - list PIDs where the collection has been deleted
 *
 */
function drush_cwrc_migration_batch_retrieve_orphan_summary()
{

  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  $query = <<<EOT
    PREFIX fre: <info:fedora/fedora-system:def/relations-external#>
    PREFIX fm: <info:fedora/fedora-system:def/model#>
    SELECT ?obj ?collection ?label
    FROM <#ri>
    WHERE {
      ?obj fre:isMemberOfCollection ?collection .
      ?obj fm:label ?label .
      OPTIONAL { ?collection fm:hasModel ?model . }
      FILTER(!bound(?model))
    }

EOT;

  drush_print($query);

  // Counter for the number of files.
  $count = 0;
  $error_count = 0;

  try
  {
    $connection = islandora_get_tuque_connection();
    $results = $connection->repository->ri->sparqlQuery($query);
    drush_print($result);
    foreach ($results as $result)
    {
      $count++;
      drush_print(
        $result['obj']['value']
        .",".$result['collection']['value']
        .","."\"".str_replace('"', '""', $result['label']['value'])."\"" 
        );
    }
  }
  catch (Exception $e) {
     // print exception
     drush_print($e->getMessage());
     $error_count++;
  }

  drush_print('Number of objects: ' . $count);
  drush_print('Number of errors: ' . $error_count);
}




/** 
 * CWRC migration batch - delete objects in list 
 * don't use with collections
 *
 */
function drush_cwrc_migration_batch_delete() 
{

  drush_print("Delete objects");

  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");

  // Counter for the number of files.
  $file_count = 0;
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("PID: $pid");

    try {
      $object = islandora_object_load($pid);

      $collection_models = islandora_basic_collection_get_collection_content_models();
      $is_a_collection = ( 
        (count(array_intersect($collection_models, $object->models)) > 0) 
        && isset($object['COLLECTION_POLICY'])
        );

      if (!$is_a_collection) {
        //$objec->delete();
        $object->repository->purgeObject($pid);
        $file_count++;
      } else { 
        drush_print("PID is a collection - delete failed: $pid");
        $error_file_count++;
      }

    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of objects deleted: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}

/** 
 * CWRC migration batch - test objects in list 
 * don't use with collections
 *
 */
function drush_cwrc_migration_batch_test() 
{

  drush_print("Test objects");

  $ext_home = drupal_get_path('module', 'cwrc_migration_batch') . '/';

  // input file of PIDs; one per line 
  $pid_file = drush_get_option(
    'pid_file'
  );

  // Include modules.
  $connection = null;
  migration_init($connection);

  // Display the user.
  drush_print("User: " . $connection->connection->username);

  // input file of PIDs
  drush_print("PID File: $pid_file");
  $file_handle = fopen($pid_file, "r");

  // Counter for the number of files.
  $file_count = 0;
  $file_error_count = 0;
  while ($file_handle && ($line = fgets($file_handle)) !== false) 
  {
    $pid = trim($line);

    if (empty($pid) || $pid == '') {
      continue;
    }

    //PHP Fatal error:  Maximum execution time of 300 seconds exceeded
    // no effect when PHP is running in safe mode
    // http://php.net/manual/en/function.set-time-limit.php
    set_time_limit(5);

    drush_print("PID: $pid");

    try {
      $object = islandora_object_load($pid);
      $file_count++;
    } catch (Exception $e) {
      $file_error_count++;
      drush_print("pid=$pid title=\"$title\" was not added.");
      drush_print($e->getMessage());
    }

  }
  fclose($file_handle);

  drush_print('Migration Complete');
  drush_print('Number of objects tested: ' . $file_count);
  drush_print('Number of errors: ' . $file_error_count);

}