fix(migration): introduce conditionality and continue indexing on failure

packages/migration/src/migrations/hearth/20220822085901-rejected-declarations-started-by.ts

@@ -79,7 +79,7 @@ async function getCompositionIdToStartedByMap(
  if (startedByMap.has(compositionId)) return
  startedByMap.set(
  compositionId,
- extractId(task.extension[0].valueReference.reference)
+ extractId(task.extension[0]?.valueReference.reference)
  )
  })
  return startedByMap

packages/migration/src/migrations/hearth/20221017133354-restore-supporting-document.ts

@@ -56,7 +56,7 @@ export const up = async (db: Db, client: MongoClient) => {
  const immediatePrevComp = compositionHistory[correctionIndex - 1]
  const hasDocumentSection = immediatePrevComp?.section?.find(
  (section: fhir.CompositionSection) =>
- section?.code?.coding?.[0].code === 'supporting-documents'
+ section?.code?.coding?.[0]?.code === 'supporting-documents'
  )
  if (hasDocumentSection) {
  await db

packages/migration/src/migrations/hearth/20221031131452-add_practioner_id_event_registrations.ts

@@ -52,7 +52,7 @@ async function migrateRegistrations(measurement: string, db: Db) {
 
  await writePoints(updatedPoints)
 
- const startTime = registrations[0].time?.getNanoTime()
+ const startTime = registrations[0]?.time?.getNanoTime()
  const endTime = registrations[registrations.length - 1].time?.getNanoTime()
 
  const deleteQuery = `DELETE FROM ${measurement} WHERE registrarPractitionerId = '' AND time >= ${startTime} AND time <= ${endTime}`
@@ -110,7 +110,7 @@ const getUpdatedPoints = async (
  )
  }
 
- let id: string = ''
+ let id = ''
  if (
  practitionerExtension &&
  practitionerExtension.valueReference &&

packages/migration/src/migrations/hearth/20221110092434-add_identifiers_DOB_gender_location.ts

@@ -52,6 +52,7 @@ export const up = async (db: Db, client: MongoClient) => {
  const body = {}
  const compositionDoc =
  (await compositionCursor.next()) as unknown as fhir.Composition
+
  await setInformantDeceasedAndLocationDetails(db, body, compositionDoc)
  }
  skip += limit
@@ -157,16 +158,18 @@ async function setInformantDeceasedAndLocationDetails(
  )
 
  if (encounterDoc.length > 0) {
- const locationId = encounterDoc[0].location[0].location.reference.replace(
- 'Location/',
- ''
- )
+ const locationId =
+ encounterDoc[0]?.location?.[0]?.location?.reference?.replace(
+ 'Location/',
+ ''
+ )
+
+ const locationDoc = locationId
+ ? await getCollectionDocuments(db, COLLECTION_NAMES.LOCATION, [
+ locationId
+ ])
+ : []
 
- const locationDoc = await getCollectionDocuments(
- db,
- COLLECTION_NAMES.LOCATION,
- [locationId]
- )
  if (locationDoc.length > 0) {
  const address: fhir.Address = locationDoc[0].address
  if (address) {

packages/migration/src/migrations/hearth/20221128065238-migrate-documents-to-minio.ts

@@ -38,7 +38,7 @@ export const up = async (db: Db, client: MongoClient) => {
  )
  while (await documentCursor.hasNext()) {
  const document = await documentCursor.next()
- const fileData = document?.content[0].attachment.data
+ const fileData = document?.content?.[0]?.attachment?.data
  if (fileData && isBase64FileString(fileData)) {
  const refUrl = await uploadBase64ToMinio(fileData)
  if (refUrl) {

packages/migration/src/migrations/hearth/20230127063226-update-practitioner-role.ts

@@ -105,8 +105,8 @@ export const up = async (db: Db, client: MongoClient) => {
 
  if (hasSystemTypes) {
  const typeCode = practitionerRole.code?.find(
- (c) => c.coding?.[0].system === systemTypes
- )?.coding?.[0].code
+ (c) => c.coding?.[0]?.system === systemTypes
+ )?.coding?.[0]?.code
  await db
  .collection('PractitionerRole')
  .updateOne(

packages/migration/src/migrations/hearth/20230221105416-create-performance-views.ts

@@ -766,7 +766,9 @@ export const up = async (db: Db, client: MongoClient) => {
  $project: {
  _id: { $concat: [{ $toString: '$id' }, '_', '$daysInYear.date'] },
  name: 1,
- date: { $dateFromString: { dateString: '$daysInYear.date' } },
+ date: {
+ $dateFromString: { dateString: '$daysInYear.date' }
+ },
  estimatedNumberOfBirths: '$daysInYear.estimatedNumberOfBirths',
  event: 'Birth'
  }

packages/migration/src/migrations/hearth/20230328065851-change-certified-status-to-issused.ts

@@ -67,7 +67,7 @@ export const up = async (db: Db, client: MongoClient) => {
  const operationHistoriesData =
  searchResult &&
  searchResult.body.hits.hits.length > 0 &&
- searchResult.body.hits.hits[0]._source?.operationHistories
+ searchResult.body.hits.hits?.[0]?._source?.operationHistories
  const lastOperationHistory =
  operationHistoriesData &&
  operationHistoriesData.length > 0 &&

packages/migration/src/migrations/hearth/20230602120944-populate-valid-locationid.ts

@@ -138,10 +138,11 @@ async function updateEventLocationIdOrJurisdictionIds(db: Db, elasticDoc: any) {
  )
 
  if (encounterDoc.length > 0) {
- const locationId = encounterDoc[0].location[0].location.reference.replace(
- 'Location/',
- ''
- )
+ const locationId =
+ encounterDoc[0]?.location?.[0]?.location?.reference?.replace(
+ 'Location/',
+ ''
+ )
 
  const locationDoc = await getCollectionDocuments(
  db,
@@ -150,10 +151,10 @@ async function updateEventLocationIdOrJurisdictionIds(db: Db, elasticDoc: any) {
  )
 
  if (locationDoc.length > 0) {
- if (locationDoc[0].type.coding[0].code === 'HEALTH_FACILITY') {
- body.eventLocationId = locationDoc[0].id
+ if (locationDoc?.[0]?.type?.coding?.[0]?.code === 'HEALTH_FACILITY') {
+ body.eventLocationId = locationDoc[0]?.id
  } else {
- const address = locationDoc[0].address
+ const address = locationDoc[0]?.address
  if (address) {
  const eventJurisdictionIds: string[] = []
  address.state && eventJurisdictionIds.push(address.state)

packages/migration/src/migrations/hearth/20231123125224-deceased-birth-date.ts

@@ -50,7 +50,7 @@ export const up = async (db: Db, client: MongoClient) => {
  })
  let processed = 0
  for await (const patient of patientIterator) {
- const age = parseInt(patient.extension[0].valueString, 10)
+ const age = parseInt(patient.extension?.[0]?.valueString, 10)
  const birthDate = subYears(new Date(patient.deceasedDateTime), age)
  db.collection('Patient').updateOne(
  { id: patient.id },

packages/search/src/features/reindex/reindex.ts

@@ -62,6 +62,8 @@ export const reindex = async () => {
  }
  })
 
+ const droppedCompositionIds: string[] = []
+
  await client.helpers.bulk(
  {
  retries: 3,
@@ -74,22 +76,26 @@ export const reindex = async () => {
  }
  }),
  onDrop(doc) {
- throw new Error(
- `Document ${doc.document.compositionId} couldn't be inserted`
- )
+ droppedCompositionIds.push(doc.document.compositionId)
  }
  },
  {
  meta: true
  }
  )
+
  const t2 = performance.now()
  logger.info(
  `Finished reindexing to ${index} in ${((t2 - t1) / 1000).toFixed(
  2
  )} seconds`
  )
 
+ if (droppedCompositionIds.length) {
+ logger.warn(`Could not index ${droppedCompositionIds.length} document(s)`)
+ logger.warn(droppedCompositionIds.join(', '))
+ }
+
  return { index }
 }