Merge pull request #112 from OHDSI/observation-period-added-definition

Observation period added definition
OHDSI · Aug 29, 2024 · 14a22ba · 14a22ba
2 parents 7306191 + 4856d35
commit 14a22ba
Show file tree

Hide file tree

Showing 13 changed files with 101 additions and 31 deletions.
diff --git a/sources/Framework/org.ohdsi.cdm.framework.etl/Transformation/CDM/CdmPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework.etl/Transformation/CDM/CdmPersonBuilder.cs
@@ -94,11 +94,6 @@ public override Attrition Build(ChunkData data, KeyMasterOffsetManager o)
                 return result.Value;
             }
 
-            //if(person.PersonId == 136899754)
-            //{
-
-            //}
-
             if(ObservationPeriodsRaw.Count == 0)
                 return Attrition.InvalidObservationTime;
 

diff --git a/sources/Framework/org.ohdsi.cdm.framework.etl/Transformation/CDM/Definitions/L_PROVIDER.xml b/sources/Framework/org.ohdsi.cdm.framework.etl/Transformation/CDM/Definitions/L_PROVIDER.xml
@@ -30,7 +30,7 @@
 			<Concepts>
 				<Concept>
 					<Fields>
-						<Field conceptId="specialty_concept_id" key="specialty_source_value" sourceKey="specialty_source_value" defaultTypeId="0"/>
+						<Field conceptId="specialty_concept_id" key="specialty_source_value" sourceKey="specialty_source_value" sourceConceptId="specialty_source_concept_id" defaultTypeId="0"/>
 					</Fields>
 				</Concept>
 			</Concepts>

diff --git a/...amework/org.ohdsi.cdm.framework.etl/Transformation/CDM/Definitions/Observation_period.xml b/...amework/org.ohdsi.cdm.framework.etl/Transformation/CDM/Definitions/Observation_period.xml
@@ -0,0 +1,22 @@
+<QueryDefinition>
+	<Query>
+		SELECT distinct
+		op.observation_period_id,
+		op.person_id,
+		op.observation_period_start_date,
+		op.observation_period_end_date,
+		op.period_type_concept_id
+		FROM {sc}.observation_period op
+		JOIN {ch_sc}._chunks ch ON ch.ChunkId = {0} AND op.PERSON_ID = ch.PERSON_ID
+		order by 1
+	</Query>
+	<ObservationPeriods>
+		<ObservationPeriodDefinition>
+			<Id>observation_period_id</Id>
+			<PersonId>person_id</PersonId>
+			<StartDate>observation_period_start_date</StartDate>
+			<EndDate>observation_period_end_date</EndDate>
+			<TypeConceptId>period_type_concept_id</TypeConceptId>
+		</ObservationPeriodDefinition>
+	</ObservationPeriods>
+</QueryDefinition>
diff --git a/sources/Framework/org.ohdsi.cdm.framework.etl/Transformation/CDM/Definitions/Person.xml b/sources/Framework/org.ohdsi.cdm.framework.etl/Transformation/CDM/Definitions/Person.xml
@@ -19,15 +19,9 @@
 		t.race_source_concept_id,
 		t.ethnicity_source_value,
 		t.ethnicity_source_concept_id,
-
-		op.observation_period_start_date,
-		op.observation_period_end_date,
-		op.period_type_concept_id,
-
 		1 as fake
 		FROM {sc}.person t
 		JOIN {ch_sc}._chunks ch ON ch.ChunkId = {0} AND t.PERSON_ID = ch.PERSON_ID
-		JOIN {sc}.observation_period op ON t.PERSON_ID = op.PERSON_ID
 		order by 1
 	</Query>
 	<Persons>
@@ -48,10 +42,6 @@
 			<RaceSourceConceptId>race_source_concept_id</RaceSourceConceptId>
 			<Ethnicity>ethnicity_source_value</Ethnicity>
 			<EthnicitySourceConceptId>ethnicity_source_concept_id</EthnicitySourceConceptId>
-
-			<StartDate>observation_period_start_date</StartDate>
-			<EndDate>observation_period_end_date</EndDate>
-			<PeriodTypeConceptId>period_type_concept_id</PeriodTypeConceptId>
 		</PersonDefinition>
 	</Persons>
 

diff --git a/...es/Framework/org.ohdsi.cdm.framework.etl/Transformation/Truven/Definitions/L_PROVIDER.xml b/...es/Framework/org.ohdsi.cdm.framework.etl/Transformation/Truven/Definitions/L_PROVIDER.xml
@@ -31,7 +31,7 @@
                   </Mapper>
                </ConceptIdMappers>
                <Fields>
-                  <Field key="SPECIALTY_SOURCE_VALUE"/>
+                  <Field key="SPECIALTY_SOURCE_VALUE" defaultConceptId="38004514"/>
                </Fields>
             </Concept>
          </Concepts>

diff --git a/sources/Framework/org.ohdsi.cdm.framework.etl/org.ohdsi.cdm.framework.etl.csproj b/sources/Framework/org.ohdsi.cdm.framework.etl/org.ohdsi.cdm.framework.etl.csproj
@@ -18,6 +18,7 @@
     <None Remove="Transformation\CDM\Definitions\L_PROVIDER.xml" />
     <None Remove="Transformation\CDM\Definitions\Measurement.xml" />
     <None Remove="Transformation\CDM\Definitions\Observation.xml" />
+    <None Remove="Transformation\CDM\Definitions\Observation_period.xml" />
     <None Remove="Transformation\CDM\Definitions\Payer_plan_period.xml" />
     <None Remove="Transformation\CDM\Definitions\Person.xml" />
     <None Remove="Transformation\CDM\Definitions\Procedure_occurrence.xml" />
@@ -329,6 +330,9 @@
     <EmbeddedResource Include="Transformation\CDM\Definitions\Payer_plan_period.xml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </EmbeddedResource>
+    <EmbeddedResource Include="Transformation\CDM\Definitions\Observation_period.xml">
+      <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+    </EmbeddedResource>
     <EmbeddedResource Include="Transformation\CDM\Definitions\Person.xml">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </EmbeddedResource>

diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Base/IPersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Base/IPersonBuilder.cs
@@ -28,6 +28,8 @@ public interface IPersonBuilder
 
         void AddEpisode(Episode data);
 
+        void AddObservationPeriod(EraEntity data);
+
         //void Build(ChunkData chunkData);
         Attrition Build(ChunkData data, KeyMasterOffsetManager offset);
 

diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Base/PersonBuilder.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Base/PersonBuilder.cs
@@ -284,6 +284,11 @@ public void AddEpisode(Episode data)
             EpisodeRecords.Add(data);
         }
 
+        public void AddObservationPeriod(EraEntity data)
+        {
+            ObservationPeriodsRaw.Add(data);
+        }
+
         /// <summary>
         /// Add raw entities to builder for further build
         /// </summary>
@@ -295,15 +300,19 @@ public void AddData(IEntity data)
                 case EntityType.Person:
                     {
                         AddEntity((Person)data, PersonRecords);
-                        AddEntity(
-                            new EraEntity
-                            {
-                                PersonId = data.PersonId,
-                                StartDate = data.StartDate,
-                                EndDate = data.EndDate,
-                                TypeConceptId = data.TypeConceptId,
-                                AdditionalFields = data.AdditionalFields
-                            }, ObservationPeriodsRaw);
+
+                        if (data.TypeConceptId.HasValue)
+                        {
+                            AddEntity(
+                                new EraEntity
+                                {
+                                    PersonId = data.PersonId,
+                                    StartDate = data.StartDate,
+                                    EndDate = data.EndDate,
+                                    TypeConceptId = data.TypeConceptId,
+                                    AdditionalFields = data.AdditionalFields
+                                }, ObservationPeriodsRaw);
+                        }
                         break;
                     }
 

diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Definitions/ObservationPeriodDefinition.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Definitions/ObservationPeriodDefinition.cs
@@ -0,0 +1,30 @@
+using org.ohdsi.cdm.framework.common.Builder;
+using org.ohdsi.cdm.framework.common.Extensions;
+using org.ohdsi.cdm.framework.common.Omop;
+using System.Data;
+
+namespace org.ohdsi.cdm.framework.common.Definitions
+{
+    public class ObservationPeriodDefinition : EntityDefinition
+    {
+        public string TypeConceptId { get; set; }
+
+        public IEnumerable<EraEntity> GetObservationPeriods(Concept concept, IDataRecord reader,
+            KeyMasterOffsetManager offset)
+        {
+            yield return new EraEntity
+            {
+                Id = reader.GetLong(Id) ?? 0,
+                PersonId = reader.GetLong(PersonId) ?? 0,
+                TypeConceptId = reader.GetLong(TypeConceptId) ?? 0,
+                StartDate = reader.GetDateTime(StartDate),
+                EndDate = reader.GetDateTime(EndDate),
+            };
+        }
+
+        public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataRecord reader, KeyMasterOffsetManager offset)
+        {
+            throw new NotImplementedException("ObservationPeriodDefinition.GetConcepts()");
+        }
+    }
+}
diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Definitions/ProviderDefinition.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Definitions/ProviderDefinition.cs
@@ -82,9 +82,7 @@ public override IEnumerable<IEntity> GetConcepts(Concept concept, IDataRecord re
                 long? specialtyConcept = null;
                 long? specialtySourceConceptId = null;
 
-                //(Unknown Physician Specialty)
-                var defaultConceptId = 38004514;
-
+                var defaultConceptId = 0;
                 if (conceptField.DefaultConceptId.HasValue)
                 {
                     defaultConceptId = conceptField.DefaultConceptId.Value;

diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Definitions/QueryDefinition.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Definitions/QueryDefinition.cs
@@ -90,6 +90,7 @@ public class QueryDefinition
         public EpisodeDefinition[] Episodes { get; set; }
 
         public EpisodeEventDefinition[] EpisodeEvents { get; set; }
+        public ObservationPeriodDefinition[] ObservationPeriods { get; set; }
 
         private static readonly string[] separator = [","];
 
@@ -255,6 +256,10 @@ public string GetPersonIdFieldName()
                 return EpisodeEvents[0].PersonId;
             }
 
+            if (ObservationPeriods != null && ObservationPeriods.Length != 0)
+            {
+                return ObservationPeriods[0].PersonId;
+            }
 
             throw new Exception("Cant find PersonId FieldName " + this.FileName);
         }

diff --git a/sources/Framework/org.ohdsi.cdm.framework/Common/Extensions/DataRecordExtensions.cs b/sources/Framework/org.ohdsi.cdm.framework/Common/Extensions/DataRecordExtensions.cs
@@ -16,9 +16,14 @@ public static string GetString(this IDataRecord reader, string fieldName)
                 if (string.IsNullOrEmpty(fieldName))
                     return null;
 
-                var value = GetValue(reader, fieldName).ToString();
+                var value = GetValue(reader, fieldName);
+
+                if (value is null || value is DBNull)
+                    return null;
+
+                var strValue = value.ToString();
 
-                return string.Intern(value.Trim());
+                return string.Intern(strValue.Trim());
             }
             catch (Exception e)
             {

diff --git a/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Base/LambdaChunkPart.cs b/sources/Presentation/org.ohdsi.cdm.presentation.lambdabuilder/Base/LambdaChunkPart.cs
@@ -321,6 +321,7 @@ private void Watchdog_Elapsed(object sender, System.Timers.ElapsedEventArgs e)
 
                             AddEntity(qd, qd.Note, reader, recordGuid, personIdsToSave);
                             AddEntity(qd, qd.Episodes, reader, recordGuid, personIdsToSave);
+                            AddEntity(qd, qd.ObservationPeriods, reader, recordGuid, personIdsToSave);
 
                             if (reader.Paused)
                                 break;
@@ -640,6 +641,15 @@ private void AddEntity(QueryDefinition queryDefinition, IEnumerable<EntityDefini
                             _personBuilders[episode.PersonId].Value.AddEpisode(episode);
                         }
                     }
+                    else if (d is ObservationPeriodDefinition)
+                    {
+                        foreach (var op in ((ObservationPeriodDefinition)d).GetObservationPeriods(conceptDef, reader, _offsetManager))
+                        {
+                            if (op == null) continue;
+
+                            _personBuilders[op.PersonId].Value.AddObservationPeriod(op);
+                        }
+                    }
                     else
                     {
                         foreach (var entity in d.GetConcepts(conceptDef, reader, _offsetManager))