Open FDA Pipeline initial commit

.gitignore

@@ -0,0 +1,37 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+
+/openfda-consumer/openfda-consumer.log
+/openfda-producer/openfda-producer.log
+/mongodb/mongodb-data-2022-01-20.zip
+/mongodb/mongodb-data-2022-01-22.zip

README.md

@@ -0,0 +1,3 @@
+# Open FDA Bigdata Pipeline
+
+More info coming soon...

mongodb/queries.js

@@ -0,0 +1,133 @@
+// TOP reported patient reactions
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {patientReactions: 1}},
+                     { $unwind : "$patientReactions" },
+                     { $group: { _id: "$patientReactions", total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+
+// TOP reported medical products
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {medicinalProduct: 1}},
+                     { $unwind: "$medicinalProduct" },
+                     { $group: { _id: "$medicinalProduct", total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+
+// TOP medical products that are causing death 
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $unwind: "$medicinalProduct"},
+                     { $unwind: "$patientReactions"},
+                     { $project: {
+                         product: "$medicinalProduct", 
+                         reaction: {$toLower: "$patientReactions"} 
+                        }
+                     },
+                     { $match: {reaction : "death"}},
+                     { $group: { _id: "$product", total: { $sum: 1 }}},
+                     { $project: {
+                         product: {"$substr": [{"$toUpper": "$_id"}, 0, 20]}, 
+                         total: "$total"
+                        }
+                     },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+// TOP reported drug brand names
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {drugBrandNames: 1}},
+                     { $unwind : "$drugBrandNames" },
+                     { $group: { _id: "$drugBrandNames", total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+// TOP reported drug substance names
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {drugSubstanceNames: 1}},
+                     { $unwind : "$drugSubstanceNames" },
+                     { $group: { _id: "$drugSubstanceNames", total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+// TOP reported drug manufacturer names
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {drugManufacturerNames: 1}},
+                     { $unwind : "$drugManufacturerNames" },
+                     { $group: { _id: "$drugManufacturerNames", total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+// TOP reported cuntries
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {country: 1}},
+                     { $group: { _id: "$country", total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 20 }
+                   ]);
+
+// patient sex
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $match: {patientSex : {$gte : 0}}},
+                     { $group: { _id: "$patientSex", total: { $sum: 1 } } },
+                     {"$project": {
+                        sex: {
+                            $switch: {
+                                branches: [
+                                    { case: { $eq: [ "$_id", 1 ] }, then: "famale" },
+                                    { case: { $eq: [ "$_id", 2 ] }, then: "male" }
+                                ], default: "unknown"
+                            }
+                        },
+                        total: 1,
+                        _id: 0
+                     }},
+                    { $sort: { total: -1 }}
+                    ]);
+
+
+
+// TOP 10 years when reported most 
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: { date: { $dateFromString: { dateString: '$receiveDate', format: "%Y%m%d" }}}},
+                     { $group: { _id: { $year: "$date" }, total: { $sum: 1 }} },
+                     { $sort: { total: -1 } },
+                     { $limit : 10 }
+                     ]);
+
+// TOP 10 months when reported most 
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: { date: { $dateFromString: { dateString: '$receiveDate', format: "%Y%m%d" }}}},
+                     { $group: { _id: { $month: "$date" }, total: { $sum: 1 }} },
+                     { $sort: { total: -1 } },
+                     { $limit : 10 }
+                     ]);
+
+// product group by patitent sex
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: { patientSex: 1, medicinalProduct: 1}},
+                     { $match: { patientSex : {$gte : 1}}},
+                     { $unwind : "$medicinalProduct" },
+                     { $group: { _id: { medicinalProduct: "$medicinalProduct", patientSex : "$patientSex"}, total: { $sum: 1 } } },
+                     { $sort: { total: -1 } },
+                     { $limit : 10 }
+                   ]);
+
+// patient reaction group by year
+db.getCollection('drugAdverseEvent').aggregate([
+                     { $project: {patientReactions: 1, date: { $dateFromString: { dateString: '$receiveDate', format: "%Y%m%d" }} }},
+                     { $unwind: "$patientReactions" },
+                     { $group: { _id: {patientReactions: "$patientReactions", year :{ $year:"$date" }}, total: { $sum: 1 }}},
+                     { $sort: { total: -1 } }
+                   ]);
+
+
+
+

openfda-consumer/.gitignore

@@ -0,0 +1,33 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+
+### STS ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### IntelliJ IDEA ###
+.idea
+*.iws
+*.iml
+*.ipr
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+!/openfda-consumer.log

openfda-consumer/.mvn/wrapper/maven-wrapper.properties

@@ -0,0 +1,2 @@
+distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.8.3/apache-maven-3.8.3-bin.zip
+wrapperUrl=https://repo.maven.apache.org/maven2/org/apache/maven/wrapper/maven-wrapper/3.1.0/maven-wrapper-3.1.0.jar

openfda-consumer/HELP.md

@@ -0,0 +1,20 @@
+# Getting Started
+
+### Reference Documentation
+For further reference, please consider the following sections:
+
+* [Official Apache Maven documentation](https://maven.apache.org/guides/index.html)
+* [Spring Boot Maven Plugin Reference Guide](https://docs.spring.io/spring-boot/docs/2.6.2/maven-plugin/reference/html/)
+* [Create an OCI image](https://docs.spring.io/spring-boot/docs/2.6.2/maven-plugin/reference/html/#build-image)
+* [Spring Boot Actuator](https://docs.spring.io/spring-boot/docs/2.6.2/reference/htmlsingle/#production-ready)
+* [Spring for Apache Kafka](https://docs.spring.io/spring-boot/docs/2.6.2/reference/htmlsingle/#boot-features-kafka)
+* [Spring Web](https://docs.spring.io/spring-boot/docs/2.6.2/reference/htmlsingle/#boot-features-developing-web-applications)
+
+### Guides
+The following guides illustrate how to use some features concretely:
+
+* [Building a RESTful Web Service with Spring Boot Actuator](https://spring.io/guides/gs/actuator-service/)
+* [Building a RESTful Web Service](https://spring.io/guides/gs/rest-service/)
+* [Serving Web Content with Spring MVC](https://spring.io/guides/gs/serving-web-content/)
+* [Building REST services with Spring](https://spring.io/guides/tutorials/bookmarks/)
+

openfda-consumer/README.md

@@ -0,0 +1,40 @@
+# Open FDA consumer
+
+Application built with SpringBoot for the backend solution.  
+Solution is dockerized.
+
+## Build and run the project
+
+* Building application with tests from command line
+```
+./mvnw clean install
+```
+
+* Running application
+
+```
+./mvnw spring-boot:run
+```
+
+## Docker image
+
+* Building docker image
+```
+./mvnw spring-boot:build-image
+```
+
+You can pull the latest docker image from docker.io registry
+```
+docker pull koziolk/openfda-consumer
+```
+
+Available env variables (see default values in application.yaml)
+
+```
+- Kafka configuration ralted
+KAFKA_BOOTSTRAP_SERVERS
+KAFKA_WAIT_FOR_SERVERS
+KAFKA_TOPIC_NAME
+KAFKA_NUMBER_OF_CONSUMERS
+KAFKA_CONSUMER_GROUP_ID
+```