Merge branch 'julia-analysis'

chapters/aggregation.qmd

@@ -159,6 +159,12 @@ aggreCAT::DistributionWAgg(expert_judgements = temp, round_2_filter = FALSE, typ
 
 ## Inter-rater reliability
 
+Inter-rater reliability is a measure of the degree to which two or more independent raters (in our case, paper evaluators) agree. Here, the ratings are the 7 aspects of each paper that evaluators were asked to rate. For each paper, we can obtain one value that summarizes the agreement between the two or three evaluators.
+
+We use Krippendorff's alpha as a measure of interrater agreement. Krippendorff's alpha is a more flexible measure of agreement and can be used with different levels of data (categorical, ordinal, interval, and ratio) as well as different numbers of raters. The calculation of alpha in the function `kripp.alpha` was implemented by Jim Lemon in the package `irr` and is based on Krippendorff, K. (1980). Content analysis: An introduction to its methodology. Beverly Hills, CA: Sage.
+
+Krippendorff's alpha can range from -1 to +1, and it can be interpreted similarly to a correlation: values closer to +1 indicate excellent agreement between evaluators; values closer to 0 indicate there is no agreement between evaluators; and negative values indicate that there is systematic disagreement between evaluators, such that ratings are reversed -- where a given evaluator tends to rate something as high, the other(s) tend to rate it as low, and vice versa.
+
 ```{r}
 #| echo: false
 #| fig-height: 8

chapters/evaluation_data_input.qmd

@@ -412,6 +412,16 @@ all_papers_p <- all_pub_records %>%
 #| include: false
 #| 
 evals_pub_long %>% 
+  mutate(rating_type = factor(rating_type, 
+                              levels = c(rating_cats, pred_cats),
+                              labels = c("Overall assessment",
+                                         "Advances our knowledge & practice",  
+                                         "Methods: justification, reasonableness, validity, robustness", 
+                                         "Logic and communication", 
+                                         "Engages with real-world, impact quantification",
+                                         "Relevance to global priorities",
+                                         "Open, collaborative, replicable science and methods",
+                                         "Predicted Journal", "Merits Journal"))) %>% 
   write_rds(file = here("shinyapp/DataExplorer", "shiny_explorer.rds"))
  
 ```

data/all_papers_p.csv

@@ -41,6 +41,7 @@ recIlJDSYJtfwrUVE,,NA,NA,NA,,NA,NA,"Development and Growth, Growth and productiv
 recJNtBg4lyBwy2Yh,,0.45,NA,0.6,,"""Attitudes and behaviors (Altruism, moral circles, animal consumption, effectiveness, political attitudes, etc.)""",Possible category: more 'economics' treatment of preferences for giving and 'ethical/donation-linked' choices; implications of these preferences; donor coordination etc.,"""Health, Education, and Welfare"", Poverty and Wellbeing","David Reinstein, Jonathan Berman",Unpublished working paper,NA,NA,internal-NBER,NA,NA,2022-10-26T14:25:41.000Z
 recJpc6KiWth0JTQP,,NA,NA,NA,,Markets for products with large externalities (focus: animal agriculture),NA,NA,NA,NA,NA,non-academic stream possibility,suggested - externally - NGO,NA,NA,2023-07-31T16:18:12.000Z
 recKL2IpTca2x2MID,,0.5700000000000001,NA,NA,,"""Health and well-being (global, including wealthy countries)""","""Innovation, meta-science, and research""","""Health, Education, and Welfare"", Health",NA,NA,NA,NA,internal-NBER,NA,NA,2022-11-23T01:00:41.000Z
+recKgrthW975DCFti,,1,NA,1,,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,2023-09-15T16:10:50.000Z
 recLC36l549oXU64R,,0.6,NA,NA,,"""Communicable diseases, bio-security and pandemic preparedness, biological risks""",NA,Health,NA,"Published, ~top journal",NA,NA,submitted,NA,TRUE,2022-12-04T18:44:20.000Z
 recLGKU3MIfrIxXjH,,NA,NA,0.85,,"""Global health; """"Health & well-being in low-income countries""""""",NA,"""Health, Education, and Welfare""",NA,Unpublished working paper,NA,considering,suggested - externally,needed/seeking,NA,2023-07-30T11:51:10.000Z
 recLlY6U68evI1tei,,0.52,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""",Economic development & governance (LMICs),"Econometrics, Experimental Design, Development and Growth","Daniela Cialfi, Ryan Briggs ",Unpublished working paper,NA,NA,internal-NBER,NA,NA,2022-10-26T14:51:25.000Z
@@ -80,7 +81,7 @@ recb2OhsLTzsTFitP,,0.63,NA,0.63,,"""Attitudes and behaviors (Altruism, moral cir
 recbSIx4vL7A2g8L8,,0.55,NA,0.55,,Economic development & governance (LMICs),NA,NA,NA,Unpublished working paper,Authors say 'no' (but maybe revisit?),NA,suggested - internally,NA,NA,2022-06-06T18:27:46.000Z
 reccAvoay5ECDSGgO,,0.6,NA,0.6,,Emerging technologies: social and economic impacts (focus: AI),Economic development & governance (LMICs),"International Economics, Globalization and International Relations, Development and Growth, Innovation and R&D",NA,NA,NA,considering,internal-NBER,NA,NA,2022-10-26T14:48:38.000Z
 reccgH2KoqMzix1f6,,0.62,NA,0.62,,"""Innovation, meta-science, and research""",NA,"Development and Growth, Innovation and R&D",NA,"published, decent journal ",NA,considering,internal-NBER,NA,NA,2022-11-05T14:56:50.000Z
-reccpREogKO0HCyXu,,0.72,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""",Economic development & governance (LMICs),"Econometrics, Experimental Design, Microeconomics, Behavioral Economics, Development and Growth, Development","Anirudh Tagat, Hansika Kapoor",NA,Acknowledged,published,internal-NBER,NA,NA,2022-11-23T01:42:35.000Z
+reccpREogKO0HCyXu,,0.72,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""",Economic development & governance (LMICs),"Econometrics, Experimental Design, Microeconomics, Behavioral Economics, Development and Growth, Development","x, Hansika Kapoor",NA,Acknowledged,published,internal-NBER,NA,NA,2022-11-23T01:42:35.000Z
 recd26i1Bs8QsR3P3,,0.64,NA,0.64,,"""Other: Economics, growth, policy, global markets and population """,NA,NA,NA,NA,Author suggested but need to confirm with co-authors,considering,suggested - externally,NA,NA,2022-07-17T10:53:38.000Z
 recdaNJkDpIpcbBBU,,0.58,NA,0.58,,"""Attitudes and behaviors (Altruism, moral circles, animal consumption, effectiveness, political attitudes, etc.)""",NA,NA,NA,NA,NA,NA,suggested - externally - NGO,NA,NA,2022-10-26T14:23:01.000Z
 receXGQBDjKhWzD6X,,0.54,NA,NA,,"""International cooperation and conflict, behavior of large governments; authoritarianism """,Economic development & governance (LMICs),NA,NA,NA,NA,NA,internal-NBER,NA,NA,2023-06-08T23:24:10.000Z
@@ -106,7 +107,7 @@ recmJNyeaL60b0UvU,,NA,NA,0.65,,"""Global health; """"Health & well-being in low-
 recmXqAzzj4MlJbwC,,0.6,NA,NA,,Economic development & governance (LMICs),"""Global health; """"Health & well-being in low-income countries""""""",NA,NA,NA,NA,NA,internal-NBER,NA,NA,2023-06-08T23:29:58.000Z
 recmatMogeg5trJZR,,0.58,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""","""The Environment: Harm to human health, biodiversity, risks and climate change""",NA,NA,NA,NA,NA,suggested - externally,NA,NA,2023-06-09T22:12:51.000Z
 recmk7X0k0H5kCYvN,,0.57,NA,NA,,"""Other: Economics, growth, policy, global markets and population """,Economic development & governance (LMICs),Migration,NA,NA,NA,NA,internal-NBER,NA,NA,2023-06-08T12:49:19.000Z
-recokbZR7Lm4IU815,,0.58,NA,0.45,,"""Innovation, meta-science, and research""",NA,"Development and Growth, Public Economics, ""Health, Education, and Welfare"", Innovation and R&D, Education","Gavin Taylor , Alexander Herwix",NA,NA,NA,internal-NBER,NA,NA,2022-11-05T15:30:16.000Z
+recokbZR7Lm4IU815,,0.58,NA,0.45,,"""Innovation, meta-science, and research""",NA,"Development and Growth, Public Economics, ""Health, Education, and Welfare"", Innovation and R&D, Education","x, Alexander Herwix",NA,NA,NA,internal-NBER,NA,NA,2022-11-05T15:30:16.000Z
 recomNDX1PTvP5gDM,,0.58,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""",NA,"Development and Growth, Development, Environmental and Resource Economics, Environment",NA,"published, decent journal ",NA,NA,internal-NBER,NA,NA,2022-10-26T14:42:33.000Z
 recpBOskwOo7MB7Ze,,NA,NA,NA,,NA,NA,NA,NA,NA,NA,non-academic stream possibility,suggested - externally - NGO,NA,NA,2023-07-31T16:18:12.000Z
 recpgeTZaxfK6t3H0,,0.54,NA,NA,,Markets for products with large externalities (focus: animal agriculture),Empirical methods,NA,NA,NA,NA,NA,suggested - externally - NGO,NA,NA,2022-11-22T16:03:53.000Z
@@ -119,7 +120,7 @@ recsiQaf3ZTSkkXLV,,0.63,NA,NA,,"""Global health; """"Health & well-being in low-
 recsii9l3QRQFerkU,,1,0.8,1,,"""Catastrophic and existential risks,  the long-term future, forecasting""",Emerging technologies: social and economic impacts (focus: AI),NA,NA,"Published, ? journal",Agreed,published,submitted,not needed (submitted by authors),NA,2022-05-08T03:58:56.000Z
 recsxlSRIz4Y1RHd3,,NA,NA,NA,,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,2023-07-31T16:18:12.000Z
 rect8c6gbgVnvz6Zt,,NA,NA,NA,,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,2023-08-14T19:32:14.000Z
-rectfSMcCGKrVVtuw,,0.63,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""",Economic development & governance (LMICs),"Public Economics, ""Health, Education, and Welfare"", Poverty and Wellbeing, Labor Economics, Demography and Aging, Labor Supply and Demand, Development and Growth, Development","Hansika Kapoor, Anirudh Tagat",NA,Emailed,published,internal-NBER,NA,NA,2022-11-23T01:51:58.000Z
+rectfSMcCGKrVVtuw,,0.63,NA,NA,,"""Global health; """"Health & well-being in low-income countries""""""",Economic development & governance (LMICs),"Public Economics, ""Health, Education, and Welfare"", Poverty and Wellbeing, Labor Economics, Demography and Aging, Labor Supply and Demand, Development and Growth, Development","Hansika Kapoor, x",NA,Emailed,published,internal-NBER,NA,NA,2022-11-23T01:51:58.000Z
 rectim9KLJ6yQ1Goa,,0.56,NA,0.56,,"""Catastrophic and existential risks,  the long-term future, forecasting""",NA,NA,NA,"Published, ~top journal",NA,NA,internal-from-syllabus-agenda-policy-database,NA,NA,2022-04-15T15:57:47.000Z
 rectraxzNjDb0cDbU,,NA,NA,NA,,NA,NA,NA,NA,NA,NA,NA,NA,NA,NA,2023-08-22T20:29:40.000Z
 recuWvH8xC8jK5Rtk,,NA,NA,1,,"""Global health; """"Health & well-being in low-income countries""""""",Economic development & governance (LMICs),Health,Ryan Briggs ,NA,Acknowledged,de-prioritized,suggested - externally - NGO,not needed (Unjournal Direct),NA,2023-06-08T12:48:58.000Z