Merge branch 'main' into sbt-dependency-graph-a9888bfbbe406a3b

.github/workflows/build.yml

@@ -50,6 +50,12 @@ jobs:
           -DAPP_SECRET="fake_secret" \
           -Duser.timezone=Australia/Sydney \
           -jar ./bin/sbt-launch.jar clean compile assets scalafmtCheckAll test Universal/packageBin
+      
+      - name: Test Summary
+        uses: test-summary/action@v2
+        with:
+          paths: "test-results/**/TEST-*.xml"
+        if: always()
 
       - uses: guardian/actions-riff-raff@v4
         env:

.gitignore

@@ -94,3 +94,4 @@ static/src/stylesheets/pasteup/.npmrc
 metals.sbt
 
 .java-version
+test-results/

admin/conf/logback.xml

@@ -30,7 +30,13 @@
 
     <root level="INFO">
         <appender-ref ref="LOGFILE"/>
-        <appender-ref ref="ASYNCSTDOUT"/>
     </root>
+    <if condition='!property("STAGE").contains("DEV")'>
+        <then>
+            <root level="INFO">
+                <appender-ref ref="ASYNCSTDOUT"/>
+            </root>
+        </then>
+    </if>
 
 </configuration>

applications/app/controllers/CrosswordsController.scala

@@ -1,10 +1,16 @@
 package controllers
 
-import com.gu.contentapi.client.model.v1.CrosswordType.{Cryptic, Quick}
-import com.gu.contentapi.client.model.v1.{Crossword, ItemResponse, Content => ApiContent, Section => ApiSection}
+import com.gu.contentapi.client.model.v1.{
+  Crossword,
+  ItemResponse,
+  SearchResponse,
+  Content => ApiContent,
+  Section => ApiSection,
+}
 import common.{Edition, GuLogging, ImplicitControllerExecutionContext}
 import conf.Static
 import contentapi.ContentApiClient
+import com.gu.contentapi.client.model.SearchQuery
 import crosswords.{
   AccessibleCrosswordPage,
   AccessibleCrosswordRows,
@@ -18,11 +24,13 @@ import html.HtmlPageHelpers.ContentCSSFile
 import model.Cached.{RevalidatableResult, WithoutRevalidationResult}
 import model._
 import model.dotcomrendering.pageElements.EditionsCrosswordRenderingDataModel
+import model.dotcomrendering.pageElements.EditionsCrosswordRenderingDataModel.toJson
 import model.dotcomrendering.{DotcomRenderingDataModel, PageType}
 import org.joda.time.{DateTime, LocalDate}
 import pages.{CrosswordHtmlPage, IndexHtmlPage, PrintableCrosswordHtmlPage}
 import play.api.data.Forms._
 import play.api.data._
+import play.api.libs.json.JsValue
 import play.api.libs.ws.WSClient
 import play.api.mvc._
 import renderers.DotcomRenderingService
@@ -117,7 +125,7 @@ class CrosswordPageController(
   }
   private def getDCRJson(crosswordPage: CrosswordPageWithContent, pageType: PageType)(implicit
       request: RequestHeader,
-  ): String =
+  ): JsValue =
     DotcomRenderingDataModel.toJson(
       DotcomRenderingDataModel.forCrossword(crosswordPage, request, pageType),
     )
@@ -304,21 +312,44 @@ class CrosswordEditionsController(
   def digitalEdition: Action[AnyContent] = Action.async { implicit request =>
     getCrosswords
       .map(parseCrosswords)
-      .flatMap {
-        case Some(crosswordPage) =>
-          remoteRenderer.getEditionsCrossword(wsClient, crosswordPage)
-        case None => Future.successful(NotFound)
+      .flatMap { crosswords =>
+        remoteRenderer.getEditionsCrossword(wsClient, crosswords)
       }
   }
 
-  private lazy val crosswordsQuery = contentApiClient.item("crosswords")
+  def digitalEditionJson: Action[AnyContent] = Action.async { implicit request =>
+    getCrosswords
+      .map(parseCrosswords)
+      .map { crosswords =>
+        Cached(CacheTime.Default)(RevalidatableResult.Ok(toJson(crosswords))).as("application/json")
+      }
+  }
 
-  private def getCrosswords: Future[ItemResponse] = contentApiClient.getResponse(crosswordsQuery)
+  private def getCrosswords: Future[SearchResponse] =
+    contentApiClient.getResponse(crosswordsQuery)
+
+  /** Search for playable crosswords sorted by print publication date. This will exclude older, originally print-only
+    * crosswords that happen to have been re-published in a digital format recently.
+    */
+  private lazy val crosswordsQuery =
+    SearchQuery()
+      .contentType("crossword")
+      .tag(crosswordTags)
+      .useDate("newspaper-edition")
+      .pageSize(25)
+
+  private lazy val crosswordTags = Seq(
+    "crosswords/series/quick",
+    "crosswords/series/cryptic",
+    "crosswords/series/prize",
+    "crosswords/series/weekend-crossword",
+    "crosswords/series/quick-cryptic",
+    "crosswords/series/everyman",
+    "crosswords/series/speedy",
+    "crosswords/series/quiptic",
+  ).mkString("|")
+
+  private def parseCrosswords(response: SearchResponse): EditionsCrosswordRenderingDataModel =
+    EditionsCrosswordRenderingDataModel(response.results.flatMap(_.crossword))
 
-  private def parseCrosswords(response: ItemResponse): Option[EditionsCrosswordRenderingDataModel] =
-    for {
-      results <- response.results
-      quick <- results.find(_.crossword.exists(_.`type` == Quick)).flatMap(_.crossword)
-      cryptic <- results.find(_.crossword.exists(_.`type` == Cryptic)).flatMap(_.crossword)
-    } yield EditionsCrosswordRenderingDataModel(quick, cryptic)
 }

applications/app/controllers/GalleryController.scala

@@ -7,6 +7,7 @@ import contentapi.ContentApiClient
 import model._
 import model.dotcomrendering.{DotcomRenderingDataModel, PageType}
 import pages.GalleryHtmlPage
+import play.api.libs.json.JsValue
 import play.api.libs.ws.WSClient
 import play.api.mvc._
 import play.twirl.api.Html
@@ -74,7 +75,7 @@ class GalleryController(
 
   private def getDCRJson(galleryPage: GalleryPage, pageType: PageType, blocks: Blocks)(implicit
       request: RequestHeader,
-  ): String = {
+  ): JsValue = {
     DotcomRenderingDataModel.toJson(DotcomRenderingDataModel.forGallery(galleryPage, request, pageType, blocks))
   }
 

applications/app/controllers/ImageContentController.scala

@@ -61,7 +61,7 @@ class ImageContentController(
 
   private def getDCRJson(content: ImageContentPage, pageType: PageType, mainBlock: Option[Block])(implicit
       request: RequestHeader,
-  ): String = {
+  ): JsValue = {
     DotcomRenderingDataModel.toJson(DotcomRenderingDataModel.forImageContent(content, request, pageType, mainBlock))
   }
 

applications/app/controllers/MediaController.scala

@@ -9,7 +9,7 @@ import implicits.{AppsFormat, JsonFormat}
 import model._
 import model.dotcomrendering.{DotcomRenderingDataModel, PageType}
 import pages.ContentHtmlPage
-import play.api.libs.json.{Format, JsObject, Json}
+import play.api.libs.json.{Format, JsObject, Json, JsValue}
 import play.api.libs.ws.WSClient
 import play.api.mvc._
 import renderers.DotcomRenderingService
@@ -96,7 +96,7 @@ class MediaController(
 
   private def getDCRJson(content: MediaPage, pageType: PageType, blocks: Blocks)(implicit
       request: RequestHeader,
-  ): String = {
+  ): JsValue = {
     DotcomRenderingDataModel.toJson(DotcomRenderingDataModel.forMedia(content, request, pageType, blocks))
   }
 

applications/app/views/fragments/gallerySlot.scala.html

@@ -12,6 +12,7 @@
         Seq("gallery-inline", "dark") ++ (if(isMobile) Some("mobile") else None),
         Map(),
         optId = if(isMobile) Some(s"$slotName--mobile") else None,
-        optClassNames = if(isMobile) Some("mobile-only") else Some("hide-until-tablet")
+        optClassNames = if(isMobile) Some("mobile-only") else Some("hide-until-tablet"),
+        useFlexContainer = true
     ){ }
 }

applications/conf/logback.xml

@@ -25,7 +25,13 @@
 
     <root level="INFO">
         <appender-ref ref="LOGFILE"/>
-        <appender-ref ref="ASYNCSTDOUT"/>
     </root>
+    <if condition='!property("STAGE").contains("DEV")'>
+        <then>
+            <root level="INFO">
+                <appender-ref ref="ASYNCSTDOUT"/>
+            </root>
+        </then>
+    </if>
 
 </configuration>

applications/conf/routes

@@ -29,6 +29,7 @@ GET        /crosswords/lookup
 
 # Crosswords digital edition
 GET        /crosswords/digital-edition                                                                                        controllers.CrosswordEditionsController.digitalEdition
+GET        /crosswords/digital-edition.json                                                                                   controllers.CrosswordEditionsController.digitalEditionJson
 
 # Email paths
 GET        /email/form/$emailType<plain|plaindark|plaintone>/$listId<[0-9]+>         controllers.EmailSignupController.renderForm(emailType: String, listId: Int)

archive/conf/logback.xml

@@ -25,7 +25,13 @@
 
     <root level="INFO">
         <appender-ref ref="LOGFILE"/>
-        <appender-ref ref="ASYNCSTDOUT"/>
     </root>
+    <if condition='!property("STAGE").contains("DEV")'>
+        <then>
+            <root level="INFO">
+                <appender-ref ref="ASYNCSTDOUT"/>
+            </root>
+        </then>
+    </if>
 
 </configuration>

article/app/controllers/ArticleController.scala

@@ -8,7 +8,7 @@ import model.Cached.{RevalidatableResult, WithoutRevalidationResult}
 import model.dotcomrendering.{DotcomRenderingDataModel, PageType}
 import model._
 import pages.{ArticleEmailHtmlPage, ArticleHtmlPage}
-import play.api.libs.json.Json
+import play.api.libs.json.{Json, JsValue}
 import play.api.libs.ws.WSClient
 import play.api.mvc._
 import renderers.DotcomRenderingService
@@ -91,7 +91,7 @@ class ArticleController(
 
   /** Returns a JSON representation of the payload that's sent to DCR when rendering the Article.
     */
-  private def getDCRJson(article: ArticlePage, blocks: Blocks)(implicit request: RequestHeader): String = {
+  private def getDCRJson(article: ArticlePage, blocks: Blocks)(implicit request: RequestHeader): JsValue = {
     val pageType: PageType = PageType(article, request, context)
     val newsletter = newsletterService.getNewsletterForArticle(article)
 

article/conf/logback.xml

@@ -25,7 +25,13 @@
 
     <root level="INFO">
         <appender-ref ref="LOGFILE"/>
-        <appender-ref ref="ASYNCSTDOUT"/>
     </root>
+    <if condition='!property("STAGE").contains("DEV")'>
+        <then>
+            <root level="INFO">
+                <appender-ref ref="ASYNCSTDOUT"/>
+            </root>
+        </then>
+    </if>
 
 </configuration>

build.sbt

@@ -70,6 +70,7 @@ val common = library("common")
       pekkoSlf4j,
       pekkoSerializationJackson,
       pekkoActorTyped,
+      janino,
     ) ++ jackson,
     TestAssets / mappings ~= filterAssets,
   )

commercial/app/controllers/CommercialControllers.scala

@@ -25,7 +25,6 @@ trait CommercialControllers {
   lazy val piggybackPixelController = wire[PiggybackPixelController]
   lazy val cmpDataController = wire[CmpDataController]
   lazy val adsDotTextFileController = wire[AdsDotTextViewController]
-  lazy val prebidAnalyticsController = wire[PrebidAnalyticsController]
   lazy val passbackController = wire[PassbackController]
   lazy val ampIframeHtmlController = wire[AmpIframeHtmlController]
   lazy val nonRefreshableLineItemsController = wire[nonRefreshableLineItemsController]

commercial/conf/logback.xml

@@ -25,7 +25,13 @@
 
     <root level="INFO">
         <appender-ref ref="LOGFILE"/>
-        <appender-ref ref="STDOUT"/>
     </root>
+    <if condition='!property("STAGE").contains("DEV")'>
+        <then>
+            <root level="INFO">
+                <appender-ref ref="STDOUT"/>
+            </root>
+        </then>
+    </if>
 
 </configuration>

commercial/conf/routes

@@ -41,9 +41,6 @@ GET         /commercial/cmp/shortvendorlist.json
 GET         /ads.txt                                                        commercial.controllers.AdsDotTextViewController.renderTextFile()
 GET         /app-ads.txt                                                    commercial.controllers.AdsDotTextViewController.renderAppTextFile()
 
-# Analytics
-POST        /commercial/api/hb                                              commercial.controllers.PrebidAnalyticsController.insert()
-
 # Passbacks
 GET         /commercial/ias-passback/:size                                  commercial.controllers.PassbackController.renderIasPassback(size)
 

common/app/common/configuration.scala

@@ -515,8 +515,6 @@ class GuardianConfiguration extends GuLogging {
 
     lazy val expiredPaidContentUrl = s"${site.host}/info/2015/feb/06/paid-content-removal-policy"
 
-    lazy val prebidAnalyticsStream = configuration.getMandatoryStringProperty("commercial.prebid.analytics.stream")
-
     lazy val prebidServerUrl =
       configuration.getStringProperty("commercial.prebid.server.url") getOrElse "http://localhost:8000"
 

common/app/common/package.scala

@@ -10,7 +10,7 @@ import model.CacheTime.RecentlyUpdated
 import model.Cached.RevalidatableResult
 import model.{ApplicationContext, Cached, NoCache}
 import play.api.Logger
-import play.api.libs.json.{JsObject, JsString}
+import play.api.libs.json.{JsObject, JsString, JsValue}
 import play.api.mvc.{RequestHeader, Result}
 import play.twirl.api.Html
 import model.ApplicationContext
@@ -162,7 +162,10 @@ object `package`
       JsonComponent(page, json)
     }
 
-  def renderJson(json: String, page: model.Page)(implicit request: RequestHeader, context: ApplicationContext): Result =
+  def renderJson(json: JsValue, page: model.Page)(implicit
+      request: RequestHeader,
+      context: ApplicationContext,
+  ): Result =
     Cached(page) {
       RevalidatableResult.Ok(json)
     }

common/app/conf/switches/ABTestSwitches.scala

@@ -77,7 +77,18 @@ trait ABTestSwitches {
     "Test new GPID prebid ad units",
     owners = Seq(Owner.withEmail("[email protected]")),
     safeState = Off,
-    sellByDate = Some(LocalDate.of(2024, 12, 18)),
+    sellByDate = Some(LocalDate.of(2025, 1, 17)),
+    exposeClientSide = true,
+    highImpact = false,
+  )
+
+  Switch(
+    ABTests,
+    "ab-region-specific-prebid",
+    "Test impact of splitting the Prebid bundle by region",
+    owners = Seq(Owner.withEmail("[email protected]")),
+    safeState = Off,
+    sellByDate = Some(LocalDate.of(2025, 1, 24)),
     exposeClientSide = true,
     highImpact = false,
   )

common/app/http/RequestLoggingFilter.scala

@@ -29,8 +29,8 @@ class RequestLoggingFilter(implicit val mat: Materializer, executionContext: Exe
                 case _ => ""
               }
           }
-        // don't log uncacheable /commercial/api/hb POST requests due to the volume of them
-        if (rh.method != "POST" || rh.path != "/commercial/api/hb") {
+        // don't log uncacheable POST requests due to the volume of them
+        if (rh.method != "POST") {
           requestLogger.withResponse(response).info(s"${rh.method} ${rh.uri}$additionalInfo")
         }
       case Failure(error) =>