MARP-1312 Refactor sync product-version (#221)

marketplace-service/src/main/java/com/axonivy/market/controller/ProductController.java

@@ -116,6 +116,10 @@ public ResponseEntity<Message> syncProducts(@RequestHeader(value = AUTHORIZATION
     return new ResponseEntity<>(message, HttpStatus.OK);
   }
 
+  /**
+   * @deprecated
+   */
+  @Deprecated(forRemoval = true , since = "1.6.0")
   @PutMapping(SYNC_PRODUCT_VERSION)
   @Operation(hidden = true)
   public ResponseEntity<Message> syncProductVersions(@RequestHeader(value = AUTHORIZATION) String authorizationHeader

marketplace-service/src/main/java/com/axonivy/market/github/service/impl/GHAxonIvyProductRepoServiceImpl.java

@@ -4,12 +4,15 @@
 import com.axonivy.market.constants.ReadmeConstants;
 import com.axonivy.market.entity.Product;
 import com.axonivy.market.entity.ProductModuleContent;
+import com.axonivy.market.enums.Language;
 import com.axonivy.market.github.service.GHAxonIvyProductRepoService;
 import com.axonivy.market.github.service.GitHubService;
 import com.axonivy.market.github.util.GitHubUtils;
+import com.axonivy.market.model.ReadmeContentsModel;
 import com.axonivy.market.service.ImageService;
 import com.axonivy.market.util.ProductContentUtils;
 import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.StringUtils;
 import org.kohsuke.github.GHContent;
 import org.kohsuke.github.GHOrganization;
 import org.springframework.stereotype.Service;
@@ -23,6 +26,7 @@
 import java.util.Optional;
 
 import static com.axonivy.market.constants.CommonConstants.IMAGE_ID_PREFIX;
+import static com.axonivy.market.util.ProductContentUtils.*;
 
 @Log4j2
 @Service
@@ -66,7 +70,10 @@ public void extractReadMeFileFromContents(Product product, List<GHContent> conte
           if (ProductContentUtils.hasImageDirectives(readmeContents)) {
             readmeContents = updateImagesWithDownloadUrl(product.getId(), contents, readmeContents);
           }
-          ProductContentUtils.getExtractedPartsOfReadme(moduleContents, readmeContents, readmeFile.getName());
+
+          ReadmeContentsModel readmeContentsModel = ProductContentUtils.getExtractedPartsOfReadme(readmeContents);
+
+          ProductContentUtils.mappingDescriptionSetupAndDemo(moduleContents, readmeFile.getName(), readmeContentsModel);
         }
         ProductContentUtils.updateProductModuleTabContents(productModuleContent, moduleContents);
       }

marketplace-service/src/main/java/com/axonivy/market/model/ReadmeContentsModel.java

@@ -0,0 +1,16 @@
+package com.axonivy.market.model;
+
+import lombok.AllArgsConstructor;
+import lombok.Getter;
+import lombok.NoArgsConstructor;
+import lombok.Setter;
+
+@Getter
+@Setter
+@AllArgsConstructor
+@NoArgsConstructor
+public class ReadmeContentsModel {
+  private String description;
+  private String demo;
+  private String setup;
+}

marketplace-service/src/main/java/com/axonivy/market/repository/ProductJsonContentRepository.java

@@ -11,5 +11,7 @@ public interface ProductJsonContentRepository extends MongoRepository<ProductJso
 
   List<ProductJsonContent> findByProductIdAndVersion(String productId, String version);
 
+  List<ProductJsonContent> findByProductIdAndVersionIn(String productId, List<String> versions);
+
   void deleteAllByProductId(String productId);
 }

marketplace-service/src/main/java/com/axonivy/market/schedulingtask/ScheduledTasks.java

@@ -2,7 +2,6 @@
 
 import com.axonivy.market.repository.ProductRepository;
 import com.axonivy.market.service.ExternalDocumentService;
-import com.axonivy.market.service.MetadataService;
 import com.axonivy.market.service.ProductService;
 import lombok.AllArgsConstructor;
 import lombok.extern.log4j.Log4j2;
@@ -15,27 +14,19 @@
 public class ScheduledTasks {
 
   private static final String SCHEDULING_TASK_PRODUCTS_CRON = "0 0 0/1 ? * *";
-  // Maven version sync will start at 00:20 in order to prevent running at the same time with product repo sync
-  private static final String SCHEDULING_TASK_MAVEN_VERSION_CRON = "0 20 0 * * *";
   // External documentation sync will start at 00:40 in order to prevent running at the same time with other
   private static final String SCHEDULING_TASK_DOCUMENTS_CRON = "0 40 0 * * *";
 
   final ProductRepository productRepo;
   final ProductService productService;
   final ExternalDocumentService externalDocumentService;
-  private final MetadataService metadataService;
 
   @Scheduled(cron = SCHEDULING_TASK_PRODUCTS_CRON)
   public void syncDataForProductFromGitHubRepo() {
     log.warn("Started sync data for product from GitHub repo");
     productService.syncLatestDataFromMarketRepo(false);
   }
 
-  @Scheduled(cron = SCHEDULING_TASK_MAVEN_VERSION_CRON)
-  public void syncDataForMavenMetadata() {
-    log.warn("Started sync data for Maven metadata");
-    metadataService.syncAllProductsMetadata();
-  }
   @Scheduled(cron = SCHEDULING_TASK_DOCUMENTS_CRON)
   public void syncDataForProductDocuments() {
     log.warn("Started sync data for product document");

marketplace-service/src/main/java/com/axonivy/market/service/MetadataService.java

@@ -1,9 +1,14 @@
 package com.axonivy.market.service;
 
+import com.axonivy.market.bo.Artifact;
 import com.axonivy.market.entity.Product;
+import java.util.List;
 
 public interface MetadataService {
 
   int syncAllProductsMetadata();
+
   boolean syncProductMetadata(Product product);
+
+  void updateArtifactAndMetadata(String productId , List<String> versions , List<Artifact> artifacts);
 }

marketplace-service/src/main/java/com/axonivy/market/service/ProductJsonContentService.java

@@ -1,6 +1,8 @@
 package com.axonivy.market.service;
 
+import com.axonivy.market.entity.ProductJsonContent;
+
 public interface ProductJsonContentService {
-  void updateProductJsonContent(String jsonContent, String currentVersion, String replaceVersion,
+  ProductJsonContent updateProductJsonContent(String jsonContent, String currentVersion, String replaceVersion,
       String productId, String productName);
 }

marketplace-service/src/main/java/com/axonivy/market/service/impl/MetadataServiceImpl.java

@@ -127,28 +127,65 @@ public boolean syncProductMetadata(Product product) {
     return true;
   }
 
+  @Override
+  public void updateArtifactAndMetadata(String productId, List<String> versions, List<Artifact> artifacts) {
+    Set<Metadata> metadataSet = new HashSet<>(metadataRepo.findByProductId(productId));
+    Set<Artifact> artifactsFromNewVersions = new HashSet<>();
+
+    if (ObjectUtils.isNotEmpty(versions)) {
+      List<ProductJsonContent> productJsonContents = productJsonRepo.findByProductIdAndVersionIn(productId, versions);
+      for (ProductJsonContent productJsonContent : productJsonContents) {
+        List<Artifact> artifactsFromNonSyncedVersions = MavenUtils.getMavenArtifactsFromProductJson(productJsonContent);
+        artifactsFromNewVersions.addAll(artifactsFromNonSyncedVersions);
+      }
+      log.info("**MetadataService: New versions detected: {} in product {}", versions, productId);
+    }
+
+    metadataSet.addAll(MavenUtils.convertArtifactsToMetadataSet(artifactsFromNewVersions, productId));
+    if (ObjectUtils.isNotEmpty(artifacts)) {
+      metadataSet.addAll(MavenUtils.convertArtifactsToMetadataSet(new HashSet<>(artifacts), productId));
+    }
+
+    if (CollectionUtils.isEmpty(metadataSet)) {
+      log.info("**MetadataService: No artifact found in product {}", productId);
+      return;
+    }
+
+    MavenArtifactVersion artifactVersion = mavenArtifactVersionRepo.findById(productId)
+        .orElse(MavenArtifactVersion.builder().productId(productId).build());
+
+    artifactVersion.setAdditionalArtifactsByVersion(new HashMap<>());
+    updateMavenArtifactVersionData(metadataSet, artifactVersion);
+
+    mavenArtifactVersionRepo.save(artifactVersion);
+    metadataRepo.saveAll(metadataSet);
+  }
+
   public void updateMavenArtifactVersionFromMetadata(MavenArtifactVersion artifactVersionCache,
       Metadata metadata) {
     // Skip to add new model for product artifact
     if (MavenUtils.isProductArtifactId(metadata.getArtifactId())) {
       return;
     }
-    metadata.getVersions().forEach(version -> {
-      if (VersionUtils.isSnapshotVersion(version)) {
-        if (VersionUtils.isOfficialVersionOrUnReleasedDevVersion(metadata.getVersions().stream().toList(), version)) {
-          updateMavenArtifactVersionForNonReleaseDevVersion(artifactVersionCache, metadata, version);
-        }
-      } else {
+
+    for (String version : metadata.getVersions()) {
+      boolean isSnapshotVersion = VersionUtils.isSnapshotVersion(version);
+      boolean isOfficialVersionOrUnReleasedDevVersion =
+          VersionUtils.isOfficialVersionOrUnReleasedDevVersion(metadata.getVersions(), version);
+
+      if (isSnapshotVersion && isOfficialVersionOrUnReleasedDevVersion) {
+        updateMavenArtifactVersionForNonReleaseDevVersion(artifactVersionCache, metadata, version);
+      } else if (!isSnapshotVersion) {
         updateMavenArtifactVersionCacheWithModel(artifactVersionCache, version, metadata);
       }
-    });
+    }
   }
 
   public void updateMavenArtifactVersionForNonReleaseDevVersion(MavenArtifactVersion artifactVersionCache,
       Metadata metadata, String version) {
     Metadata snapShotMetadata = MavenUtils.buildSnapShotMetadataFromVersion(metadata, version);
-    MetadataReaderUtils.updateMetadataFromMavenXML(MavenUtils.getMetadataContentFromUrl(snapShotMetadata.getUrl()),
-        snapShotMetadata, true);
+    String xmlDataForSnapshotMetadata = MavenUtils.getMetadataContentFromUrl(snapShotMetadata.getUrl());
+    MetadataReaderUtils.updateMetadataFromMavenXML(xmlDataForSnapshotMetadata, snapShotMetadata, true);
     updateMavenArtifactVersionCacheWithModel(artifactVersionCache, version, snapShotMetadata);
   }
 
@@ -157,12 +194,13 @@ public Set<Artifact> getArtifactsFromNonSyncedVersion(String productId, List<Str
     if (CollectionUtils.isEmpty(nonSyncedVersions)) {
       return artifacts;
     }
-    nonSyncedVersions.forEach(version -> {
-      ProductJsonContent productJson =
-          productJsonRepo.findByProductIdAndVersion(productId, version).stream().findAny().orElse(null);
-      List<Artifact> artifactsInVersion = MavenUtils.getMavenArtifactsFromProductJson(productJson);
+
+    List<ProductJsonContent> productJsonContents = productJsonRepo.findByProductIdAndVersionIn(productId, nonSyncedVersions);
+    for (ProductJsonContent productJsonContent : productJsonContents) {
+      List<Artifact> artifactsInVersion = MavenUtils.getMavenArtifactsFromProductJson(productJsonContent);
       artifacts.addAll(artifactsInVersion);
-    });
+    }
+
     return artifacts;
   }
 }

marketplace-service/src/main/java/com/axonivy/market/service/impl/ProductContentServiceImpl.java

@@ -4,7 +4,9 @@
 import com.axonivy.market.constants.CommonConstants;
 import com.axonivy.market.constants.ProductJsonConstants;
 import com.axonivy.market.constants.ReadmeConstants;
+import com.axonivy.market.entity.Image;
 import com.axonivy.market.entity.ProductModuleContent;
+import com.axonivy.market.model.ReadmeContentsModel;
 import com.axonivy.market.service.FileDownloadService;
 import com.axonivy.market.service.ImageService;
 import com.axonivy.market.service.ProductContentService;
@@ -13,7 +15,6 @@
 import com.axonivy.market.util.ProductContentUtils;
 import lombok.AllArgsConstructor;
 import lombok.extern.log4j.Log4j2;
-import org.apache.commons.lang3.ObjectUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.logging.log4j.util.Strings;
 import org.springframework.stereotype.Service;
@@ -39,8 +40,7 @@ public class ProductContentServiceImpl implements ProductContentService {
   @Override
   public ProductModuleContent getReadmeAndProductContentsFromVersion(String productId, String version, String url,
       Artifact artifact, String productName) {
-    ProductModuleContent productModuleContent = ProductContentUtils.initProductModuleContent(productId,
-        version);
+    ProductModuleContent productModuleContent = ProductContentUtils.initProductModuleContent(productId, version);
     String unzippedFolderPath = Strings.EMPTY;
     try {
       unzippedFolderPath = fileDownloadService.downloadAndUnzipProductContentFile(url, artifact);
@@ -59,54 +59,60 @@ public ProductModuleContent getReadmeAndProductContentsFromVersion(String produc
 
   public void updateDependencyContentsFromProductJson(ProductModuleContent productModuleContent,
       String productId, String unzippedFolderPath, String productName) throws IOException {
-    List<Artifact> artifacts = MavenUtils.convertProductJsonToMavenProductInfo(
-        Paths.get(unzippedFolderPath));
+    List<Artifact> artifacts = MavenUtils.convertProductJsonToMavenProductInfo(Paths.get(unzippedFolderPath));
     ProductContentUtils.updateProductModule(productModuleContent, artifacts);
     Path productJsonPath = Paths.get(unzippedFolderPath, ProductJsonConstants.PRODUCT_JSON_FILE);
     String content = MavenUtils.extractProductJsonContent(productJsonPath);
+
     productJsonContentService.updateProductJsonContent(content, productModuleContent.getVersion(),
         ProductJsonConstants.VERSION_VALUE, productId, productName);
   }
 
   private void extractReadMeFileFromContents(String productId, String unzippedFolderPath,
       ProductModuleContent productModuleContent) {
-    try {
-      List<Path> readmeFiles;
-      Map<String, Map<String, String>> moduleContents = new HashMap<>();
-      try (Stream<Path> readmePathStream = Files.walk(Paths.get(unzippedFolderPath))) {
-        readmeFiles = readmePathStream.filter(Files::isRegularFile).filter(
-            path -> path.getFileName().toString().startsWith(ReadmeConstants.README_FILE_NAME)).toList();
-      }
-      if (ObjectUtils.isNotEmpty(readmeFiles)) {
-        for (Path readmeFile : readmeFiles) {
-          String readmeContents = Files.readString(readmeFile);
-          if (ProductContentUtils.hasImageDirectives(readmeContents)) {
-            readmeContents = updateImagesWithDownloadUrl(productId, unzippedFolderPath, readmeContents);
-          }
-          ProductContentUtils.getExtractedPartsOfReadme(moduleContents, readmeContents,
-              readmeFile.getFileName().toString());
+    Map<String, Map<String, String>> moduleContents = new HashMap<>();
+    try (Stream<Path> readmePathStream = Files.walk(Paths.get(unzippedFolderPath))){
+      List<Path> readmeFiles  = readmePathStream.filter(Files::isRegularFile)
+          .filter(path -> path.getFileName().toString().startsWith(ReadmeConstants.README_FILE_NAME))
+          .toList();
+
+      for (Path readmeFile : readmeFiles) {
+        String readmeContents = Files.readString(readmeFile);
+        if (ProductContentUtils.hasImageDirectives(readmeContents)) {
+          readmeContents = updateImagesWithDownloadUrl(productId, unzippedFolderPath, readmeContents);
         }
-        ProductContentUtils.updateProductModuleTabContents(productModuleContent, moduleContents);
+
+        ReadmeContentsModel readmeContentsModel = ProductContentUtils.getExtractedPartsOfReadme(readmeContents);
+
+        ProductContentUtils.mappingDescriptionSetupAndDemo(moduleContents, readmeFile.getFileName().toString(),
+            readmeContentsModel);
       }
-    } catch (Exception e) {
+      ProductContentUtils.updateProductModuleTabContents(productModuleContent, moduleContents);
+    } catch (IOException e) {
       log.error("Cannot get README file's content from folder {}: {}", unzippedFolderPath, e.getMessage());
     }
   }
 
   public String updateImagesWithDownloadUrl(String productId, String unzippedFolderPath,
-      String readmeContents) throws IOException {
-    List<Path> allImagePaths;
+      String readmeContents) {
     Map<String, String> imageUrls = new HashMap<>();
     try (Stream<Path> imagePathStream = Files.walk(Paths.get(unzippedFolderPath))) {
-      allImagePaths = imagePathStream.filter(Files::isRegularFile).filter(
+      List<Path> allImagePaths = imagePathStream.filter(Files::isRegularFile).filter(
           path -> path.getFileName().toString().toLowerCase().matches(CommonConstants.IMAGE_EXTENSION)).toList();
-    }
-    for (Path imagePath : allImagePaths) {
-      Optional.of(imageService.mappingImageFromDownloadedFolder(productId, imagePath)).ifPresent(
-          image -> imageUrls.put(imagePath.getFileName().toString(),
-              CommonConstants.IMAGE_ID_PREFIX.concat(image.getId())));
-    }
 
-    return ProductContentUtils.replaceImageDirWithImageCustomId(imageUrls, readmeContents);
+      for (Path imagePath : allImagePaths) {
+        Image currentImage = imageService.mappingImageFromDownloadedFolder(productId, imagePath);
+        Optional.ofNullable(currentImage).ifPresent(image -> {
+          String imageFileName = imagePath.getFileName().toString();
+          String imageIdFormat = CommonConstants.IMAGE_ID_PREFIX.concat(image.getId());
+          imageUrls.put(imageFileName, imageIdFormat);
+        });
+      }
+
+      return ProductContentUtils.replaceImageDirWithImageCustomId(imageUrls, readmeContents);
+    } catch (Exception e) {
+      log.error(e.getMessage());
+    }
+    return readmeContents;
   }
 }

marketplace-service/src/main/java/com/axonivy/market/service/impl/ProductJsonContentServiceImpl.java

@@ -14,7 +14,7 @@ public class ProductJsonContentServiceImpl implements ProductJsonContentService
   private final ProductJsonContentRepository productJsonRepo;
 
   @Override
-  public void updateProductJsonContent(String jsonContent, String currentVersion, String replaceVersion,
+  public ProductJsonContent updateProductJsonContent(String jsonContent, String currentVersion, String replaceVersion,
       String productId, String productName) {
     if (ObjectUtils.isNotEmpty(jsonContent)) {
       ProductJsonContent productJsonContent = new ProductJsonContent();
@@ -23,7 +23,8 @@ public void updateProductJsonContent(String jsonContent, String currentVersion,
       ProductFactory.mappingIdForProductJsonContent(productJsonContent);
       productJsonContent.setName(productName);
       productJsonContent.setContent(jsonContent.replace(replaceVersion, currentVersion));
-      productJsonRepo.save(productJsonContent);
+      return productJsonRepo.save(productJsonContent);
     }
+    return null;
   }
 }