Bugfix/marp 1062 product module content got duplicated after several syncs

marketplace-service/src/main/java/com/axonivy/market/constants/MetaConstants.java

@@ -8,4 +8,5 @@ public class MetaConstants {
   public static final String META_FILE = "meta.json";
   public static final String DEFAULT_VENDOR_NAME = "Axon Ivy AG";
   public static final String DEFAULT_VENDOR_URL = "https://www.axonivy.com";
+  public static final String INITIAL_VERSION = "1.0";
 }

marketplace-service/src/main/java/com/axonivy/market/constants/ReadmeConstants.java

@@ -10,4 +10,11 @@ public class ReadmeConstants {
   public static final String README_FILE_NAME = "README";
   public static final String DEMO_PART = "## Demo";
   public static final String SETUP_PART = "## Setup";
+  public static final String DEMO_SETUP_TITLE = "(?i)## Demo|## Setup";
+  public static final String IMAGE_EXTENSION = "(.*?).(jpeg|jpg|png|gif)";
+  public static final String README_IMAGE_FORMAT = "\\(([^)]*?%s[^)]*?)\\)";
+  public static final String IMAGE_DOWNLOAD_URL_FORMAT = "(%s)";
+  public static final String DESCRIPTION = "description";
+  public static final String DEMO = "demo";
+  public static final String SETUP = "setup";
 }

marketplace-service/src/main/java/com/axonivy/market/entity/GitHubRepoMeta.java

@@ -16,4 +16,5 @@ public class GitHubRepoMeta {
   private String repoName;
   private Long lastChange;
   private String lastSHA1;
+  private boolean isSyncInProgress;
 }

marketplace-service/src/main/java/com/axonivy/market/factory/ProductFactory.java

@@ -1,5 +1,6 @@
 package com.axonivy.market.factory;
 
+import com.axonivy.market.constants.MetaConstants;
 import com.axonivy.market.entity.Product;
 import com.axonivy.market.github.model.Meta;
 import com.axonivy.market.github.util.GitHubUtils;
@@ -56,6 +57,7 @@ public static Product mappingByMetaJSONFile(Product product, GHContent ghContent
     }
 
     product.setId(meta.getId());
+    extractSourceUrl(product, meta);
     product.setNames(mappingMultilingualismValueByMetaJSONFile(meta.getNames()));
     product.setMarketDirectory(extractParentDirectory(ghContent));
     product.setListed(meta.getListed());
@@ -72,9 +74,14 @@ public static Product mappingByMetaJSONFile(Product product, GHContent ghContent
     product.setContactUs(BooleanUtils.isTrue(meta.getContactUs()));
     product.setCost(StringUtils.isBlank(meta.getCost()) ? "Free" : StringUtils.capitalize(meta.getCost()));
     product.setCompatibility(meta.getCompatibility());
-    extractSourceUrl(product, meta);
     product.setArtifacts(meta.getMavenArtifacts());
-    product.setReleasedVersions(new ArrayList<>());
+
+    boolean isSourceUrlNotExist = StringUtils.isBlank(product.getSourceUrl());
+    if (isSourceUrlNotExist) {
+      product.setReleasedVersions(List.of(MetaConstants.INITIAL_VERSION));
+      product.setNewestReleaseVersion(MetaConstants.INITIAL_VERSION);
+    }
+
     return product;
   }
 

marketplace-service/src/main/java/com/axonivy/market/github/service/impl/GHAxonIvyProductRepoServiceImpl.java

@@ -53,13 +53,6 @@ public class GHAxonIvyProductRepoServiceImpl implements GHAxonIvyProductRepoServ
   private final ProductJsonContentRepository productJsonContentRepository;
   private String repoUrl;
   private static final ObjectMapper objectMapper = new ObjectMapper();
-  public static final String DEMO_SETUP_TITLE = "(?i)## Demo|## Setup";
-  public static final String IMAGE_EXTENSION = "(.*?).(jpeg|jpg|png|gif)";
-  public static final String README_IMAGE_FORMAT = "\\(([^)]*?%s[^)]*?)\\)";
-  public static final String IMAGE_DOWNLOAD_URL_FORMAT = "(%s)";
-  public static final String DESCRIPTION = "description";
-  public static final String DEMO = "demo";
-  public static final String SETUP = "setup";
 
   public GHAxonIvyProductRepoServiceImpl(GitHubService gitHubService,
       ProductJsonContentRepository productJsonContentRepository) {
@@ -190,9 +183,9 @@ public void extractReadMeFileFromContents(Product product, List<GHContent> conte
           String locale = getReadmeFileLocale(readmeFile.getName());
           getExtractedPartsOfReadme(moduleContents, readmeContents, locale);
         }
-        productModuleContent.setDescription(replaceEmptyContentsWithEnContent(moduleContents.get(DESCRIPTION)));
-        productModuleContent.setDemo(replaceEmptyContentsWithEnContent(moduleContents.get(DEMO)));
-        productModuleContent.setSetup(replaceEmptyContentsWithEnContent(moduleContents.get(SETUP)));
+        productModuleContent.setDescription(replaceEmptyContentsWithEnContent(moduleContents.get(ReadmeConstants.DESCRIPTION)));
+        productModuleContent.setDemo(replaceEmptyContentsWithEnContent(moduleContents.get(ReadmeConstants.DEMO)));
+        productModuleContent.setSetup(replaceEmptyContentsWithEnContent(moduleContents.get(ReadmeConstants.SETUP)));
       }
     } catch (Exception e) {
       log.error("Cannot get README file's content {}", e.getMessage());
@@ -270,7 +263,7 @@ public String updateImagesWithDownloadUrl(Product product, List<GHContent> conte
       throws IOException {
     Map<String, String> imageUrls = new HashMap<>();
     List<GHContent> productImages = contents.stream().filter(GHContent::isFile)
-        .filter(content -> content.getName().toLowerCase().matches(IMAGE_EXTENSION)).toList();
+        .filter(content -> content.getName().toLowerCase().matches(ReadmeConstants.IMAGE_EXTENSION)).toList();
     if (!CollectionUtils.isEmpty(productImages)) {
       for (GHContent productImage : productImages) {
         imageUrls.put(productImage.getName(), productImage.getDownloadUrl());
@@ -279,9 +272,9 @@ public String updateImagesWithDownloadUrl(Product product, List<GHContent> conte
       getImagesFromImageFolder(product, contents, imageUrls);
     }
     for (Map.Entry<String, String> entry : imageUrls.entrySet()) {
-      String imageUrlPattern = String.format(README_IMAGE_FORMAT, Pattern.quote(entry.getKey()));
+      String imageUrlPattern = String.format(ReadmeConstants.README_IMAGE_FORMAT, Pattern.quote(entry.getKey()));
       readmeContents = readmeContents.replaceAll(imageUrlPattern,
-          String.format(IMAGE_DOWNLOAD_URL_FORMAT, entry.getValue()));
+          String.format(ReadmeConstants.IMAGE_DOWNLOAD_URL_FORMAT, entry.getValue()));
 
     }
     return readmeContents;
@@ -303,7 +296,7 @@ private void getImagesFromImageFolder(Product product, List<GHContent> contents,
   // missing one of them
   public void getExtractedPartsOfReadme(Map<String,Map<String,String>> moduleContents, String readmeContents,
       String locale) {
-    String[] parts = readmeContents.split(DEMO_SETUP_TITLE);
+    String[] parts = readmeContents.split(ReadmeConstants.DEMO_SETUP_TITLE);
     int demoIndex = readmeContents.indexOf(ReadmeConstants.DEMO_PART);
     int setupIndex = readmeContents.indexOf(ReadmeConstants.SETUP_PART);
     String description = Strings.EMPTY;
@@ -328,9 +321,9 @@ public void getExtractedPartsOfReadme(Map<String,Map<String,String>> moduleConte
       setup = parts[1];
     }
     locale = StringUtils.isEmpty(locale) ? Language.EN.getValue() : locale.toLowerCase();
-    addLocaleContent(moduleContents, DESCRIPTION, description.trim(), locale);
-    addLocaleContent(moduleContents, DEMO, demo.trim(), locale);
-    addLocaleContent(moduleContents, SETUP, setup.trim(), locale);
+    addLocaleContent(moduleContents, ReadmeConstants.DESCRIPTION, description.trim(), locale);
+    addLocaleContent(moduleContents, ReadmeConstants.DEMO, demo.trim(), locale);
+    addLocaleContent(moduleContents, ReadmeConstants.SETUP, setup.trim(), locale);
   }
 
   private void addLocaleContent(Map<String, Map<String, String>> moduleContents, String type, String content, String locale) {
@@ -355,7 +348,7 @@ private boolean hasChildConnector(GHRepository ghRepository) {
   }
 
   private boolean hasImageDirectives(String readmeContents) {
-    Pattern pattern = Pattern.compile(IMAGE_EXTENSION);
+    Pattern pattern = Pattern.compile(ReadmeConstants.IMAGE_EXTENSION);
     Matcher matcher = pattern.matcher(readmeContents);
     return matcher.find();
   }

marketplace-service/src/main/java/com/axonivy/market/github/util/GitHubUtils.java

@@ -37,7 +37,7 @@ public static String getDownloadUrl(GHContent content) {
     } catch (IOException e) {
       log.error("Cannot get DownloadURl from GHContent: ", e);
     }
-    return "";
+    return StringUtils.EMPTY;
   }
 
   public static <T> List<T> mapPagedIteratorToList(PagedIterable<T> paged) {
@@ -79,7 +79,7 @@ public static String extractMessageFromExceptionMessage(String exceptionMessage)
         return json.substring(startIndex, endIndex);
       }
     }
-    return "";
+    return StringUtils.EMPTY;
   }
 
   private static String extractJson(String text) {
@@ -88,6 +88,6 @@ private static String extractJson(String text) {
     if (start != -1 && end != -1) {
       return text.substring(start, end);
     }
-    return "";
+    return StringUtils.EMPTY;
   }
 }

marketplace-service/src/main/java/com/axonivy/market/repository/ProductModuleContentRepository.java

@@ -4,7 +4,11 @@
 import org.springframework.data.mongodb.repository.MongoRepository;
 import org.springframework.stereotype.Repository;
 
+import java.util.List;
+
 @Repository
 public interface ProductModuleContentRepository extends MongoRepository<ProductModuleContent, String> {
     ProductModuleContent findByTagAndProductId(String tag, String productId);
+
+    List<ProductModuleContent> findByProductId(String productId);
 }

marketplace-service/src/main/java/com/axonivy/market/service/impl/ProductServiceImpl.java

@@ -14,6 +14,7 @@
 import java.util.*;
 
 import com.axonivy.market.comparator.MavenVersionComparator;
+import com.axonivy.market.constants.MetaConstants;
 import com.axonivy.market.util.VersionUtils;
 import org.apache.commons.lang3.BooleanUtils;
 import org.apache.commons.lang3.ObjectUtils;
@@ -91,7 +92,6 @@ public class ProductServiceImpl implements ProductService {
   private String marketRepoBranch;
 
   public static final String NON_NUMERIC_CHAR = "[^0-9.]";
-  private static final String INITIAL_VERSION = "1.0";
   private final SecureRandom random = new SecureRandom();
 
   public ProductServiceImpl(ProductRepository productRepository,
@@ -351,10 +351,8 @@ private void syncProductsFromGitHubRepo() {
 
   private void updateProductContentForNonStandardProduct(Map.Entry<String, List<GHContent>> ghContentEntity, Product product) {
     ProductModuleContent initialContent = new ProductModuleContent();
-    initialContent.setTag(INITIAL_VERSION);
+    initialContent.setTag(MetaConstants.INITIAL_VERSION);
     initialContent.setProductId(product.getId());
-    product.setReleasedVersions(List.of(INITIAL_VERSION));
-    product.setNewestReleaseVersion(INITIAL_VERSION);
     axonIvyProductRepoService.extractReadMeFileFromContents(product, ghContentEntity.getValue(), initialContent);
     productModuleContentRepository.save(initialContent);
   }
@@ -383,7 +381,12 @@ private void updateProductFromReleaseTags(Product product, GHRepository productR
       ghTags = ghTags.stream().filter(t -> !currentTags.contains(t.getName())).toList();
     }
 
+    List<String> existedTag = productModuleContentRepository.findByProductId(product.getId()).stream()
+        .map(ProductModuleContent::getTag).toList();
     for (GHTag ghTag : ghTags) {
+      if (existedTag.contains(ghTag.getName())) {
+        continue;
+      }
       ProductModuleContent productModuleContent =
           axonIvyProductRepoService.getReadmeAndProductContentsFromTag(product, productRepo, ghTag.getName());
       if (productModuleContent != null) {

marketplace-service/src/main/java/com/axonivy/market/util/VersionUtils.java

@@ -125,6 +125,7 @@ public static String getOldestVersion(List<GHTag> tags) {
         }
         return result;
     }
+
     public static List<String> getReleaseTagsFromProduct(Product product) {
         if (Objects.isNull(product)) {
             return new ArrayList<>();

marketplace-service/src/test/java/com/axonivy/market/service/impl/ProductServiceImplTest.java

@@ -17,6 +17,7 @@
 import static org.mockito.ArgumentMatchers.anyString;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
 import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 import static org.mockito.Mockito.when;
@@ -336,7 +337,6 @@ void testSyncProductsFirstTime() throws IOException {
     mockGHContentMap.put(SAMPLE_PRODUCT_ID, List.of(mockContent));
     when(marketRepoService.fetchAllMarketItems()).thenReturn(mockGHContentMap);
     when(productModuleContentRepository.saveAll(anyList())).thenReturn(List.of(mockReadmeProductContent()));
-
     // Executes
     productService.syncLatestDataFromMarketRepo();
     verify(productModuleContentRepository).saveAll(argumentCaptorProductModuleContents.capture());
@@ -346,6 +346,36 @@ void testSyncProductsFirstTime() throws IOException {
         .isEqualTo(List.of(mockReadmeProductContent()));
   }
 
+  @Test
+  void testSyncProductsFirstTimeWithExistedData() throws IOException {
+    var mockCommit = mockGHCommitHasSHA1(SHA1_SAMPLE);
+    when(marketRepoService.getLastCommit(anyLong())).thenReturn(mockCommit);
+    when(repoMetaRepository.findByRepoName(anyString())).thenReturn(null);
+    when(gitHubService.getRepository(any())).thenReturn(ghRepository);
+
+    GHTag mockTag = mock(GHTag.class);
+    GHCommit mockGHCommit = mock(GHCommit.class);
+
+    when(mockTag.getName()).thenReturn(RELEASE_TAG);
+    when(mockTag.getCommit()).thenReturn(mockGHCommit);
+    when(mockGHCommit.getCommitDate()).thenReturn(new Date());
+
+    when(gitHubService.getRepositoryTags(anyString())).thenReturn(List.of(mockTag));
+    var mockContent = mockGHContentAsMetaJSON();
+    InputStream inputStream = this.getClass().getResourceAsStream(SLASH.concat(META_FILE));
+    when(mockContent.read()).thenReturn(inputStream);
+    Map<String, List<GHContent>> mockGHContentMap = new HashMap<>();
+    mockGHContentMap.put(SAMPLE_PRODUCT_ID, List.of(mockContent));
+    when(marketRepoService.fetchAllMarketItems()).thenReturn(mockGHContentMap);
+
+    ProductModuleContent mockReturnProductContent = mockReadmeProductContent();
+    mockReturnProductContent.setTag(RELEASE_TAG);
+    when(productModuleContentRepository.findByProductId("amazon-comprehend")).thenReturn(List.of(mockReturnProductContent));
+    // Executes
+    productService.syncLatestDataFromMarketRepo();
+    verify(productModuleContentRepository, never()).saveAll(anyList());
+  }
+
   @Test
   void testSyncProductsFirstTimeWithOutSourceUrl() throws IOException {
     var mockCommit = mockGHCommitHasSHA1(SHA1_SAMPLE);
@@ -390,12 +420,10 @@ void testSyncProductsSecondTime() throws IOException {
 
     ProductModuleContent mockReturnProductContent = mockReadmeProductContent();
     mockReturnProductContent.setTag("v10.0.3");
-
     when(ghAxonIvyProductRepoService.getReadmeAndProductContentsFromTag(any(), any(), anyString()))
         .thenReturn(mockReturnProductContent);
     when(productModuleContentRepository.saveAll(anyList()))
         .thenReturn(List.of(mockReadmeProductContent(), mockReturnProductContent));
-
     // Executes
     productService.syncLatestDataFromMarketRepo();
 
@@ -604,6 +632,7 @@ private GHContent mockGHContentAsMetaJSON() {
   private ProductModuleContent mockReadmeProductContent() {
     ProductModuleContent productModuleContent = new ProductModuleContent();
     productModuleContent.setId("123");
+    productModuleContent.setProductId("amazon-comprehend");
     productModuleContent.setTag("v10.0.2");
     productModuleContent.setName("Amazon Comprehend");
     Map<String, String> description = new HashMap<>();

marketplace-service/src/test/java/com/axonivy/market/util/VersionUtilsTest.java

@@ -1,5 +1,6 @@
 package com.axonivy.market.util;
 
+import com.axonivy.market.entity.Product;
 import com.axonivy.market.enums.NonStandardProduct;
 import org.apache.commons.lang3.StringUtils;
 import org.junit.jupiter.api.Assertions;
@@ -179,4 +180,16 @@ void testGetOldestVersionWithNonNumericCharacters() {
 
         Assertions.assertEquals("1.0", oldestTag); // Assuming the replacement of non-numeric characters works correctly
     }
+
+    @Test
+    void testGetReleaseTagsFromProduct() {
+        Assertions.assertEquals(0, VersionUtils.getReleaseTagsFromProduct(null).size());
+
+        Product mockProduct = new Product();
+        mockProduct.setId("portal");
+        List<String> releasedVersions = List.of("10.0.1", "10.0.2");
+        mockProduct.setReleasedVersions(releasedVersions);
+
+        Assertions.assertEquals(releasedVersions.get(0), VersionUtils.getReleaseTagsFromProduct(mockProduct).get(0));
+    }
 }