Skip to content

Commit

Permalink
feat(Rest): New endpoint allow load SPDX license info from attachment…
Browse files Browse the repository at this point in the history 
… of release (ISR, CLX, CLI)
  • Loading branch information
@hoangnt2
hoangnt2 committed Aug 23, 2023
1 parent 165d56d commit 5dfae41
Show file tree
Hide file tree
Showing 4 changed files with 225 additions and 1 deletion.
11 changes: 11 additions & 0 deletions libraries/datahandler/src/main/java/org/eclipse/sw360/datahandler/common/SW360Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -90,6 +90,17 @@ public class SW360Constants {
public static final String PLEASE_ENABLE_FLEXIBLE_PROJECT_RELEASE_RELATIONSHIP = "Please enable flexible project " +
"release relationship configuration to use this function (enable.flexible.project.release.relationship = true)";

public static final String RDF_FILE_EXTENSION = ".rdf";
public static final String MAIN_LICENSE_FILES = "LICENSE.*|License.*|license|license.txt|license.html|COPYING.*|Copying.*|copying|copying.txt|copying.html";
public static final String LICENSE_PREFIX = "license";
public static final String CONCLUDED_LICENSE_IDS = "Concluded License Ids";
public static final String LICENSE_IDS = "licenseIds";
public static final String MAIN_LICENSE_ID = "Main License Id";
public static final String OTHER_LICENSE = "otherLicense";
public static final String OTHER_LICENSE_IDS = "Other License Ids";
public static final String OTHER_LICENSE_IDS_KEY = "otherLicenseIds";
public static final String POSSIBLE_MAIN_LICENSE_IDS = "Possible Main License Ids";
public static final String TOTAL_FILE_COUNT = "totalFileCount";
public static final String SVM_COMPONENT_ID;
public static final String SVM_MONITORINGLIST_ID;
public static final Boolean SPDX_DOCUMENT_ENABLED;
Expand Down
14 changes: 14 additions & 0 deletions rest/resource-server/src/docs/asciidoc/releases.adoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -502,3 +502,17 @@ include::{snippets}/should_document_write_spdx_licenses_info_into_release/curl-r

===== Example response
include::{snippets}/should_document_write_spdx_licenses_info_into_release/http-response.adoc[]

[[resources-load-SPDX-license-info-from-attachment-of-release]]
==== Get SPDX License Info from attachment of release

A `GET` request will will retrieve SPDX License Information from the attachment of the release.

===== Example request
include::{snippets}/should_document_load_spdx_licenses_info_from_isr/curl-request.adoc[]

===== Example response for Initial Scan Report (ISR) type
include::{snippets}/should_document_load_spdx_licenses_info_from_isr/http-response.adoc[]

===== Example response for Component license information (CLX or CLI) type
include::{snippets}/should_document_load_spdx_licenses_info_from_clx_or_cli/http-response.adoc[]
111 changes: 111 additions & 0 deletions ...server/src/main/java/org/eclipse/sw360/rest/resourceserver/release/ReleaseController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,12 +24,18 @@
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.LinkedHashMap;
import java.util.Objects;
import java.util.TreeSet;
import java.util.concurrent.locks.ReentrantLock;
import java.util.stream.Collectors;

import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import com.google.common.base.Predicate;
import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.apache.thrift.TException;
Expand All @@ -46,8 +52,12 @@
import org.eclipse.sw360.datahandler.thrift.VerificationStateInfo;
import org.eclipse.sw360.datahandler.thrift.attachments.Attachment;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentDTO;
import org.eclipse.sw360.datahandler.thrift.attachments.AttachmentType;
import org.eclipse.sw360.datahandler.thrift.components.Release;
import org.eclipse.sw360.datahandler.thrift.components.ReleaseLink;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfo;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfoParsingResult;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseNameWithText;
import org.eclipse.sw360.datahandler.thrift.projects.Project;
import org.eclipse.sw360.datahandler.thrift.components.Component;
import org.eclipse.sw360.datahandler.thrift.components.ExternalToolProcess;
Expand All @@ -66,6 +76,7 @@
import org.eclipse.sw360.rest.resourceserver.packages.PackageController;
import org.eclipse.sw360.rest.resourceserver.packages.SW360PackageService;
import org.eclipse.sw360.rest.resourceserver.vendor.Sw360VendorService;
import org.eclipse.sw360.rest.resourceserver.licenseinfo.Sw360LicenseInfoService;
import org.eclipse.sw360.rest.resourceserver.vulnerability.Sw360VulnerabilityService;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.domain.Pageable;
Expand Down Expand Up @@ -130,6 +141,9 @@ public class ReleaseController implements RepresentationModelProcessor<Repositor
@NonNull
private RestControllerHelper restControllerHelper;

@NonNull
private Sw360LicenseInfoService sw360LicenseInfoService;

@NonNull
private final com.fasterxml.jackson.databind.Module sw360Module;

Expand Down Expand Up @@ -741,6 +755,103 @@ public ResponseEntity writeSpdxLicenseInfoIntoRelease(
return new ResponseEntity<>(halRelease, HttpStatus.OK);
}

@GetMapping(value = RELEASES_URL + "/{id}/spdxLicensesInfo")
public ResponseEntity<?> loadSpdxLicensesInfo(
@PathVariable("id") String releaseId,
@RequestParam("attachmentId") String attachmentId,
@RequestParam(value = "includeConcludedLicense", required = false, defaultValue = "false") boolean includeConcludedLicense) {
User user = restControllerHelper.getSw360UserFromAuthentication();
Map<String, Set<String>> licenseToSrcFilesMap = new LinkedHashMap<>();
Set<String> mainLicenseNames = new TreeSet<>();
Set<String> otherLicenseNames = new TreeSet<>();
final Set<String> concludedLicenseIds = new TreeSet<>();

AttachmentType attachmentType;
String attachmentName;
long totalFileCount = 0;
Map<String, Object> responseBody = new LinkedHashMap<>();
Predicate<LicenseInfoParsingResult> filterLicenseResult = result -> (null != result.getLicenseInfo() &&
null != result.getLicenseInfo().getLicenseNamesWithTexts());

try {
Release release = releaseService.getReleaseForUserById(releaseId, user);
attachmentType = release.getAttachments().stream()
.filter(att -> attachmentId.equals(att.getAttachmentContentId())).map(Attachment::getAttachmentType).findFirst().orElse(null);
if (null == attachmentType) {
return new ResponseEntity<>("Cannot retrieve license information for attachment id " + attachmentId + " in release "
+ releaseId + ".", HttpStatus.INTERNAL_SERVER_ERROR);
}
if (!attachmentType.equals(AttachmentType.COMPONENT_LICENSE_INFO_XML) &&
!attachmentType.equals(AttachmentType.COMPONENT_LICENSE_INFO_COMBINED) &&
!attachmentType.equals(AttachmentType.INITIAL_SCAN_REPORT)) {
return new ResponseEntity<>("Cannot retrieve license information for attachment type " + attachmentType + ".", HttpStatus.INTERNAL_SERVER_ERROR);
}
attachmentName = release.getAttachments().stream()
.filter(att -> attachmentId.equals(att.getAttachmentContentId())).map(Attachment::getFilename).findFirst().orElse("");
final boolean isISR = AttachmentType.INITIAL_SCAN_REPORT.equals(attachmentType);
if (isISR) {
includeConcludedLicense = true;
}
List<LicenseInfoParsingResult> licenseInfoResult = sw360LicenseInfoService.getLicenseInfoForAttachment(release, user, attachmentId, includeConcludedLicense);
List<LicenseNameWithText> licenseWithTexts = licenseInfoResult.stream()
.filter(filterLicenseResult)
.map(LicenseInfoParsingResult::getLicenseInfo).map(LicenseInfo::getLicenseNamesWithTexts).flatMap(Set::stream)
.filter(license -> !license.getLicenseName().equalsIgnoreCase(SW360Constants.LICENSE_NAME_UNKNOWN)
&& !license.getLicenseName().equalsIgnoreCase(SW360Constants.NA)
&& !license.getLicenseName().equalsIgnoreCase(SW360Constants.NO_ASSERTION)) // exclude unknown, n/a and noassertion
.collect(Collectors.toList());

if (attachmentName.endsWith(SW360Constants.RDF_FILE_EXTENSION)) {
if (isISR) {
totalFileCount = licenseInfoResult.stream().map(LicenseInfoParsingResult::getLicenseInfo).map(LicenseInfo::getLicenseNamesWithTexts).flatMap(Set::stream)
.map(LicenseNameWithText::getSourceFiles).filter(Objects::nonNull).flatMap(Set::stream).distinct().count();
licenseToSrcFilesMap = CommonUtils.nullToEmptyList(licenseWithTexts).stream().collect(Collectors.toMap(LicenseNameWithText::getLicenseName,
LicenseNameWithText::getSourceFiles, (oldValue, newValue) -> oldValue));
licenseWithTexts.forEach(lwt -> {
lwt.getSourceFiles().forEach(sf -> {
if (sf.replaceAll(".*/", "").matches(SW360Constants.MAIN_LICENSE_FILES)) {
concludedLicenseIds.add(lwt.getLicenseName());
}
});
});
} else {
concludedLicenseIds.addAll(licenseInfoResult.stream().flatMap(singleResult -> singleResult.getLicenseInfo().getConcludedLicenseIds().stream())
.collect(Collectors.toCollection(() -> new TreeSet<String>(String.CASE_INSENSITIVE_ORDER))));
}
otherLicenseNames = licenseWithTexts.stream().map(LicenseNameWithText::getLicenseName).collect(Collectors.toCollection(() -> new TreeSet<String>(String.CASE_INSENSITIVE_ORDER)));
otherLicenseNames.removeAll(concludedLicenseIds);
} else if (attachmentName.endsWith(SW360Constants.XML_FILE_EXTENSION)) {
mainLicenseNames = licenseWithTexts.stream()
.filter(license -> license.getType().equalsIgnoreCase(SW360Constants.LICENSE_TYPE_GLOBAL))
.map(LicenseNameWithText::getLicenseName).collect(Collectors.toCollection(() -> new TreeSet<String>(String.CASE_INSENSITIVE_ORDER)));
otherLicenseNames = licenseWithTexts.stream()
.filter(license -> !license.getType().equalsIgnoreCase(SW360Constants.LICENSE_TYPE_GLOBAL))
.map(LicenseNameWithText::getLicenseName).collect(Collectors.toCollection(() -> new TreeSet<String>(String.CASE_INSENSITIVE_ORDER)));
}
} catch (TException e) {
log.error(e.getMessage());
return new ResponseEntity<>("Cannot retrieve license information for attachment id " + attachmentId + " in release "
+ releaseId + ".", HttpStatus.INTERNAL_SERVER_ERROR);
}

if (CommonUtils.isNotEmpty(concludedLicenseIds)) {
responseBody.put(SW360Constants.LICENSE_PREFIX, SW360Constants.CONCLUDED_LICENSE_IDS);
responseBody.put(SW360Constants.LICENSE_IDS, concludedLicenseIds);
} else if (CommonUtils.isNotEmpty(mainLicenseNames)) {
responseBody.put(SW360Constants.LICENSE_PREFIX, SW360Constants.MAIN_LICENSE_ID);
responseBody.put(SW360Constants.LICENSE_IDS, mainLicenseNames);
}
responseBody.put(SW360Constants.OTHER_LICENSE, SW360Constants.OTHER_LICENSE_IDS);
responseBody.put(SW360Constants.OTHER_LICENSE_IDS_KEY, otherLicenseNames);
if (AttachmentType.INITIAL_SCAN_REPORT.equals(attachmentType)) {
responseBody.put(SW360Constants.LICENSE_PREFIX, SW360Constants.POSSIBLE_MAIN_LICENSE_IDS);
responseBody.put(SW360Constants.TOTAL_FILE_COUNT, totalFileCount);
}
responseBody.putAll(licenseToSrcFilesMap);

return new ResponseEntity<>(responseBody, HttpStatus.OK);
}

private RequestStatus linkOrUnlinkPackages(String id, Set<String> packagesInRequestBody, boolean link)
throws URISyntaxException, TException {
User sw360User = restControllerHelper.getSw360UserFromAuthentication();
Expand Down
90 changes: 89 additions & 1 deletion ...-server/src/test/java/org/eclipse/sw360/rest/resourceserver/restdocs/ReleaseSpecTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@
import java.util.Set;
import java.util.UUID;

import com.google.common.collect.Sets;
import org.apache.thrift.TException;
import org.eclipse.sw360.datahandler.thrift.*;
import org.eclipse.sw360.datahandler.thrift.attachments.*;
Expand Down Expand Up @@ -69,6 +70,11 @@
import org.eclipse.sw360.rest.resourceserver.attachment.Sw360AttachmentService;
import org.eclipse.sw360.rest.resourceserver.license.Sw360LicenseService;
import org.eclipse.sw360.rest.resourceserver.packages.SW360PackageService;
import org.eclipse.sw360.rest.resourceserver.licenseinfo.Sw360LicenseInfoService;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfo;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfoParsingResult;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseInfoRequestStatus;
import org.eclipse.sw360.datahandler.thrift.licenseinfo.LicenseNameWithText;
import org.eclipse.sw360.rest.resourceserver.release.Sw360ReleaseService;
import org.eclipse.sw360.rest.resourceserver.user.Sw360UserService;
import org.eclipse.sw360.rest.resourceserver.vendor.Sw360VendorService;
Expand Down Expand Up @@ -120,14 +126,19 @@ public class ReleaseSpecTest extends TestRestDocsSpecBase {

@MockBean
private Sw360LicenseService licenseServiceMock;
private Release release, release3;

@MockBean
private Sw360LicenseInfoService licenseInfoMockService;

private Release release, release3, releaseTest;
private Attachment attachment;
Component component;
private Project project;
private Map<String, ReleaseRelationship> releaseIdToRelationship1;

private final String releaseId = "3765276512";
private final String attachmentSha1 = "da373e491d3863477568896089ee9457bc316783";
private final String attachmentId = "11112222";

@Before
public void before() throws TException, IOException {
Expand Down Expand Up @@ -530,6 +541,11 @@ public void before() throws TException, IOException {
given(this.vulnerabilityServiceMock.getVulnerabilityDTOByExternalId(any(), any())).willReturn(vulUpdates);
given(this.vulnerabilityServiceMock.getVulnerabilitiesByReleaseId(any(), any())).willReturn(vulDtos);
given(this.vulnerabilityServiceMock.updateReleaseVulnerabilityRelation(any(), any())).willReturn(RequestStatus.SUCCESS);

releaseTest = new Release();
releaseTest.setId("12121212");
releaseTest.setName("Test Load SPDX");
releaseTest.setVersion("1.0");
}
@Test
public void should_document_get_releases() throws Exception {
Expand Down Expand Up @@ -1184,4 +1200,76 @@ public void should_document_write_spdx_licenses_info_into_release() throws Excep
subsectionWithPath("_links").description("<<resources-index-links,Links>> to other resources")
)));
}

@Test
public void should_document_load_spdx_licenses_info_from_isr() throws Exception {
mockLicensesInfo(AttachmentType.INITIAL_SCAN_REPORT);
String accessToken = TestHelper.getAccessToken(mockMvc, testUserId, testUserPassword);
mockMvc.perform(get("/api/releases/" + releaseTest.getId() + "/spdxLicensesInfo?attachmentId=" + attachmentId)
.header("Authorization", "Bearer " + accessToken)
.accept(MediaTypes.HAL_JSON))
.andExpect(status().isOk());
}

@Test
public void should_document_load_spdx_licenses_info_from_clx_or_cli() throws Exception {
mockLicensesInfo(AttachmentType.COMPONENT_LICENSE_INFO_COMBINED);
String accessToken = TestHelper.getAccessToken(mockMvc, testUserId, testUserPassword);
mockMvc.perform(get("/api/releases/" + releaseTest.getId() + "/spdxLicensesInfo?attachmentId=" + attachmentId)
.header("Authorization", "Bearer " + accessToken)
.accept(MediaTypes.HAL_JSON))
.andExpect(status().isOk());
}

public void mockLicensesInfo(AttachmentType attachmentType) throws TException{
Set<Attachment> listAttachment = new HashSet<>();
Attachment attachmentTest = new Attachment();

attachmentTest.setSha1(attachmentSha1);
attachmentTest.setAttachmentType(attachmentType);
attachmentTest.setCreatedOn("2016-12-18");
attachmentTest.setCreatedBy("[email protected]");
attachmentTest.setFilename("SPDX_filename.rdf");
attachmentTest.setAttachmentContentId(attachmentId);
listAttachment.add(attachmentTest);
listAttachment.add(attachment);
releaseTest.setAttachments(listAttachment);

List<LicenseInfoParsingResult> licenseInfoResults = new ArrayList<>();
LicenseInfoParsingResult licenseInfoParsingResult = new LicenseInfoParsingResult();
licenseInfoParsingResult.setStatus(LicenseInfoRequestStatus.SUCCESS);
licenseInfoParsingResult.setRelease(releaseTest);
LicenseInfo licenseInfo = new LicenseInfo();
licenseInfo.setFilenames(Collections.singletonList("SPDX_filename.rdf"));
LicenseNameWithText license1 = new LicenseNameWithText();
license1.setLicenseName("MIT");
license1.setLicenseText("MIT Text");
license1.setLicenseSpdxId("MIT");

LicenseNameWithText license2 = new LicenseNameWithText();
license2.setLicenseName("RSA-Security");
license2.setLicenseText("License by Nomos.");
license2.setLicenseSpdxId("RSA-Security");

if (AttachmentType.INITIAL_SCAN_REPORT.equals(attachmentType)) {
license1.setSourceFiles(Sets.newHashSet(
"test-3.2.tar.gz/test-3.2/sample",
"test-3.2.tar.gz/test-3.2/support/sys/cron",
"test-3.2.tar.gz/test-3.2/support/README")
);
license2.setSourceFiles(Sets.newHashSet(
"test-3.2.tar.gz/test-3.2/support/md5.h")
);
} else {
licenseInfo.setConcludedLicenseIds(Sets.newHashSet("GPL", "BSD-3-Clause"));
}

licenseInfo.setLicenseNamesWithTexts(Sets.newHashSet(license1, license2));

licenseInfoParsingResult.setLicenseInfo(licenseInfo);
licenseInfoResults.add(licenseInfoParsingResult);
boolean includeConcludedLicense = AttachmentType.INITIAL_SCAN_REPORT.equals(attachmentType);
given(this.releaseServiceMock.getReleaseForUserById(eq(releaseTest.getId()), any())).willReturn(releaseTest);
given(this.licenseInfoMockService.getLicenseInfoForAttachment(any(), any(), any(), eq(includeConcludedLicense))).willReturn(licenseInfoResults);
}
}

0 comments on commit 5dfae41

Please sign in to comment.