From 07f96d8c7079d8b2b2a17c4b399c2cba74cea959 Mon Sep 17 00:00:00 2001 From: dragove Date: Sun, 23 Jul 2023 22:20:40 +0800 Subject: [PATCH 1/2] =?UTF-8?q?feat:=20=E4=BD=BF=E7=94=A8=20Http=20Interfa?= =?UTF-8?q?ces=20API=20=E4=BB=A3=E6=9B=BF=20OpenFeign=20=E5=B9=B6=E5=88=A0?= =?UTF-8?q?=E9=99=A4=20Spring=20Cloud=20=E4=BE=9D=E8=B5=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- build.gradle | 59 ++++--- gradle/wrapper/gradle-wrapper.properties | 3 +- .../backend/config/HttpInterfaceConfig.java | 35 ++++ .../maa/backend/config/OpenFeignConfig.java | 12 -- .../backend/repository/GithubRepository.java | 30 +--- .../maa/backend/service/ArkLevelService.java | 152 +++--------------- .../repository/GithubRepositoryTest.java | 11 +- 7 files changed, 93 insertions(+), 209 deletions(-) create mode 100644 src/main/java/plus/maa/backend/config/HttpInterfaceConfig.java delete mode 100644 src/main/java/plus/maa/backend/config/OpenFeignConfig.java diff --git a/build.gradle b/build.gradle index 3755b6c9..b307f56e 100644 --- a/build.gradle +++ b/build.gradle @@ -1,9 +1,9 @@ plugins { id 'java' - id 'org.springframework.boot' version '3.0.6' - id "io.spring.dependency-management" version "1.1.0" - id "io.freefair.lombok" version "6.6" - id "org.springdoc.openapi-gradle-plugin" version "1.6.0" + id 'org.springframework.boot' version '3.1.2' + id 'io.spring.dependency-management' version '1.1.2' + id 'io.freefair.lombok' version '8.1.0' + id 'org.springdoc.openapi-gradle-plugin' version '1.6.0' id 'org.hidetake.swagger.generator' version '2.19.2' } @@ -25,36 +25,36 @@ ext { } dependencies { - implementation platform("org.springframework.cloud:spring-cloud-dependencies:${springCloudVersion}") - annotationProcessor "org.springframework.boot:spring-boot-configuration-processor" + annotationProcessor 'com.github.therapi:therapi-runtime-javadoc-scribe:0.13.0' annotationProcessor 'org.mapstruct:mapstruct-processor:1.5.3.Final' - annotationProcessor "com.github.therapi:therapi-runtime-javadoc-scribe:0.13.0" + annotationProcessor 'org.springframework.boot:spring-boot-configuration-processor' implementation 'org.springframework.boot:spring-boot-starter-test' implementation 'org.springframework.boot:spring-boot-starter-web' + implementation 'org.springframework.boot:spring-boot-starter-webflux' implementation 'org.springframework.boot:spring-boot-starter-security' implementation 'org.springframework.boot:spring-boot-starter-data-redis' implementation 'org.springframework.boot:spring-boot-starter-data-mongodb' - implementation 'org.springframework.cloud:spring-cloud-starter-openfeign' implementation 'org.springframework.boot:spring-boot-starter-validation' implementation 'org.springframework.boot:spring-boot-starter-cache' //springdoc相关依赖没有被自动管理，必须保留版本号 implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.1.0' implementation 'com.github.therapi:therapi-runtime-javadoc:0.13.0' + implementation 'org.aspectj:aspectjrt:1.9.19' implementation 'com.squareup.okhttp3:okhttp:4.10.0' implementation 'com.sun.mail:javax.mail:1.6.2' - implementation 'cn.hutool:hutool-all:5.8.11' + implementation 'cn.hutool:hutool-extra:5.8.20' + implementation 'cn.hutool:hutool-jwt:5.8.20' implementation 'org.mapstruct:mapstruct:1.5.3.Final' - implementation 'org.eclipse.jgit:org.eclipse.jgit:6.4.0.202211300538-r' - implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache:6.4.0.202211300538-r' + implementation 'org.eclipse.jgit:org.eclipse.jgit:6.6.0.202305301015-r' + implementation 'org.eclipse.jgit:org.eclipse.jgit.ssh.apache.agent:6.6.0.202305301015-r' implementation 'org.freemarker:freemarker:2.3.32' - implementation 'com.github.ben-manes.caffeine:caffeine:3.1.2' - implementation 'com.github.erosb:everit-json-schema:1.14.1' - implementation 'com.google.guava:guava:31.1-jre' + implementation 'com.github.ben-manes.caffeine:caffeine:3.1.6' + implementation 'com.github.erosb:everit-json-schema:1.14.2' + implementation 'com.google.guava:guava:32.1.1-jre' -// swaggerCodegen 'io.swagger.codegen.v3:swagger-codegen-cli:3.0.41' swaggerCodegen 'org.openapitools:openapi-generator-cli:6.5.0' } @@ -63,53 +63,48 @@ test { useJUnitPlatform() } -def swagger_output_dir = "$buildDir/docs" +def swagger_output_dir = '$buildDir/docs' def swagger_output_name = 'swagger.json' openApi { - apiDocsUrl.set("http://localhost:8848/v3/api-docs") + apiDocsUrl.set('http://localhost:8848/v3/api-docs') outputDir.set(file(swagger_output_dir)) outputFileName.set(swagger_output_name) waitTimeInSeconds.set(30) } swaggerSources { - def client_dir = "$buildDir/clients" TsFetch { - inputFile = file("$swagger_output_dir/$swagger_output_name") + inputFile = file('$swagger_output_dir/$swagger_output_name') code { language = 'typescript-fetch' configFile = file('client-config/ts-fetch.json') -// templateDir = file('client-config/typescript-fetch') - rawOptions = ["-e", "mustache"] - outputDir = file("$client_dir/ts-fetch-client") + rawOptions = ['-e', 'mustache'] + outputDir = file('$client_dir/ts-fetch-client') } } CSharp { - inputFile = file("$swagger_output_dir/$swagger_output_name") + inputFile = file('$swagger_output_dir/$swagger_output_name') code { language = 'csharp-netcore' configFile = file('client-config/csharp-netcore.json') - outputDir = file("$client_dir/csharp-client") -// rawOptions = [ -// "--type-mappings", "binary=System.IO.Stream" -// ] + outputDir = file('$client_dir/csharp-client') } } Cpp { - inputFile = file("$swagger_output_dir/$swagger_output_name") + inputFile = file('$swagger_output_dir/$swagger_output_name') code { language = 'cpp-restsdk' configFile = file('client-config/cpp.json') - outputDir = file("$client_dir/cpp-client") + outputDir = file('$client_dir/cpp-client') } } - Rust{ - inputFile = file("$swagger_output_dir/$swagger_output_name") + Rust { + inputFile = file('$swagger_output_dir/$swagger_output_name') code { language = 'rust' configFile = file('client-config/rust.json') - outputDir = file("$client_dir/rust-client") + outputDir = file('$client_dir/rust-client') } } } \ No newline at end of file diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 070cb702..17a8ddce 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,6 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.2.1-bin.zip +networkTimeout=10000 zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/plus/maa/backend/config/HttpInterfaceConfig.java b/src/main/java/plus/maa/backend/config/HttpInterfaceConfig.java new file mode 100644 index 00000000..2515e236 --- /dev/null +++ b/src/main/java/plus/maa/backend/config/HttpInterfaceConfig.java @@ -0,0 +1,35 @@ +package plus.maa.backend.config; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.web.reactive.function.client.ExchangeStrategies; +import org.springframework.web.reactive.function.client.WebClient; +import org.springframework.web.reactive.function.client.support.WebClientAdapter; +import org.springframework.web.service.invoker.HttpServiceProxyFactory; +import plus.maa.backend.repository.GithubRepository; + +@Configuration +public class HttpInterfaceConfig { + + @Bean + GithubRepository githubRepository() { + WebClient client = WebClient.builder() + .baseUrl("https://api.github.com") + .exchangeStrategies(ExchangeStrategies + .builder() + .codecs(codecs -> codecs + .defaultCodecs() + // 最大 20MB + .maxInMemorySize(20 * 1024 * 1024)) + .build()) + .defaultHeaders(headers -> { + headers.add("Accept", "application/vnd.github+json"); + headers.add("X-GitHub-Api-Version", "2022-11-28"); + }) + .build(); + return HttpServiceProxyFactory.builder(WebClientAdapter.forClient(client)) + .build() + .createClient(GithubRepository.class); + } + +} diff --git a/src/main/java/plus/maa/backend/config/OpenFeignConfig.java b/src/main/java/plus/maa/backend/config/OpenFeignConfig.java deleted file mode 100644 index 285aeaf9..00000000 --- a/src/main/java/plus/maa/backend/config/OpenFeignConfig.java +++ /dev/null @@ -1,12 +0,0 @@ -package plus.maa.backend.config; - -import org.springframework.cloud.openfeign.EnableFeignClients; -import org.springframework.context.annotation.Configuration; - -@EnableFeignClients(basePackages = { - "plus.maa.backend.repository" -}) -@Configuration -public class OpenFeignConfig { - -} diff --git a/src/main/java/plus/maa/backend/repository/GithubRepository.java b/src/main/java/plus/maa/backend/repository/GithubRepository.java index d14e4d70..69872aa4 100644 --- a/src/main/java/plus/maa/backend/repository/GithubRepository.java +++ b/src/main/java/plus/maa/backend/repository/GithubRepository.java @@ -1,9 +1,8 @@ package plus.maa.backend.repository; -import org.springframework.cloud.openfeign.FeignClient; -import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; -import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RequestHeader; +import org.springframework.web.service.annotation.GetExchange; import plus.maa.backend.repository.entity.github.GithubCommit; import plus.maa.backend.repository.entity.github.GithubTrees; @@ -13,30 +12,15 @@ * @author dragove * created on 2022/12/23 */ -@FeignClient(value = "githubRepository", url = "https://api.github.com") public interface GithubRepository { /** * api doc: git trees api */ - @GetMapping(value = "/repos/{repo}/git/trees/{sha}", - headers = { - "Accept: application/vnd.github+json", - "Authorization: {token}", - "X-GitHub-Api-Version: 2022-11-28" - }) - GithubTrees getTrees(@RequestParam("token") String token, - @PathVariable("repo") String repo, - @PathVariable("sha") String sha); + @GetExchange(value = "/repos/MaaAssistantArknights/MaaAssistantArknights/git/trees/{sha}") + GithubTrees getTrees(@RequestHeader("Authorization") String token, @PathVariable("sha") String sha); + + @GetExchange(value = "/repos/MaaAssistantArknights/MaaAssistantArknights/commits") + List getCommits(@RequestHeader("Authorization") String token); - @GetMapping(value = "/repos/{repo}/commits", - headers = { - "Accept: application/vnd.github+json", - "Authorization: {token}", - "X-GitHub-Api-Version: 2022-11-28" - }) - List getCommits(@RequestParam("token") String token, - @PathVariable("repo") String repo, - @RequestParam("path") String path, - @RequestParam("per_page") int prePage); } diff --git a/src/main/java/plus/maa/backend/service/ArkLevelService.java b/src/main/java/plus/maa/backend/service/ArkLevelService.java index 0e46c295..9d00a99d 100644 --- a/src/main/java/plus/maa/backend/service/ArkLevelService.java +++ b/src/main/java/plus/maa/backend/service/ArkLevelService.java @@ -1,35 +1,16 @@ package plus.maa.backend.service; -import java.io.File; -import java.io.IOException; -import java.net.URLEncoder; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.*; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import com.google.common.collect.Lists; -import org.eclipse.jgit.api.Git; -import org.eclipse.jgit.lib.Constants; -import org.eclipse.jgit.lib.ObjectInserter; -import org.eclipse.jgit.lib.Repository; -import org.eclipse.jgit.storage.file.FileRepositoryBuilder; +import com.fasterxml.jackson.databind.ObjectMapper; +import lombok.Data; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; +import okhttp3.*; import org.jetbrains.annotations.NotNull; import org.springframework.beans.factory.annotation.Value; import org.springframework.cache.annotation.Cacheable; import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Service; import org.springframework.util.CollectionUtils; - -import com.fasterxml.jackson.databind.ObjectMapper; - -import lombok.Data; -import lombok.RequiredArgsConstructor; -import lombok.extern.slf4j.Slf4j; -import okhttp3.*; import plus.maa.backend.common.utils.converter.ArkLevelConverter; import plus.maa.backend.controller.response.copilot.ArkLevelInfo; import plus.maa.backend.repository.ArkLevelRepository; @@ -42,6 +23,15 @@ import plus.maa.backend.repository.entity.github.GithubTree; import plus.maa.backend.repository.entity.github.GithubTrees; +import java.io.IOException; +import java.net.URLEncoder; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Consumer; +import java.util.stream.Collectors; + /** * @author dragove * created on 2022/12/23 @@ -66,12 +56,6 @@ public class ArkLevelService { @Value("${maa-copilot.github.repo.tile.path:resource/Arknights-Tile-Pos}") private String tilePosPath; - @Value("${maa-copilot.ark-level-git.repository:}") - private String gitRepository; - @Value("${maa-copilot.ark-level-git.local-repository:}") - private String localRepository; - @Value("${maa-copilot.ark-level-git.json-path:}") - private String jsonPath; private final GithubRepository githubRepo; private final RedisCache redisCache; private final ArkLevelRepository arkLevelRepo; @@ -80,7 +64,7 @@ public class ArkLevelService { private final ObjectMapper mapper = new ObjectMapper(); private final OkHttpClient okHttpClient; - private final List bypassFileNames = Lists.newArrayList("overview.json"); + private final List bypassFileNames = List.of("overview.json"); @Cacheable("arkLevels") public List getArkLevelInfos() { @@ -109,7 +93,7 @@ public List queryLevelByKeyword(String keyword) { public void runSyncLevelDataTask() { log.info("[LEVEL]开始同步地图数据"); //获取地图文件夹最新的commit, 用于判断是否需要更新 - List commits = githubRepo.getCommits(githubToken, maaRepo, tilePosPath, 1); + List commits = githubRepo.getCommits(githubToken); if (CollectionUtils.isEmpty(commits)) { log.info("[LEVEL]获取地图数据最新commit失败"); return; @@ -124,7 +108,7 @@ public void runSyncLevelDataTask() { //获取根目录文件列表 GithubTrees trees; List files = Arrays.stream(tilePosPath.split("/")).toList(); - trees = githubRepo.getTrees(githubToken, maaRepo, commit.getSha()); + trees = githubRepo.getTrees(githubToken, commit.getSha()); //根据路径获取文件列表 for (String file : files) { if (trees == null || CollectionUtils.isEmpty(trees.getTree())) { @@ -139,7 +123,7 @@ public void runSyncLevelDataTask() { log.info("[LEVEL]地图数据获取失败, 未找到文件夹{}", file); return; } - trees = githubRepo.getTrees(githubToken, maaRepo, tree.getSha()); + trees = githubRepo.getTrees(githubToken, tree.getSha()); } if (trees == null || CollectionUtils.isEmpty(trees.getTree())) { log.info("[LEVEL]地图数据获取失败, 未找到文件夹{}", tilePosPath); @@ -264,104 +248,4 @@ private void checkFinish() { } } - @Async - public void runSyncLevelDataTaskForJGit() { - log.info("[LEVEL]开始同步地图数据"); - Git git = getGitRepository(); - if (git == null) { - return; - } - String lastCommit; - try { - lastCommit = git.log().setMaxCount(1).call().iterator().next().getName(); - } catch (Exception exception) { - exception.printStackTrace(); - log.warn("[LEVEL]获取最新commit失败"); - return; - } - String cacheCommit = redisCache.getCacheLevelCommit(); - if (Objects.equals(cacheCommit, lastCommit)) { - log.info("[LEVEL]地图数据已是最新"); - return; - } - if (!gitCheckout(git, lastCommit)) { - return; - } - - File file = new File(Path.of(localRepository, jsonPath).toUri()); - if (!file.exists()) { - log.warn("[LEVEL]地图数据获取失败, 未找到文件夹{}", jsonPath); - return; - } - List jsons = Arrays.stream(Objects.requireNonNull(file.listFiles())).filter(f -> f.isFile() && f.getName().endsWith(".json")).toList(); - log.info("[LEVEL]已发现{}份地图数据", jsons.size()); - - List shaList = arkLevelRepo.findAllShaBy().stream().map(ArkLevelSha::getSha).toList(); - try (ObjectInserter inserter = git.getRepository().newObjectInserter()) { - - int errorCount = 0; - int handleCount = 0; - for (File ignored : jsons) { - try { - byte[] bytes = Files.readAllBytes(file.toPath()); - String sha = inserter.idFor(Constants.OBJ_BLOB, bytes).name(); - if (shaList.contains(sha)) { - continue; - } - ArkTilePos tilePos = mapper.readValue(bytes, ArkTilePos.class); - ArkLevel level = parserService.parseLevel(tilePos, sha); - if (level == null) { - errorCount++; - log.info("[LEVEL]地图数据解析失败:" + file.getPath()); - continue; - } - arkLevelRepo.save(level); - handleCount++; - } catch (IOException e) { - errorCount++; - e.printStackTrace(); - } - } - log.info("[LEVEL]{}份地图数据更新成功", handleCount); - if (errorCount > 0) { - log.warn("[LEVEL]{}份地图数据更新失败", errorCount); - return; - } - redisCache.setCacheLevelCommit(lastCommit); - } - } - - private Git getGitRepository() { - if (Files.exists(Path.of(localRepository, ".git"))) { - try { - Repository repository = new FileRepositoryBuilder().setGitDir(Path.of(localRepository, ".git").toFile()).build(); - return new Git(repository); - } catch (Exception exception) { - exception.printStackTrace(); - log.warn("[LEVEL]地图数据pull失败"); - return null; - } - } else { - try { - //TODO:需要超时机制 - return Git.cloneRepository().setURI(gitRepository).setDirectory(new File(localRepository)).setNoCheckout(true).call(); - } catch (Exception exception) { - exception.printStackTrace(); - log.warn("[LEVEL]地图数据拉取失败失败"); - return null; - } - } - } - - private boolean gitCheckout(Git git, String lastCommit) { - try { - git.pull().call(); - git.checkout().setStartPoint(lastCommit).addPath(jsonPath).call(); - } catch (Exception exception) { - exception.printStackTrace(); - log.warn("[LEVEL]地图数据checkout失败"); - return false; - } - return true; - } } diff --git a/src/test/java/plus/maa/backend/repository/GithubRepositoryTest.java b/src/test/java/plus/maa/backend/repository/GithubRepositoryTest.java index 26659da8..cc4a3989 100644 --- a/src/test/java/plus/maa/backend/repository/GithubRepositoryTest.java +++ b/src/test/java/plus/maa/backend/repository/GithubRepositoryTest.java @@ -5,8 +5,7 @@ import org.springframework.boot.test.context.SpringBootTest; import plus.maa.backend.config.external.MaaCopilotProperties; -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.*; @SpringBootTest class GithubRepositoryTest { @@ -16,19 +15,17 @@ class GithubRepositoryTest { @Autowired private MaaCopilotProperties properties; - static final String MAA_REPO = "MaaAssistantArknights/MaaAssistantArknights"; - @Test public void testGetTrees() { - var maaTree = repository.getTrees(properties.getGithub().getToken(), MAA_REPO, "d989739981db071e80df1c66e473c729b50e8073"); + var maaTree = repository.getTrees(properties.getGithub().getToken(), "d989739981db071e80df1c66e473c729b50e8073"); assertNotNull(maaTree); } @Test public void testGetCommits() { - var commits = repository.getCommits(properties.getGithub().getToken(), MAA_REPO, "/src/Cpp", 10); + var commits = repository.getCommits(properties.getGithub().getToken()); assertNotNull(commits); - assertTrue(commits.size() > 0 && commits.size() < 10); + assertFalse(commits.isEmpty()); } } \ No newline at end of file From 9256a80756b00d7e2f128496cd414cba17aeb731 Mon Sep 17 00:00:00 2001 From: dragove Date: Sun, 23 Jul 2023 22:25:32 +0800 Subject: [PATCH 2/2] fix: aspectjrt -> aspectjweaver --- build.gradle | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle b/build.gradle index b307f56e..0b736905 100644 --- a/build.gradle +++ b/build.gradle @@ -42,7 +42,6 @@ dependencies { implementation 'org.springdoc:springdoc-openapi-starter-webmvc-ui:2.1.0' implementation 'com.github.therapi:therapi-runtime-javadoc:0.13.0' - implementation 'org.aspectj:aspectjrt:1.9.19' implementation 'com.squareup.okhttp3:okhttp:4.10.0' implementation 'com.sun.mail:javax.mail:1.6.2' implementation 'cn.hutool:hutool-extra:5.8.20' @@ -54,6 +53,7 @@ dependencies { implementation 'com.github.ben-manes.caffeine:caffeine:3.1.6' implementation 'com.github.erosb:everit-json-schema:1.14.2' implementation 'com.google.guava:guava:32.1.1-jre' + implementation 'org.aspectj:aspectjweaver:1.9.19' swaggerCodegen 'org.openapitools:openapi-generator-cli:6.5.0'