From 2beae9f2f406edb01ef41ddb73c3614927bc44aa Mon Sep 17 00:00:00 2001 From: HeYQ Date: Sun, 8 Dec 2024 22:55:58 +0800 Subject: [PATCH] update: Document parser --- .../document-parser-apache-pdfbox/pom.xml | 76 +++++ .../pdfbox/ApachePdfBoxDocumentParser.java | 60 ++++ .../ApachePdfBoxDocumentParserTest.java | 45 +++ .../src/test/resources/blank-file.pdf | Bin 0 -> 4911 bytes .../src/test/resources/test-file.pdf | Bin 0 -> 22958 bytes .../document-parser-markdown/pom.xml | 71 +++++ .../markdown/MarkdownDocumentParser.java | 206 ++++++++++++++ .../config/MarkdownDocumentParserConfig.java | 122 ++++++++ .../markdown/MarkdownDocumentParserTest.java | 263 ++++++++++++++++++ .../src/test/resources/blockquote.md | 8 + .../src/test/resources/code.md | 25 ++ .../src/test/resources/horizontal-rules.md | 27 ++ .../src/test/resources/lists.md | 17 ++ .../src/test/resources/only-headers.md | 20 ++ .../src/test/resources/simple.md | 8 + .../src/test/resources/with-formatting.md | 9 + .../ai/parser/tika/TikaDocumentParser.java | 17 +- .../tika/ApacheTikaDocumentParserTest.java | 8 +- .../reader/github/GitHubDocumentReader.java | 28 +- .../tencent/cos/TencentCosDocumentReader.java | 8 +- .../cos/TencentCosDocumentLoaderIT.java | 5 + .../cloud/ai/document/DocumentParser.java | 3 +- .../cloud/ai/document/JsonDocumentParser.java | 112 ++++++++ .../cloud/ai/document/TextDocumentParser.java | 10 +- .../dashscope/rag/AnalyticdbVectorTest.java | 30 +- 25 files changed, 1134 insertions(+), 44 deletions(-) create mode 100644 community/document-parsers/document-parser-apache-pdfbox/pom.xml create mode 100644 community/document-parsers/document-parser-apache-pdfbox/src/main/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParser.java create mode 100644 community/document-parsers/document-parser-apache-pdfbox/src/test/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParserTest.java create mode 100644 community/document-parsers/document-parser-apache-pdfbox/src/test/resources/blank-file.pdf create mode 100644 community/document-parsers/document-parser-apache-pdfbox/src/test/resources/test-file.pdf create mode 100644 community/document-parsers/document-parser-markdown/pom.xml create mode 100644 community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParser.java create mode 100644 community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/config/MarkdownDocumentParserConfig.java create mode 100644 community/document-parsers/document-parser-markdown/src/test/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParserTest.java create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/blockquote.md create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/code.md create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/horizontal-rules.md create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/lists.md create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/only-headers.md create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/simple.md create mode 100644 community/document-parsers/document-parser-markdown/src/test/resources/with-formatting.md create mode 100644 spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/JsonDocumentParser.java diff --git a/community/document-parsers/document-parser-apache-pdfbox/pom.xml b/community/document-parsers/document-parser-apache-pdfbox/pom.xml new file mode 100644 index 00000000..7a1a0b9d --- /dev/null +++ b/community/document-parsers/document-parser-apache-pdfbox/pom.xml @@ -0,0 +1,76 @@ + + + 4.0.0 + + com.alibaba.cloud.ai + spring-ai-alibaba + ${revision} + ../../../pom.xml + + + document-parser-apache-pdfbox + document-parser-apache-pdfbox + document-parser-apache-pdfbox for Spring AI Alibaba + jar + https://github.com/alibaba/spring-ai-alibaba + + https://github.com/alibaba/spring-ai-alibaba + git://github.com/alibaba/spring-ai-alibaba.git + git@github.com:alibaba/spring-ai-alibaba.git + + + + 17 + 17 + UTF-8 + 2.0.32 + + + + + com.alibaba.cloud.ai + spring-ai-alibaba-core + ${project.parent.version} + + + + org.apache.pdfbox + pdfbox + ${pdfbox.version} + + + commons-logging + commons-logging + + + + + + + org.springframework.ai + spring-ai-test + test + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.assertj + assertj-core + test + + + + \ No newline at end of file diff --git a/community/document-parsers/document-parser-apache-pdfbox/src/main/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParser.java b/community/document-parsers/document-parser-apache-pdfbox/src/main/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParser.java new file mode 100644 index 00000000..a8b6f257 --- /dev/null +++ b/community/document-parsers/document-parser-apache-pdfbox/src/main/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParser.java @@ -0,0 +1,60 @@ +package com.alibaba.cloud.ai.parser.apache.pdfbox; + +import com.alibaba.cloud.ai.document.DocumentParser; +import org.apache.pdfbox.pdmodel.PDDocument; +import org.apache.pdfbox.pdmodel.PDDocumentInformation; +import org.apache.pdfbox.text.PDFTextStripper; +import org.springframework.ai.document.Document; +import org.springframework.util.Assert; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * @author HeYQ + * @since 2024-12-08 22:34 + */ + +public class ApachePdfBoxDocumentParser implements DocumentParser { + + private final boolean includeMetadata; + + public ApachePdfBoxDocumentParser() { + this(false); + } + + public ApachePdfBoxDocumentParser(boolean includeMetadata) { + this.includeMetadata = includeMetadata; + } + + @Override + public List parse(InputStream inputStream) { + try (PDDocument pdfDocument = PDDocument.load(inputStream)) { + PDFTextStripper stripper = new PDFTextStripper(); + String text = stripper.getText(pdfDocument); + Assert.notNull(text, "Text cannot be null"); + return includeMetadata ? Collections.singletonList(new Document(text, toMetadata(pdfDocument))) + : Collections.singletonList(new Document(text)); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Map toMetadata(PDDocument pdDocument) { + PDDocumentInformation documentInformation = pdDocument.getDocumentInformation(); + Map metadata = new HashMap<>(); + for (String metadataKey : documentInformation.getMetadataKeys()) { + String value = documentInformation.getCustomMetadataValue(metadataKey); + if (value != null) { + metadata.put(metadataKey, value); + } + } + return metadata; + } + +} diff --git a/community/document-parsers/document-parser-apache-pdfbox/src/test/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParserTest.java b/community/document-parsers/document-parser-apache-pdfbox/src/test/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParserTest.java new file mode 100644 index 00000000..ec7f4e4a --- /dev/null +++ b/community/document-parsers/document-parser-apache-pdfbox/src/test/java/com/alibaba/cloud/ai/parser/apache/pdfbox/ApachePdfBoxDocumentParserTest.java @@ -0,0 +1,45 @@ +package com.alibaba.cloud.ai.parser.apache.pdfbox; + +import com.alibaba.cloud.ai.document.DocumentParser; +import org.junit.jupiter.api.Test; +import org.springframework.ai.document.Document; + +import java.io.IOException; +import java.io.InputStream; + +import static org.assertj.core.api.Assertions.assertThat; + +class ApachePdfBoxDocumentParserTest { + + @Test + void should_parse_pdf_file() { + try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("test-file.pdf")) { + DocumentParser parser = new ApachePdfBoxDocumentParser(); + Document document = parser.parse(inputStream).get(0); + + assertThat(document.getContent()).isEqualToIgnoringWhitespace("test content"); + assertThat(document.getMetadata()).isEmpty(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + @Test + void should_parse_pdf_file_include_metadata() { + try (InputStream inputStream = getClass().getClassLoader().getResourceAsStream("test-file.pdf")) { + DocumentParser parser = new ApachePdfBoxDocumentParser(true); + Document document = parser.parse(inputStream).get(0); + + assertThat(document.getContent()).isEqualToIgnoringWhitespace("test content"); + assertThat(document.getMetadata()).containsEntry("Author", "ljuba") + .containsEntry("Creator", "WPS Writer") + .containsEntry("CreationDate", "D:20230608171011+15'10'") + .containsEntry("SourceModified", "D:20230608171011+15'10'"); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + +} \ No newline at end of file diff --git a/community/document-parsers/document-parser-apache-pdfbox/src/test/resources/blank-file.pdf b/community/document-parsers/document-parser-apache-pdfbox/src/test/resources/blank-file.pdf new file mode 100644 index 0000000000000000000000000000000000000000..757bb1f36404ff475efeb6fda3350468de425004 GIT binary patch literal 4911 zcmeHLe{2)i9XCJ9kXvt2mjY9@?&LNdEIr@5cX$4AvrUq7+=Rr$*iixkCeC;7k}GGQ zxjQFLTE;>IZRpl^i&V8+D^*duij``EwSTSch#*r}3D`unF|A58CN>q(GNE*6bcnsP zoj7)l#iah3lP$gbzW4pS&-e4*_u1dq6putn+T(RKUHtpG^DZZ32|(zn$6Uc66;(A0 z8B|mgEfhhLE+cF*Op9QtZj>3Y1_?-esjUP}^^Rl_)suQGX{Ezq(=w2lb~y)UgU@d3 zIt3aJ-TGkv&ZjTTzWNaVjd8WH@id%y_exV&hI_&LlZSt{d9H8f&*s#G;2)-i2PT=-pI`j<)7@J??fBP+Ig{Oe_4CistH+5q6RP)Tf!xKmP-#PZJ z@taSAiS^w;e(BfDJO4fO7JWe*oO)rySNSnx@|V{eTl@Swu70%Vl={-!-`jPqch}>g zCtBY4wB@eJf4+EW^d@=I@x(9U$6QX7kxN11mMs?K{vXxYKt0h;#6kR5$NxBdmO9hC z4Sab0Z1<*TyzENnnwiT__O5&N@>d@E`RnWc`j-`}H#p{Q-2G-PHr_j66IfV77GgH= zc1sA8AP)M$TQ;TZ&1cAG*6edN9!!05bHP*I!!iztkVO-!Y;bjynJGeDwH@_l|CQabm;Nii!4? zm!0eO>~B3f^Pkt+e}8nQg;{mv+T?h$YwELAyFQv163tt1AoJtby6){t+;`8s@SVF8 zW25JjpImXQyvK3jm2W>^E2?r(r6gcD$Wh5$%CeJSd#t-CWei!OELvhYAA+0nSy9@H zEFy&l)J({I^*6tD6RI3?_j552%L-^v?HDysa&&92G`d#`$nMQyXE4v_)7dmeBl2l2 zWAgcsTg00V@wjc9DK}9VV(ksNTkQ^Fdn`@}x`7DJ!;%sJ9O3hOXx7I-?;`{RG^Bu^ zf(%JBJg`4R>2ZghxD_;Hg-=AHrRnfn$USITS)QVXhlf4Gj7K*HC^`@bPykX8l6VAZ zj$|w`PiD*qiy#X)5oAh+nzd9tL)f@tO3zs#x7(g*@hQfYT{2O|^b}(BNP3#W9Te?h zs9H`~nniXMPI)nloxts!VzK&e>2ys`Sz2_<<_xW%ge*}=Lur(;Ogtk^Eee3+3u5kk z+|cEmgp6>jWauf;A|k43shWli!s`LV#(otaBE#mUpdD#>bXlSxj*=Jfp^A@T{0&0@ zyd+>q;O(V39(Z{Mw{Zgmsj88+Fm$6=*R^nAtHhI>N7gi=4;iMtM4ShD;1ZP5*eaNq zF00B&i3*LW_(+=Tr2$Vv9@5Rgzer`t$ZCvIdfXAp(y^5|u^1oGrJSvPXC$1H7uT&Wve^83`@mGF*V+K#Czng#{!l z`$aM!dnLROwBMUz6(z-Z7jT#OF2R*0UeS%TXobaWR#PR>P78a{gH*Yza=fKl8Y-ma zVzw0WvsJSyt@dWK6HHU>yyZ19GZ1p;Nf{|(PP5$MHf-5T;2(QpL9ZZH9AAl3v{u3w z=v6ZbV#<=eCv~F=!G=wueU;WSzia}jL;QgOAaT4gouspc_ig5bqEnXieg)E zXJ&fLTS8TY> z54neNJyxS4r0H@=xKQ;G5Ze+|TpU2%VhU*{Vb|nEgGu-c?ZTyrfJ;P89|(tC3s(tD z8Lh?zOsVFiLAkE%m{_?MIPkNg203WYRl3h9-r>YWBW>c_9YNP}>5PdlbK(Y)RSEx* zBWM^7*ShxUjO+|X+I&7ACkX8j`l8-81_&$*B2gAXIwCLu5DvCQ1wn{1Q4aUh4BOfk zVFbXo2HIJnl?LAS@NTLzqv%AS5Pm5VKcEu2ZV?3)6I4q)`1kX{;u_ zyBzS~Y%1kg`7oG%Uwi-L{H?1a8}W?}-7&V~hbWW0|W^gXhI02fk5K~cY@Qn(^x}j+}$O(yF0<%-4cSk6M_b4+;zyi z_cwc=Yv!7Behk0Xs#;a6s_wh$X@gc#Oo9pYkqeWyr*?4uB_;^K0CMIjClm^D;D z;H8O`@n3gr|8Xnq=wN1T2cU<2aWsT{cs^@wX=!5hd;$HxHZ~9|*MF_;Odt;C)>dK= z2NM9j*e6yNRyGz+7H$w1hy?^<0CCWPSm^%R{5|*Aqq?FpK;6#V!Nl%aDr4g6Y;9-! zFSl~m#{WAnigwn^ahB zdlfr~jm@(mvjhZYZ}Oiqfd0Dvv4T370YE%|lfkric!tjMC8kSyMpQRu4-0nC zk;p1Fw7aZ&f6)hhXj|ww`?bdlCE?^M*W(MzMi75UNI>*9jrUVNssxcCewa)Oaum@t zUdk)cmf#T#9HTbhwS*ll!)gRI{3yWGkw zaO&@HwzTtH`O_@E&{Xvu5YRh&yHZN=3M_3&{^sX<#wC)Q{c9Zx5&74Ap{eYn^1$;+ zoaA^-KBD+Jmv6}BkLtvvgDrZVG79jw4a%YMwO`bPhf3ayNh}hVVeO|l%-4P_OLe4u z!{hvG`J`{L9OzW0GhTI7GF!c2wO&o&cqUNcPw8`6W!0tO(H4_N(KmhjfG8Fj!tnnB zFzD|%{C8ay?2Jw9{-QLtQpRN9H8aygY}u=q7WM?6LYAU1Avq5c`E*^VH^NVY@p|Wp9c%n z9>B)_uS7-unR!ecEUf=hgIL*F{_*+TPJgqMfLNNtp3(Ss_dVnBkJKFYocpu4e=74= z^7p9+_w`58|&0LSy{U*rC-zyHFES>xZivvaclSLOc3`hU{UENtx0CHvRF z1p&C&ICwb!*A73(Kyy*)zq%T;Y)xAg6#)d&<{+cD%e7oLLam$97@=a z{yNkr#4k73#|jbC>0qAWQctN{&-cKylg*wECZfprwBzYDR_m)Un$pZ>58amMVY7D@ z#e9V_jyXy1v3>BB&6=No3z92?ipTM2q;=AhQvT7hgOJ(_?S~25dqReFK|UFWR(W=Q zm(#jtthN0!EwUJVv__-T&9D8`cLY?D^K-&R+%j#@%T9y`)NxzzLhfV|#RVmR_I*0N z&6hvJpQjQ%vmM{2#>f)J3P2v0H}6b4W_-=m_1|s?8@kqBv@CkL?lN!mv^?Km10r#NGj0wiS%!quy#PLfr|>xhj!(NrVUsp5j^4CS>%ATr=l8 z-s0GOpfgLd>lqFX4?e(3eZfV7Y1%D_*!KscP6jTVf5|duJ_fMN_UdcadFYw!lw6IR z_?Y(ckG}Q@XI1om!7q8>gUlf@AOj5lyo|L=G_D{*Y;4bye2X!x-&Hwti1;O&C|Qqs zf^u$3?iZ?z6~Yi!-$2JG-&3)guj2hG0%$Z=Dry8 zsB8v850AIH!v@y1;}RnDy#BM_r81rwT0h0UEusij`UZECcBIHt@nk-SC$6@?_P2oC zEN##I4%O0lIF6sQ~K8aq2P;xV({W|P3BL|_V#A|Bg@B}Heri+ zuHiq?kmx`F=`iYz0scKr7bhIOe1gEd{IuS6*wD`j>z+x2WL(Fx%cR0Cm8@8`ZbYr; zP}N^pXBBq%tA;I8QVun1V@}t=JyVc3MLy=1(UKt9TS$HmY!>nY&e>lUYqjwaVREtX z-sjj^DfAZE;8a+8sxzC=b2ZSnMZS6`*R#_T0?111QcY~b#zt1xxP7l*_h$M#x9poQ zc}!i&4g?&Zf=yKnAUic|BeS6;V%i}MEm1YoV=tu zAf?Th>7p2|58FESHG6%hGT(S{#cNNHtn$+uEjc&YMjzJ-)xGISOfjEB!mYE58BO;0 zBX2irImYrs*YqBH;+JllU|;mZM%?8by|8QTZu!G2qHD3hh#UdbL*aSJ?!hXarVCqB zdb7MXbYA2{!X)O?8f@ciSSdAe>Cfi$%6o~@8O}*PP^AH92q*U0{+B%<4E<;dCQoqb zcO}8eh2`F@jj?1v--yJiY{avc7a9DOXE}V!mc`n_9g-H}N^(}<{3))1ewpB`Bx0A0 zCTf7pbyaXP{rk4a1$&lJ`iwp4fSSeXVgXe?7-XFTqZ?lL5V%c#yV;hfD{oY~6B}sw zI4+^|F$>pYzzT_@f54F@>m&3GtB9iEb#I{8;%m4q(ssuzx--Kfy<(IhHi?f#cqU2I z>sUEUnY8JKZHy|CfD_v-z>qp-Gm52dfp2P12hCsr>z8kmlOBD{U1wC~{c(k@f5^fL z0JFPNR(a&jN{3P#IE%)xZ2TZM6z@deV!F#R-fyUuC552H_RmwK_pUhk5BIL4b;(iC z=>T@BrRsT3(kqulyO*Z=ni`h|l(iS%;N=RRYcmk7Fl)F)}dn!n44&-k;9vL*aEkeX=ip>D~U@3rZqDbQ`#mD%gh&s z;lo&`KAgIRH&xASijhJSdgrBhP|Ti;TysvkgA7>Hpc20k(S4-<*sdYmeV*AeCoOqx zP$^abAsfvQN4-gK+q<1>FR;Gch-5CSihE!iz((by$Iq#7J-RN{i3ao2@g!MXWtms$ zWhwf~W5by?SLZ&l>O(17&6$uJ{()Wo$ltZBghRa^VdUoCUbB9rdppGi)7UX_H^VW?p{yFiwKgIoOq8wzMD9Sd=X&nQ;Nxrh-`woJCF* z=D}W@l{T_Y!=qJ0gDWe!4C< zdIbL_El2qS*7EwZ*UnB%5d)429wCl;H27Rj`C%_Oncz@usGe;|*9NV71)Sl}gKP#h zP(XCYT|B_Le;gkj&zteIA}dD-n`x$>(gCWdPoc9YI6`zByE_G4yYkNhMYXv6PzPjtnwWRr8>|h#knULL?0fuBk z(n_eZy^gJMJm0;m%{a|jL%Z$e$F?Ked;7-C9Zokq!Chwe;pP)E48V-{Tf z8++hNVPb%$IH%+5K*<-+W4I+QZo^yBD#X&Y9MQ2Rkif<<3y_k&2Yhj803v`C5ni7LN5& z9kTsJDc&R;DOo1MEfIzi2!F1iJV#Qn@H_aSPW(uKI(=W&^3|jpm;Ypzk0!WbHJSM zBOy?uB4|rl#*aW#n7Yrz6ZIJ)nJUQQDhxkV0&K_*mV~uQ>l^H4 zx1#LSIf^+l%QUZfLBAU6?7LFWd*q)n(5^X|xydi*y})ZpfBKHqZ>U$s`-}}GXnk}l#psOTl_b6g)y12y!-J?_l*tf)rv znd8mf7uZS1Ik1vWsq42rK3kHW^O}HEc(;)yUc5A*=fyKcyR!IMeb0eqlHRWvy1^%3 zoh~^pfw1Qt!d!gUz9?!&ccFw9t^wx0u*iwt!)iq(Qj>{kEz6PGT=7_ z`&EbE;dZ%bndTc>UNSD;d!48y{vLWT zER(#Dwa`y4=U>RQX11UrwxCY@xc&Gevb}EXmFOJ|Rz8JhPKh|N+emU$EwDEd)W%oUhH+4l~z-;YL%=t z1~)pmzO{ajtRF=#v)wbIQ24b4ffD#x3hcb%=;Bw`E<=Xj@B}zM0>;Q0Q(3>-IU?6? zY9h{?XV5vk5YLV;=c3(7r=137@zEXg&}b;`hi8wi@lYA;yy$=;z+P&5q8_97s~p|c zws;ILM>>KLTdBNX*V?Qp95dFi6v2b}9dPdp!2}jh3UfFb?#<;3t7SLk<&E>?#_nie z6u#uX=_6r%v0qN9()Ca+-EF*XjnSvX%qLz`O|j?WC@|ZH?U#^(D7L!1oO(tA#j>jlvAXa8ft?Ejd*FU&fnw7Lz;lFP*PGnpl&_ASf?E!oFir@zAd zlC<>c?u_SZAEAQZ{ctd+7V4GGjc3h2C7zg0H+JbUc6FVS$v2lK666>;_|uOa(Uqk1 z(6v3P|MvH1sZH@tVn_$U-DF$&4&;6w)!V%Bs`{8;z0`87vhJ8)Ic~>{*e27S8p^%f zI+wxEwc+lyQU`|)3JMUvph2*3wB1fFb$+qJa(O{$#0W*NfM>sho6O_EuUq89l}V^r z1|IsZy67N`x07wU5B@jnws9zbE@iI|Q3Ir&#hAPr-s<5!LW3Cvpy)zK6j!E{yxDN4nb1KM^f#DA5V#cx7c1 z8iDpck&GRD49M{N^cbn-lR|$4^2~OXbw@IiPQLlDy?EmGZqgΝ=4^A#qpYcd={I zO;Zb4LOHae|902?%gB|LCjW?dyCB+95M1G?J9$1n*K#ZmJI-@^k6&tocDOQJb{?QS zK}QF!IAPyfl&-3{@rkv}DURx%mig(!-?NR!CwU>+*UQ0=i*HUsT8C+0+%ub8)%&%a zZD#6wr4ID4JNUNEp2S9GkqUUga}O$t73tI=6yFY5HsECWp~ zSojlbxFx~jCDJzhR@7zNcN^Y7K0J2eUxh>^u||ct=l>>otp|{n619HgJT*^(N#5P? z8cy{foZ;8NP1=T)Sq1B$u2MCS>5^xfgyel#o9b`aTGk)EQm&^>o>ItZ&welrH!pMy zhC5@xk{Ue@2uh=B1b!}JXZbJXD0GOY)!oF0)#wb5$rbraSNXVA*D4!S3J9+FaBOGX zQ#g{%38JO_((}9N0BtY7${npot{X&+rl%^kE~nW`&nS^+TLTMhT*$kaGTB&uf<`!2 zv|ucwn)a$0js0Oz+To+zqssqA)`V|f%eL^7{xgH9>;prdJq6`(N^+ z$}sNTL^Gqp7Xf^6CH21Ep5N_#yqUkZ6lX-81@2$a-4y>6e0fUoXesdcPM2sv^***F26U2_A-peKp_wrR`QEsnU z6L`#Lhol<@QDt+6wKGHZZFe!_sQfun4;_;Yx_#@iH?_|1S2^VnZn+GggB^GoA55l~ zwF%7aM|Ym=nex2}svG;S9# zdR2O^uNY5@3gou^JWp$L?pcupjEaWyQCfdx>T4aJ-)^s6e6!vz))b?b>73<)wl&tf zm}g$bzgV|$bn&se^S7=URqs9H73hd|Soh)ayT0-@SI=w+-I_RpKNHWi-<|eR5$uJ! zzns9~O93kh-zYEP^$9_u`f^uN4U~aixx_RjtF#Lv?6-WO&w-dF!B>aTpwrlawexTRP+BV_@&&u3eD$PGD=4O$g zqa})8$x;v*tM$dczc{`Tp0_?@bf>mP=!KQSW<9uVuyE}Q{6V1rnI^gCTGIjrUh5nr zOYGFY;QXrEDmAM9vb2h-qNw?xoxN)9Iw6&H-reTlf!mYfcTlXFr<-_e zEzkf8#@^3pNhUj*xye)xFG83(8U&e6FX#p2Vpn(z+A(t|ok(X#a4mz7!A{nQyf@c*CZbiCGODneF1_ zA|hQ1{B&pCx@n~-z|BwuEZ|r_sooyk*1&6FFUB_O|T1%QQa$}JFU&$T2X&^e!*+f zPMq9*88CK8S371>bxu0V8%nl@)Zyo3x5*naJbEAAY44J#j7}BdUXH&tv9EcTI9SqI z{hDwb24_l*a4MVs9TQxy(Fx7LW-zM}AiMD^7!$ytEpX#9$WpfJWyU=fu8QywI5+M< zOWn`Wz(P15vzTw@(vGvZsZ*<%xmYpizz?w~n&Hui^Oo`7+DS0T??Ti6WMyb=I8e$9od)k4af1KC(J;3{2_3NiiieIscHd%SVY) zG37(t@NTsfQgaH{U8}E*n5sB=4Q=D?i<%>Iyv0y1vp$SJVu&v_<|-soSCaUL1HQmo z-<;An!&7!~$H18GotlEOFDdd|*Nz5yST%0KOQye@jjdJ6vK$8K2xBw=&P#*|f3Dul z=y7CKLG$F+gaR9$-?QOm*4r*YQcd|Hclg5JPK-IPcks{?q&<l3b;bFy?O6aLUi{RK`||J?ht z>Qd7#0Un@G^4UB=G4y=(+pPftiLApSmi#);_{~<9(gmxrMYs8x&gl0s+JnY-#nM$~ z`eARwcwU=Xs&0I#*0C`q-o!pR97#FqikjVdN8+q`#ZbRY`g)jubm$B>dJ5p}mu{j2 z@kHG~3&U1IRZ}-S>TNm1(h{0^`yCfqwb_r&9hq-s2^lI7NMQg8EY(7N!W|nBw6DKN z!Pe*WVR}u&SRuZg@S2TDUh{j4ma%Ppze8BRA6pTM>Xp_(butVxBje~P739wzX8Kz0 z%S6umjz4E^->Y|xa%KbOEK4qKNcEjG)5kiMnT*q#KG%Ky=JeaEe&ej++il%~)*luA zcijG~p4+d5XBaqm*b=%gNvh;s2b^>dlGju3ddZK7Xjk23fr0f?JoSB(Ps6Rqpex8HG-9H=w6*v;!?9`(#_Ti03zaI3C_G zl#%VMR>K|9gEy&?#pK)lftc>jYq84$1L>wZ7)$PC1;@@R6%xymAm@a)RB z6+FDACU7t9xg(s!lxdb2$t=AoYse!S%KIN`#}bmc7N9g|27+Q=8k6-;#sF)6&B`rE zW?3!B_PZpZDyuBs9H_}z**bJAbPD4~c|AVJsV{h2@GOF*+S_k9X7Mw^)qc5#HZbL@ zJdzsmDY&z2@2}jKndZzQT(xEdnX6W*4h3dl^A@5B8W}@F8+$7(k;z~E`TUg+J{21i zMX@(}gJ+WYd7?@|l5P5AdL>yzTno{%mgZ^vCZ!1B?&Xr(*QQ*>x$P{tDD;hPX*P{Y`=|rP=D?RFJ7jb;c4A*5 z*1-v6|0L?<7k_AQ#&WnL^zcdXeeeWDgO+4UBuyc=GklX zwbV}a8BC`e`uP|c`ZVtWV?mG|5NG4&jLC;7?>$@!~piD~}Hp4<0R8X_qk)=BMY zZE2}0ooQ@SeEkB}1^n0m$eGa!LHt)7gG+(x| z9?7n->p47#SQxiHUZu7mA4Dl~VQ^&LL>UHMBXpwJMj)_{{7PvV)LY2w@8pc(Fzjeb z&Vo&Wn4CB4`@*R3$~N3uL95|W2;q(Yll=?(Zv zUo3}K+Y7$jIqAr8c#!@Q>i7kFb~Eq%qXDoZ5Fi#IW@o0n^~hd zu=j)9Ua@y1$0C1IRNy@fUxxUm5hR#cQSDQ>iwihJlG-7x7fFK|0Ky*u{R`~*wn26- zHy%UuKR^5&`WaSJ!&=%-AG6}W<4r!G6Q6W*a(bJ%0OT6*DZy6T-aDap0||9Nt#m^d!b~+IFMa1 zJzI^wVTU$M_3=y&4mv{#$At_86Pn40>i4)t(OF6dJ!dN6~_)A40yhkO1- zzrC@W$X7=*T1N{w#z#13S0-b}SDswdk8e8GQheQCe}7=zw`Vnh-CubIr;eC^A0xS0 z_+(l8LV^O#vRZ7(kJJ9u1xJYAo0Vdw*>20agz5JzL5rxfbe!)WzBTrFV6#v~+tR(M ze~^`|pEGOI3hkKkuYuD}OE7oY`)Pwl_!n8P1!d06(wil5)zy4r+_+b8Z+!;4`3=}e z3~Y-)!I9tO_lhnHHB-YR3?kiGZOFXoFlQyOwCC}ujFOAr;?9Mv&-1(1lW}uCv4`e+3Mbs<>`v+)>Se|7ysm0 zUoA{Y_8v5NgddR0FJX3?PU~iTy&uVU7Z!|0zxB_|Mv*P_9`_&01JqT#JqP_4Lx1zu^h>)>p;nBKhk@94W^9 z{qR64FH;H%SO^Vtl}cXj!sr#wvF41RVV|0s%r#;)+g>Hln&iV!R#&Ly)f9Dwzz|`G zmALjnt;zJQK+@M)&*TRs^952}i@nvcZeEepjdJdh@*2x>ahU*jqM9gGT()F6BvRz` zS$<7da_uWitF3}oqR1^~%=wv{s)^SzIb}I2RXUo+9f+$+v&lu%Du-TVd_^7x0>y0# zpI{LngXYtYn^!L6ZrVRRk@+BLB1n%G@%khD-f<(|L=taK8y8O0A+Rh!=M(6*_UV;^ zVfrmD3)v&r?iG!~69S3o*%L_6Amd))2i>i@Z0+Wor`QaU#VPZyK*RYH?2qx2Sc6_n zlbu2oJdQ?T;)378DjYYGLtGpUEQD4(CZcjiZcgpxU@Lf|Xb+Tim4d=|c$^biwli5s ztzKPO5MDP(@sZ&{^NTua;vjycu^q(mC+;My_DYB-dD#g1o*MC`@$wN>Hh+No0|BJ~ z?$uM{{HI?xGrsK`g1sl4dwXhpopU4n2nc4K8^^MR*DQyOLs-d3YnSVRwZ+{#XclV- zJ4hBSeq_ek=^e1*uapt>k<59~qxv?~_D_MsTN=SHuNlGz%)EtKm3G}oev5~=vy@!m z6(4EdiUe2$9w(|~iFg;Iv<@^+`2DGJcp|!p9vKGH z$=|t;Ss$SYtTDL+L4Q<}96Pf}q^z9l#2p;vcq6(6z-#ZGjkWZMQJ&K9*%JM zsi)Q=jZu@hW1Yl*lt7|pNxHIVt7FCxsR;pANpp`Xv-Yc3f@+0JCu5on&B)n7 zu=TIDP1eXCm)z4~&Bie|T|(eVAngg#_QW>ry;P^G-!tKw zM|RUX)MnCXBJaBV)bo@5f0SH1r2CH`!4$binNhCoN2AA$iyj`Mr@+ZOzWc!o=u$xQ z3DvQ}9B;!9jtRXAjDwV-ZpT?KlclryGzN=5j?G-3AE*vUXMgb-!H2ZmbPwggPmM;q zohxg+YmX_*4Bzz5dOOw|{zN54VE!1D?z_20fnV+TDj6Q79$gjP9b9n{yZ@TGd+=uj zmhIPH-C38_uj#MXu2~~7l|4KXW<5NPI38S4)gSUM{Fb)p?9?xRcnr$+b?m%*btGzG z{bXrjEx6b+d(Y>7hi-v``dPsJPS)CRMX*2iMBvZ&c26~2+p_f8h1o`5rxp#FZno!_ zrU&8LmeRG;2nr$eI%)Ri@pZOcFfnNmFojUDs@z7FDfMAmN9QbyS$2!EKG0oB(l zs$<4>L2bwRxo>eeYHk^fqz1kgt2c;`70B*k?6L+mEuC15FHEvu$fBt*jH!9GN8N5V z`YY&_d$_IDp8lA8gW;s*$p1ZbCvC2${t6A7}fTZq47t0zX&t91Mpnjt&= zKnFo2>$fnTS+A~4bWbj0@2c5FYdG`eGR!@|cwfw&<~!tAn_=9VNTu~90+#i(V5>Fs z{;jc)h2q-cjyHm>-8xl>dx3~PHd*hJQGhjIk^_@kpU*vOkIkVWjhS)m!OJ-#aL~UbZ zu4px9T5yZ9=MVhk(?9F`^X>CBq&RZ&qE}g52v=)K7j=QhUu|kaOQ*UV#t|7Cca)Xy z7^YQzvWC^Sj$pWb)y&oZ@_GqZNm)`<$KJo+0DJZ^Cc3d#P^~E?5<2{F3`<))yf~5c zxHEYR6LPPjf+1V`E*KDyd|pA47y5`;sby7Ii5%tkCd4}|N-#`NI6$?MlX$XzmC+>0 zIr^I`nz_;{WRcj9)vAt( z`3&2+$c+8ajOr2noLajiZ`%s#kfODuNmXO1-G@p3-|$keO?aufL3#mm-ExNQSovyq zgF>;pE_Wo&kt1N>;LOMx@|fLeRpU%mF<8soZ4dL?VIPwlw$W7bwEd$Gw!Wm(v5>y0j9YU9@YhFjA1jCKjN5mgil2m>7qRyj zKYl9i9@{STm%ykH6rKBnLTJe@?k&kj6Ol?zOl^f+KIP3OIMJD3z_-$97FiuPKs~ZQ z?|69iVbYy&nWo2LKV}^cI-Q7&yurF6K|SUSCW)b7O!G$VdXH=q;Y=xc*ga54wS-qi$6~^H-81H zIzr|$3-iK%kS<^#y`@eq=oC}N&V@^`=vGkW!@r+7Jy4#Z(PR{wS&T;&PIzhK-#bN< z&mD0#j2qLSKGF?S!Ag$Pgq`T|HyOB z`Ex50e`Ib)^XOF0>}uoLDv%Mhp=ljFy1GPO#4q|0F0ogEn~e`g+B4plo~m~TIUVIR zND!PcoiE_?<9UgWG#*3&MuKd(@(Tv6uHiYRi$!KMjVDI{{V#kW?oq<&LEot+4InIy zU%7qFm_4HfcSGWe_+|ShCj|3_EK!GgItZ))BNHvjSN56Nd`nIN<=GVZ)hb8f?6-Lp z-^_Aaa8J0v+MC{(KN|%|OqyTe&sPL5hwpSGhQe?{8;ZuId$SYVJ`U7L#v~2eJfQ1B zTjj6+;QkUQsKeXncO0NoG0UfZ;*Ck-H7jGn{UaD~j24|q{;*Eo*Nkf{>_3EhSel@EVtz;%8Rydp0gCD0UyevRUax`=GWaxe>ml zMd|9I4vZmJZZB!O(#_)0Q$zd0;>MZRYAZ`pJM0lngM_j(?UKv)*LA9CgLgmPw-Ze!hp>|=RlJts*$p%K!f>X&s(vWk30WIa zzB?-R@Vc}vG6+xZS}m&EQN-Pl*~>=Bd3hi3*n)L0-wqq2ekfOgh361PcoOs731s!w zQ7jkhci$FhmK2U#?kTgY&HPc@dU;+Vwiw~XOK3FVav@%0(W#8>O*awf8u=)_*DTpjFw=^3!+Px$;B%kV4uMBUmXr4lBzF`>s5j7w)}`QxkguygdiV!6UZ zfRbnC-r1OIO=sTOrveg%<0E`B>a~Y}j47!*bPu0TH>5`WgHm}I=Xp)mlJmTrz|FO4 zK3qteQly*UQ)WTO!O@2k=ZSWQuc*Ajt?{W%n9SO2&Z7sKY*5)OHK&v54{>w}@{FW{ zRtL!wd6-zIE4{%l-^6mcjCUb4KiplwHTfe#*zsT+V1|gvF%8WiQ*Jw`2v%2yuP{cL@B{j6UG`ap9{o zH`PRg92rm8Cp8+GmY#5+6dGI0&vQnX1SK}1ZV%1wS{+>_K+ZYZ4oxOvezo}0X%=c+6kBA1`oT#hk)lBzxWokHO%e>*bmYdKuwTGeA)(Tcf%Yo%&U-D{6`sfGY{_drY@{`tENs+A(Ii!wTZVS^BLmWd|DMY(7Ub) zdZ*tMXYL_F+srzN#rNe^tDTtMSKC?!hRuY%UGY)E%i@}_vZph;L07eK1*-+b`mv{? z-+~o%?sav!Cj{@=H2JERQxa(REL}PV6Hxk9VF?_qxsIqk#Kbvx${HNlKdy|qp>DYu z?P4!n-qk``Of&?d48_mHt}YcPkQox`Xn4c67iJIJ1>#YK~PY~`1ow?)8Gq;=ii_E^z0N1XiQ41)dY8^+@s|CkV(sR^S4{!ZvY29s z$+&d+p~ttfk7eXELH_)s9jCF0p_y2ML_chpF9|nd`Qww)kJ5J;ZE{~WpPp(iXy~5p zr11`?@aD5*30jsWU{|i=&+%d<1%qugKTWTi`Sbk2IIXsGraG)}x-55W&T2a_z5y~b zmjIX&IJN>X0~sWa7_j*Kr?Ap9PyUP&CXOXC0VxR6S=)-aZol9J@j~UR*?9PPirTKL znhiZ%1AnRtu(g`msh@}0K|5pZcB<|VG5F3z&ywCVnqEcb>(Dy8FuyARS7H{`I7tP{ zX!4e^JayRQZr1mnF{p~eb*D?Sb3qkbYiI;({hq4ZKlxHHN8S z^E4Nc(@l|a5%|bJ*}ve=_pKJq4u*A!{E6=J6q&~z^d!5T_2J%fcr7xqvJdDo4;yD< zwlf4sMExj27C+~RpiS|ZXAj;fYKyk10ibxAv(pb>A?F*}v4QVsbf_d=wcWX>$k{3JYCzxoO= z#DC;QV2kR%juSWVJ5C7!=_{mt$B?c1h|iFiNJfaii(wXse3JD?xO_g@D1Q+PQ*66= z-;UwM;hRGYk;qApZ@D% zriD-;Q$eAm3*D&Q0EP&h&tuR(qw;G28(h&b#VMNsOzUJg~5QnaMVaip%)dr9oMKmK<7-QE_>d^WH*7bSpE~+uW|RUoyn9-$ zCrC(XUL&l5m7`lZDDi`T6e<*zZEMy%Z`!O(M zkI__FIFnvMkD5z0^rt2PodysEgOc}s;pFeHqlCi%N>0SDi%{3;aw}1ct*ucz+@nSw z^zhB8ariCB_la#Pqn{uJLwAUA_bZy0u&Hc&|9t$`2_i@xQ>&yl+G zP|)Rct-$?LxWlXYZe{V&pl$b4W4zU@WLAciyxINmu(IQ!@g5HzdGiB1L6u6Za-eU& z{AF?PS@~e^#+8v0!6p*H#IN2Ryi#w~F-|*5vJ9V3{FQ?5(aLvNF6a}3GCq9vT#P+Z zhF-j}E43Ewz_>}IUGw2Xz4^3h5nUy0Rkok zCQyTa#0?pgV8;##m0(9pSE$^|6lAH~N*6?_*yYDEB`}tpCrUGyoX3aC08_w!s1w5S zCv|}<`ID+Z-TX;SU`GC=67Ws_q#h8SKdA<^%%9W(l7h`?6B5DZ)CmqH>`BsWCF}{( z1SR>OX;py{1;68^X-e{?Xf=VK!Jp%$MN8PDrTtX)WN7t(w%`)F1dWnDDOxR{68LMp zbbv}v0si|0#gYIiS`DBu7$aWVM}-1hEz|#=NEFzj!d_AgR>~Ji%uNQLNEb*0q4{=N zz@q%qU=^>VA#3oBDzH;(3tDpjE`h4x)L*3|X~-FTqXwiauoD5UNo`q_c+e(nfN#J+ zeW@+`5|8%@3I(TMRRj`;B*8a|Kq{#%&yqH}1m}X&NR`#ZA#Jd$GSFITEBX0H$U_!j zS5;u8)Rsv}+q(qff>S@0%%mZEu&WwyOlqrGB{OkI8SDxMDoAbFmhin#kSwqh14c=0 zK}z`EC7>0Y`luWx4S9lj7!yRn&Fa7_sjX}k9H{~@kO+JtwPjVZOq=ikUZ6{82RADN zouy2oRIHMQEWypHz}5o0FcquBp^qg8@k4UpW+fnRzMa2HgUps&2`p|%tppY`&~Xq_)14=+GuyfHmn77z)526BzPq)PPAcTdXBzF`?}4gyxs@lLt!q z3wcttk};-HZ$~J_Q~CdQirRv=a4PXd!O%KyLh~p3$w#I9hdik}$(YNi|34M~i3LtS zxu}$1pC@G@88aLCw)~B_7H?>s6QMbrz8#lktE#1iwOSG%wmsBG5|Ys4Xl`Pt9zxVp zuM=XD@UD4Fqy7mgs6f?WcYPvEs*@uFZn|s zHg#bbU@Cx~;P>w#)CD}{z~r}m#??1KM!eoYlJ|Xz)IsPmbbSWYs2Cu{5CoA;%!y9c94awE>RYw< z=fS%)V)7;;AH|$ZdR~MuQoCVI$mr~U_}v`vo$$mB-8KeqMf|&!hy&3?EBb_Hh`fjc zVXnVaSvwwF3}_i>Mk8hsqY;ExF4mSPQiu0eH)LCcoglXaPfl#oTqHk~og8bOFgHf( zlX!^nE9TE3V$@`q6G9=#)O+f)RLHNG#dJ&_y74XoLWnX!B4Su$^we(yL-6K$0?`RF zb3_8rUolpoV7#Ii^J&_~>-iW^&MJW0Qyfl(n1hvrp`hACNF9RJPuTO*pI)`cQ6gkY zBprY55wBt`@LDcJKk&MM)rz}sEWx%H;Duj76?iQia@1dc6tJ&NEstp{qH`w^QW?0T zE4TZXy=PgsDsVk}@^dHq%_Br5P)O6N{71UF#kPNa>luT`Xt#;=V@#~S0r)TBKI z&st@c9KlvLWG-;WLZl7Y#~{+i=-2kHFRbSbg*ub?HxG3tk#Od#-^IrsS|LRNF5nQl z02iGA57{P4qTeBSQ6Uz%SSQiP9*QAo0ZsZLcLCgtB9! zlPU<(TcikxAc7#hcLYMOdJyR#y#!Q>6j5nXOEy7!d#!~5ZVXRW>GS^t^Y&$DM{ z&4<~0ejR4}!17(1_50p?Qrw>O$QED=nRg?wJ$g=HK%b|cRYOod8lz&fC^~SY>Q0NS zMAvCxYG{#-!0dshB%X9iRU4riIB4L#stw;j5ckZZ6mx)iE6|**-VFUN22&+b!i3a! z7!E|gSFNf@6o|197${B$d!p&n!AyzipVGnX+!^HcChFbl1J*nP>qD8>(HcBws@x1@ zUM%X>v35{1T2T-EJRMBS<%k|ANl5v9DkHf|0AP)QY~cak)W$nBAq7A_o{vP_ zZzygHtEU1#L^BBiV45G@ar0O`pF(4jcwO^{Y9ie{JGbXYhVpBS$V{MG4AULvy~x4J zM8^Oo8|IO$lf0oVQy`&Gp|$EYpo&k-7J`|wU3r_|CM0GHq~I%9_)Nv#ToXZ+YRcTn zRuJHD2teDWr^*6hF<#`U0_w>_+QVOlH2U6mN)+G~WCP(sAd9Ij9`(&3v?4G^5cG{( z&Pa(% z_LgoCfj6gCP;!J;#5;w^)#UKFUbWuFCmh#ua}JdcUKE{2ZQfWWrHWEJ=w@7sawGNL z!@CevAwCFik7D3krhg^l5;o4Ka~_#-V~^4*!;c?BxQpKswb)awB9eArZH2&7^94Lb zK1$S}jr=TWJRqDk@1B;`EU4N;+fC@AIee$bv5wrGh!yz7swhAon^#mxNXieVml1FY z!B*?+5a9lOC@j8FJtN0oeFt~>)p{Y?0%C$&gUf^GSBm%Zigm{)ql;Vho(V6vA1)Wy zXj^0X_FwPw1v9oYwlZQE_u<$sL}#MzduYx6{*U0z)}sd#-#135LUT)Zb4u+;jIG9u zO|lpj#UA7`D9pkZVNPG%rAJ1yJCV&!%_}32^TK(-5iJe=Gl&oR6Y=HtV^VmH~FNC$|d>wiH(2ORa;jn(s^}bDgNA zLV3nEtvwOHIC@9AieBQJ@5>+>fA!5Z&y$t4eY>1EOx!;EI>*w3lwjZm>HSfE>bdNl z{nggm_SKcHeZ?H#rTH>aSKta)YXrmSH;FQQ2@P`H+8Oei=;Z5NhT{UgX& z{%%TvjavVdFSWe0bIot5(nEz+M@3YqZnzo%*4)J5!}xfp*63*^;B<#P*ZDY+OLP8mnQ@npjMw zot>^lCUZ$!U&@cV@gk%2O9?foxpTIGtR1%wx8S)$taE)0@U-q5lzfymmzLovT~Qk( ziiEVhCU1B}wk9v5+ILi76W`3hd-Jdr_0?_nMeFv3k`3{Y!p1G9uHCB_tzL~`-!a*f zZWncPyH1a5XBAR6FtVr5gEFQAT1m{6~m z?=HE;HHL_oyZD*Z8xgH{sWM+HUAt2-iZ|6OX+$DQV%$q7&U|YfQDjkIK*V>KfQdR@ zg@%drO(giPBoi?}1%Hqf)JullKnU6vP!X#20)OENRPJ&y;rXb9jZ+^IhbU7Gl33>#Ddhzq6kjRoWlu7e^Z<5B^^n5g`nj(y-;gxc z?ck5qDtxT>-wV}4B1Iz3@RQzPEc}2fkhOz&h3H;53hRHs5=h@cU`rwv!PrZIeQ>}Q z$e=*@II^_Y7^`zY6v(DP!a=On1H#%J00V&vc>AO$5&c*ohH;6IDv~hox(OaNhINVf zI4b0>7>0HU|2TZH*8^)-$GSw|Nz&e9;6;m}*~OC%-@^vgQMwab_xRt1%n?aNQev&V zC^0NH#4}O;SY0Ek8a%h&Pgse%Slyr+qwy>H!!Y7Ygvyoq|IT!f>S4;ed(B9kQJc_5j-F%GzBPn?<342oVU{`v~AAhn>=GQs{aUu&B==zCw6 z-Bn*`M?#_tT@^bIYyG_um_zR23JDc!zM>-QTKr_%*jeVBS`|$dY8uJ~;Z;5AbOe$n0he7hk>TPxxg!gl`C=Tv%ixj+m z6)l=yJoANkY4f^`rmWi54dLb{ZlC=4s{4Jr z;%gLZySid`ST5Q#r1D_y1tWg!9vba)2GY~tmisPiOIf#XykfrZ0eR@T&eDFy1=Bbg zhx3 zHs>zV)=(hL7@D+#dQ%Q@9X9e^uNmpMa0&@=`!x2@A!G?_3|VYVaA^t|D@AC@j}r#0 zNmrVdEMJ2=OM5TRUcLETC*OL`u%O?vZn-oH6^I%07{6Bv+gc2mv|We`bMi;%54Jli zh52w)Qmp2ed44tD)OJB_@iYCs;>pD2A)eJ>zaJ531a-KqK?jM;GrO1CYPP6s;Ij+>vtHN;q7-oAOn zE3vNyTUAE6?1ssxArO=BD$n@+L^O=#-)iWmRvE;p^E2`i2DL~k=UxlY% zzjte4O@6mN{FeNO^#5wSRC>+zZFIsGs;UE$_s@@zL^lV{6m2SLa}OEC7uO8IP6q59 zWUUrB&4SW&-wt{Cq!b+9kJ)w|%s)GVrB6GjX6g3C*LhpmJswhWmM2Uud!)Z-;!>wno z2t=&0B+-{h8E<~lK18aK2QfBgBms)tv)K73=CZDLfrij#pI37IDGP8De{jiqKy zU+6SdYjM)h zfIf@A$$pRdv$B?DwYg+o|3)g|+K}>KGF7zKgNwUBd)k}31RqaG`!E;B%Hq#{qbBnH z0ZkbHSjX}a@RrO}^+We9AL7_UebLbpbVIF)rZne zqH6kK(k7y*Am9XYek2%db}_+Q84K$Q_6*hh>Q5_=`@CMw6{2kxhr!EcVZJ_7)ZV@m z`AMJ#yxg8Rc#FE3bfGC5v<^~TH)hpr%V5iF*+_l8+_xhgPmvU#;Qr+4q~xb@ILQ1{ zskA|WTmef#>SL8^&#ByEKUYIUV~{^|Y%6o#TT@F!klBjd$ zM_*`6j1Zwwh0^|8-PN3$lERWNDuai|ZAapMm7}fl=yxATEqO2NShjR^T)rLxE?MBP z0wJ-T_!{ZKK(XqurCE)9h}!${@LJXb0H?1++ED_y(XnvyK~77E65K@5ZFAehDp|)1 zI%%x{mwH*6n(@`v(&}m)Q~U0wq=E_MlqxIHL->sH^ckieW%NZ@c16W#NtI zp?FVyucf%ZW51uL|F-$izR6}6 zHTRtyptY5!Ohk?x6DFIbPNgu=y@o?J5;yOzDs*{wpW?s48yX%?+)dj!z4q(I5+d=Bawnvjqb!dmo#JK1HA!)9x9V z=-Wt$qNwbUXM($tShF1_q$eXHMvtUuKt7VK?J>f2uBYBI>HD&&xh|(i6X>S$6hR~# z#Zr`*U}ph(Ja&HY0N)iHpEVBY;9#|vGS1Qz9N~HT3dkp7q2qhg<$iH%iKKD5x|^T& zD?#XlYyBg$1bv*93Y{KEROSLYMopE9lD+Y0|g(`uhN<))#R(edI8pYB>(ghgpo2txuyN30b? z`6}mmJ1;I9otES}QO{)0HGynZR1dek8;P;!Cbfp9doN;%W~gW_Xm8yek%(y8CNhg7KN0F~(DfzmQ|T2w zB?ZZvvrN9b)+KL2>3I}d+H;TR8&5osqo#8XkDSK9{je6N$cd`3UGLkKsqnTJiCdF= zAk?kXw_didw`;AN*xq);2)>D%HmF_^OCY-_N#yARJpsKkN<}vAf64eNnKbvIVTMLJ zhyUFhz)P*XcUoMIV`j~bADRNyXHfO!V$H1WK1MsK^VKLqf9n+3O_;Cx=7yyQZ5HO+ zlA4Wv7A0KbmG^4R2y!>8WS-CiKS}QRo%6mc!_+i%8aDi1oqR-e=Ou%gm|~mv9g|D1 zU{xR0qDGXf)p?0d(qwSr)=EAGk$@e{Jiyk?9iUIWg54jwKW+4ny1|}xL)vvEL?$>( zU?bqOWQdY~vC!OP6bRez>cQVQyX5N{S=5TbjU>Q9m(f%I26W7fZM+Q$M8`X5_ z*k3SBZUiF_n`ConChC18%}Qna(z2T>o*A#ZVuXb!y`p6@777Q(zZQdJl#{QWHyFPR zN@Bg}e&ki>)6k&)PLTY-yP-UY@f`eBpo~n9ya9nCtw64Q#Fro{jy9$H>HYeRua9@b zHyv?tu+Qbyo4`2eW0j{WX)5$|y4IGa#UaE?;TmhpzRF=9C{usD?-8$GOfUNmK1j8s z{OoCWcsyx)-zZo^6Ae#ERWq^Fw6c5}m27-R&DPd9GR;`k*w$9TOk`w4q`JDSOz7>1Ncp>-9z483u(b650FwL@&QI;0|1SHw`NLPaiVSf3}sC{kvaru%tK+a{AZjfpPEk4^39;AN9l`Qh$$G91MZ{N%Qlu z!!aU#aQItemmmjR_uZo&BD?p`?5E)5H8F3j02}x;` zgACG98ihjHOM@k)?Ho`@MFmHc3|Inb=V<4M6ql5ifXJZi9dNmXEW}X;B`qbb_-iD6 X{p@`FejhUl38*-fl8;ZrP?PdspsZ8$ literal 0 HcmV?d00001 diff --git a/community/document-parsers/document-parser-markdown/pom.xml b/community/document-parsers/document-parser-markdown/pom.xml new file mode 100644 index 00000000..0b8b47ce --- /dev/null +++ b/community/document-parsers/document-parser-markdown/pom.xml @@ -0,0 +1,71 @@ + + + 4.0.0 + + com.alibaba.cloud.ai + spring-ai-alibaba + ${revision} + ../../../pom.xml + + + document-parser-markdown + document-parser-markdown + document-parser-markdown for Spring AI Alibaba + jar + https://github.com/alibaba/spring-ai-alibaba + + https://github.com/alibaba/spring-ai-alibaba + git://github.com/alibaba/spring-ai-alibaba.git + git@github.com:alibaba/spring-ai-alibaba.git + + + + + 17 + 17 + UTF-8 + 0.22.0 + + + + + com.alibaba.cloud.ai + spring-ai-alibaba-core + ${project.parent.version} + + + + org.commonmark + commonmark + ${commonmark.version} + + + + + org.springframework.ai + spring-ai-test + test + + + + org.springframework.boot + spring-boot-starter-test + test + + + + org.junit.jupiter + junit-jupiter-engine + test + + + + org.assertj + assertj-core + test + + + + \ No newline at end of file diff --git a/community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParser.java b/community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParser.java new file mode 100644 index 00000000..56ec3dae --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParser.java @@ -0,0 +1,206 @@ +package com.alibaba.cloud.ai.parser.markdown; + +import com.alibaba.cloud.ai.document.DocumentParser; +import com.alibaba.cloud.ai.parser.markdown.config.MarkdownDocumentParserConfig; +import org.commonmark.node.AbstractVisitor; +import org.commonmark.node.BlockQuote; +import org.commonmark.node.Code; +import org.commonmark.node.FencedCodeBlock; +import org.commonmark.node.HardLineBreak; +import org.commonmark.node.Heading; +import org.commonmark.node.ListItem; +import org.commonmark.node.Node; +import org.commonmark.node.SoftLineBreak; +import org.commonmark.node.Text; +import org.commonmark.node.ThematicBreak; +import org.commonmark.parser.Parser; +import org.springframework.ai.document.Document; + +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +/** + * @author HeYQ + * @since 2024-12-08 21:32 + */ + +public class MarkdownDocumentParser implements DocumentParser { + + /** + * Configuration to a parsing process. + */ + private final MarkdownDocumentParserConfig config; + + /** + * Markdown parser. + */ + private final Parser parser; + + public MarkdownDocumentParser() { + this(MarkdownDocumentParserConfig.defaultConfig()); + } + + /** + * Create a new {@link MarkdownDocumentParser} instance. + * + */ + public MarkdownDocumentParser(MarkdownDocumentParserConfig config) { + this.config = config; + this.parser = Parser.builder().build(); + } + + @Override + public List parse(InputStream inputStream) { + try (var input = inputStream) { + Node node = this.parser.parseReader(new InputStreamReader(input)); + + DocumentVisitor documentVisitor = new DocumentVisitor(this.config); + node.accept(documentVisitor); + + return documentVisitor.getDocuments(); + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + /** + * A convenient class for visiting handled nodes in the Markdown document. + */ + static class DocumentVisitor extends AbstractVisitor { + + private final List documents = new ArrayList<>(); + + private final List currentParagraphs = new ArrayList<>(); + + private final MarkdownDocumentParserConfig config; + + private Document.Builder currentDocumentBuilder; + + DocumentVisitor(MarkdownDocumentParserConfig config) { + this.config = config; + } + + /** + * Visits the document node and initializes the current document builder. + */ + @Override + public void visit(org.commonmark.node.Document document) { + this.currentDocumentBuilder = Document.builder(); + super.visit(document); + } + + @Override + public void visit(Heading heading) { + buildAndFlush(); + super.visit(heading); + } + + @Override + public void visit(ThematicBreak thematicBreak) { + if (this.config.horizontalRuleCreateDocument) { + buildAndFlush(); + } + super.visit(thematicBreak); + } + + @Override + public void visit(SoftLineBreak softLineBreak) { + translateLineBreakToSpace(); + super.visit(softLineBreak); + } + + @Override + public void visit(HardLineBreak hardLineBreak) { + translateLineBreakToSpace(); + super.visit(hardLineBreak); + } + + @Override + public void visit(ListItem listItem) { + translateLineBreakToSpace(); + super.visit(listItem); + } + + @Override + public void visit(BlockQuote blockQuote) { + if (!this.config.includeBlockquote) { + buildAndFlush(); + } + + translateLineBreakToSpace(); + this.currentDocumentBuilder.withMetadata("category", "blockquote"); + super.visit(blockQuote); + } + + @Override + public void visit(Code code) { + this.currentParagraphs.add(code.getLiteral()); + this.currentDocumentBuilder.withMetadata("category", "code_inline"); + super.visit(code); + } + + @Override + public void visit(FencedCodeBlock fencedCodeBlock) { + if (!this.config.includeCodeBlock) { + buildAndFlush(); + } + + translateLineBreakToSpace(); + this.currentParagraphs.add(fencedCodeBlock.getLiteral()); + this.currentDocumentBuilder.withMetadata("category", "code_block"); + this.currentDocumentBuilder.withMetadata("lang", fencedCodeBlock.getInfo()); + + buildAndFlush(); + + super.visit(fencedCodeBlock); + } + + @Override + public void visit(Text text) { + if (text.getParent() instanceof Heading heading) { + this.currentDocumentBuilder.withMetadata("category", "header_%d".formatted(heading.getLevel())) + .withMetadata("title", text.getLiteral()); + } + else { + this.currentParagraphs.add(text.getLiteral()); + } + + super.visit(text); + } + + public List getDocuments() { + buildAndFlush(); + + return this.documents; + } + + private void buildAndFlush() { + if (!this.currentParagraphs.isEmpty()) { + String content = String.join("", this.currentParagraphs); + + Document.Builder builder = this.currentDocumentBuilder.withContent(content); + + this.config.additionalMetadata.forEach(builder::withMetadata); + + Document document = builder.build(); + + this.documents.add(document); + + this.currentParagraphs.clear(); + } + this.currentDocumentBuilder = Document.builder(); + } + + private void translateLineBreakToSpace() { + if (!this.currentParagraphs.isEmpty()) { + this.currentParagraphs.add(" "); + } + } + + } + +} diff --git a/community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/config/MarkdownDocumentParserConfig.java b/community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/config/MarkdownDocumentParserConfig.java new file mode 100644 index 00000000..1db101e7 --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/main/java/com/alibaba/cloud/ai/parser/markdown/config/MarkdownDocumentParserConfig.java @@ -0,0 +1,122 @@ +package com.alibaba.cloud.ai.parser.markdown.config; + +import org.springframework.ai.document.Document; +import org.springframework.util.Assert; + +import java.util.HashMap; +import java.util.Map; + +/** + * @author HeYQ + * @since 2024-12-08 21:38 + */ + +public class MarkdownDocumentParserConfig { + + public final boolean horizontalRuleCreateDocument; + + public final boolean includeCodeBlock; + + public final boolean includeBlockquote; + + public final Map additionalMetadata; + + public MarkdownDocumentParserConfig(Builder builder) { + this.horizontalRuleCreateDocument = builder.horizontalRuleCreateDocument; + this.includeCodeBlock = builder.includeCodeBlock; + this.includeBlockquote = builder.includeBlockquote; + this.additionalMetadata = builder.additionalMetadata; + } + + /** + * @return the default configuration + */ + public static MarkdownDocumentParserConfig defaultConfig() { + return builder().build(); + } + + public static Builder builder() { + return new Builder(); + } + + public static final class Builder { + + private boolean horizontalRuleCreateDocument = false; + + private boolean includeCodeBlock = false; + + private boolean includeBlockquote = false; + + private Map additionalMetadata = new HashMap<>(); + + private Builder() { + } + + /** + * Text divided by horizontal lines will create new {@link Document}s. The default + * is {@code false}, meaning text separated by horizontal lines won't create a new + * document. + * @param horizontalRuleCreateDocument flag to determine whether new documents are + * created from text divided by horizontal line + * @return this builder + */ + public Builder withHorizontalRuleCreateDocument(boolean horizontalRuleCreateDocument) { + this.horizontalRuleCreateDocument = horizontalRuleCreateDocument; + return this; + } + + /** + * Whatever to include code blocks in {@link Document}s. The default is + * {@code false}, which means all code blocks are in separate documents. + * @param includeCodeBlock flag to include code block into paragraph document or + * create new with code only + * @return this builder + */ + public Builder withIncludeCodeBlock(boolean includeCodeBlock) { + this.includeCodeBlock = includeCodeBlock; + return this; + } + + /** + * Whatever to include blockquotes in {@link Document}s. The default is + * {@code false}, which means all blockquotes are in separate documents. + * @param includeBlockquote flag to include blockquotes into paragraph document or + * create new with blockquote only + * @return this builder + */ + public Builder withIncludeBlockquote(boolean includeBlockquote) { + this.includeBlockquote = includeBlockquote; + return this; + } + + /** + * Adds this additional metadata to the all built {@link Document}s. + * @return this builder + */ + public Builder withAdditionalMetadata(String key, Object value) { + Assert.notNull(key, "key must not be null"); + Assert.notNull(value, "value must not be null"); + this.additionalMetadata.put(key, value); + return this; + } + + /** + * Adds this additional metadata to the all built {@link Document}s. + * @return this builder + */ + public Builder withAdditionalMetadata(Map additionalMetadata) { + Assert.notNull(additionalMetadata, "additionalMetadata must not be null"); + this.additionalMetadata = additionalMetadata; + return this; + } + + /** + * @return the immutable configuration + */ + public MarkdownDocumentParserConfig build() { + return new MarkdownDocumentParserConfig(this); + } + + } + +} diff --git a/community/document-parsers/document-parser-markdown/src/test/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParserTest.java b/community/document-parsers/document-parser-markdown/src/test/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParserTest.java new file mode 100644 index 00000000..230af67a --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/java/com/alibaba/cloud/ai/parser/markdown/MarkdownDocumentParserTest.java @@ -0,0 +1,263 @@ +/* + * Copyright 2023-2024 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.alibaba.cloud.ai.parser.markdown; + +import com.alibaba.cloud.ai.parser.markdown.config.MarkdownDocumentParserConfig; +import org.junit.jupiter.api.Test; +import org.springframework.ai.document.Document; +import org.springframework.core.io.DefaultResourceLoader; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.groups.Tuple.tuple; + +/** + * @author HeYQ + * @since 2024-12-08 21:38 + */ +class MarkdownDocumentParserTest { + + @Test + void testOnlyHeadersWithParagraphs() throws IOException { + MarkdownDocumentParser reader = new MarkdownDocumentParser(); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/only-headers.md").getInputStream()); + + assertThat(documents).hasSize(4) + .extracting(Document::getMetadata, Document::getContent) + .containsOnly(tuple(Map.of("category", "header_1", "title", "Header 1a"), + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue."), + tuple(Map.of("category", "header_1", "title", "Header 1b"), + "Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam lobortis risus libero, sed sollicitudin risus cursus in. Morbi enim metus, ornare vel lacinia eget, venenatis vel nibh."), + tuple(Map.of("category", "header_2", "title", "Header 2b"), + "Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero."), + tuple(Map.of("category", "header_2", "title", "Header 2c"), + "Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit.")); + } + + @Test + void testWithFormatting() throws IOException { + MarkdownDocumentParser reader = new MarkdownDocumentParser(); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/with-formatting.md").getInputStream()); + + assertThat(documents).hasSize(2) + .extracting(Document::getMetadata, Document::getContent) + .containsOnly(tuple(Map.of("category", "header_1", "title", "This is a fancy header name"), + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim."), + tuple(Map.of("category", "header_3", "title", "Header 3"), + "Aenean eu leo eu nibh tristique posuere quis quis massa.")); + } + + @Test + void testDocumentDividedViaHorizontalRules() throws IOException { + MarkdownDocumentParserConfig config = MarkdownDocumentParserConfig.builder() + .withHorizontalRuleCreateDocument(true) + .build(); + + MarkdownDocumentParser reader = new MarkdownDocumentParser(config); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/horizontal-rules.md").getInputStream()); + + assertThat(documents).hasSize(7) + .extracting(Document::getMetadata, Document::getContent) + .containsOnly(tuple(Map.of(), + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida."), + tuple(Map.of(), + "Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim."), + tuple(Map.of(), + "Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis et magna."), + tuple(Map.of(), + "Vestibulum nec eros non felis fermentum posuere eget ac risus. Curabitur et fringilla massa. Cras facilisis nec nisl sit amet sagittis."), + tuple(Map.of(), + "Aenean eu leo eu nibh tristique posuere quis quis massa. Nullam lacinia luctus sem ut vehicula."), + tuple(Map.of(), + "Aenean quis vulputate mi. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Nam tincidunt nunc a tortor tincidunt, nec lobortis diam rhoncus."), + tuple(Map.of(), "Nulla facilisi. Phasellus eget tellus sed nibh ornare interdum eu eu mi.")); + } + + @Test + void testDocumentNotDividedViaHorizontalRulesWhenIsDisabled() throws IOException { + + MarkdownDocumentParserConfig config = MarkdownDocumentParserConfig.builder() + .withHorizontalRuleCreateDocument(false) + .build(); + MarkdownDocumentParser reader = new MarkdownDocumentParser(config); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/horizontal-rules.md").getInputStream()); + + assertThat(documents).hasSize(1); + + Document documentsFirst = documents.get(0); + assertThat(documentsFirst.getMetadata()).isEmpty(); + assertThat(documentsFirst.getContent()).startsWith("Lorem ipsum dolor sit amet, consectetur adipiscing elit") + .endsWith("Phasellus eget tellus sed nibh ornare interdum eu eu mi."); + } + + @Test + void testSimpleMarkdownDocumentWithHardAndSoftLineBreaks() throws IOException { + + MarkdownDocumentParser reader = new MarkdownDocumentParser(); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/simple.md").getInputStream()); + + assertThat(documents).hasSize(1); + + Document documentsFirst = documents.get(0); + assertThat(documentsFirst.getMetadata()).isEmpty(); + assertThat(documentsFirst.getContent()).isEqualTo( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim.Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis et magna. Vestibulum nec eros non felis fermentum posuere eget ac risus.Aenean eu leo eu nibh tristique posuere quis quis massa. Nullam lacinia luctus sem ut vehicula."); + } + + @Test + void testCode() throws IOException { + MarkdownDocumentParserConfig config = MarkdownDocumentParserConfig.builder() + .withHorizontalRuleCreateDocument(true) + .build(); + + MarkdownDocumentParser reader = new MarkdownDocumentParser(config); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/code.md").getInputStream()); + + assertThat(documents).satisfiesExactly(document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of()); + assertThat(document.getContent()).isEqualTo("This is a Java sample application:"); + }, document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "java", "category", "code_block")); + assertThat(document.getContent()).startsWith("package com.example.demo;") + .contains("SpringApplication.run(DemoApplication.class, args);"); + }, document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of("category", "code_inline")); + assertThat(document.getContent()).isEqualTo( + "Markdown also provides the possibility to use inline code formatting throughout the entire sentence."); + }, document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of()); + assertThat(document.getContent()) + .isEqualTo("Another possibility is to set block code without specific highlighting:"); + }, document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "", "category", "code_block")); + assertThat(document.getContent()).isEqualTo("./mvnw spring-javaformat:apply\n"); + }); + } + + @Test + void testCodeWhenCodeBlockShouldNotBeSeparatedDocument() throws IOException { + MarkdownDocumentParserConfig config = MarkdownDocumentParserConfig.builder() + .withHorizontalRuleCreateDocument(true) + .withIncludeCodeBlock(true) + .build(); + + MarkdownDocumentParser reader = new MarkdownDocumentParser(config); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/code.md").getInputStream()); + + assertThat(documents).satisfiesExactly(document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "java", "category", "code_block")); + assertThat(document.getContent()).startsWith("This is a Java sample application: package com.example.demo") + .contains("SpringApplication.run(DemoApplication.class, args);"); + }, document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of("category", "code_inline")); + assertThat(document.getContent()).isEqualTo( + "Markdown also provides the possibility to use inline code formatting throughout the entire sentence."); + }, document -> { + assertThat(document.getMetadata()).isEqualTo(Map.of("lang", "", "category", "code_block")); + assertThat(document.getContent()).isEqualTo( + "Another possibility is to set block code without specific highlighting: ./mvnw spring-javaformat:apply\n"); + }); + } + + @Test + void testBlockquote() throws IOException { + + MarkdownDocumentParser reader = new MarkdownDocumentParser(); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/blockquote.md").getInputStream()); + + assertThat(documents).hasSize(2) + .extracting(Document::getMetadata, Document::getContent) + .containsOnly(tuple(Map.of(), + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue."), + tuple(Map.of("category", "blockquote"), + "Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit.")); + } + + @Test + void testBlockquoteWhenBlockquoteShouldNotBeSeparatedDocument() throws IOException { + MarkdownDocumentParserConfig config = MarkdownDocumentParserConfig.builder() + .withIncludeBlockquote(true) + .build(); + + MarkdownDocumentParser reader = new MarkdownDocumentParser(config); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/blockquote.md").getInputStream()); + + assertThat(documents).hasSize(1); + + Document documentsFirst = documents.get(0); + assertThat(documentsFirst.getMetadata()).isEqualTo(Map.of("category", "blockquote")); + assertThat(documentsFirst.getContent()).isEqualTo( + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit."); + } + + @Test + void testLists() throws IOException { + + MarkdownDocumentParser reader = new MarkdownDocumentParser(); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/lists.md").getInputStream()); + + assertThat(documents).hasSize(2) + .extracting(Document::getMetadata, Document::getContent) + .containsOnly(tuple(Map.of("category", "header_2", "title", "Ordered list"), + "Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien odio. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor."), + tuple(Map.of("category", "header_2", "title", "Unordered list"), + "Aenean eu leo eu nibh tristique posuere quis quis massa. Aenean imperdiet libero dui, nec malesuada dui maximus vel. Vestibulum sed dui condimentum, cursus libero in, dapibus tortor. Etiam facilisis enim in egestas dictum.")); + } + + @Test + void testWithAdditionalMetadata() throws IOException { + MarkdownDocumentParserConfig config = MarkdownDocumentParserConfig.builder() + .withAdditionalMetadata("service", "some-service-name") + .withAdditionalMetadata("env", "prod") + .build(); + + MarkdownDocumentParser reader = new MarkdownDocumentParser(config); + + List documents = reader + .parse(new DefaultResourceLoader().getResource("classpath:/simple.md").getInputStream()); + + assertThat(documents).hasSize(1); + + Document documentsFirst = documents.get(0); + assertThat(documentsFirst.getMetadata()).isEqualTo(Map.of("service", "some-service-name", "env", "prod")); + assertThat(documentsFirst.getContent()).startsWith("Lorem ipsum dolor sit amet, consectetur adipiscing elit."); + } + +} diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/blockquote.md b/community/document-parsers/document-parser-markdown/src/test/resources/blockquote.md new file mode 100644 index 00000000..d92ac44f --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/blockquote.md @@ -0,0 +1,8 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed +nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. + +> Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget +> sapien odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a +> porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum +> suscipit. + diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/code.md b/community/document-parsers/document-parser-markdown/src/test/resources/code.md new file mode 100644 index 00000000..31d7c7b0 --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/code.md @@ -0,0 +1,25 @@ +This is a Java sample application: + +```java +package com.example.demo; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; + +@SpringBootApplication +public class DemoApplication { + public static void main(String[] args) { + SpringApplication.run(DemoApplication.class, args); + } +} +``` + +Markdown also provides the possibility to `use inline code formatting throughout` the entire sentence. + +--- + +Another possibility is to set block code without specific highlighting: + +``` +./mvnw spring-javaformat:apply +``` diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/horizontal-rules.md b/community/document-parsers/document-parser-markdown/src/test/resources/horizontal-rules.md new file mode 100644 index 00000000..f7affefc --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/horizontal-rules.md @@ -0,0 +1,27 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. + +--- + +Cras accumsan tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu +elementum dignissim. + +*** +Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis +et magna. + +* * * + +Vestibulum nec eros non felis fermentum posuere eget ac risus. Curabitur et fringilla massa. Cras facilisis nec nisl sit +amet sagittis. + +***** + +Aenean eu leo eu nibh tristique posuere quis quis massa. Nullam lacinia luctus sem ut vehicula. + +--------------------------------------- + +Aenean quis vulputate mi. Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Nam tincidunt nunc a tortor tincidunt, nec lobortis diam rhoncus. + +- - - + +Nulla facilisi. Phasellus eget tellus sed nibh ornare interdum eu eu mi. diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/lists.md b/community/document-parsers/document-parser-markdown/src/test/resources/lists.md new file mode 100644 index 00000000..f82e7e34 --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/lists.md @@ -0,0 +1,17 @@ +## Ordered list + +1. Lorem ipsum dolor sit *amet*, consectetur adipiscing elit. **Curabitur** diam eros, laoreet sit _amet_ cursus vitae, + varius sed nisi. +2. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. +3. Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget + sapien odio. + 1. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum + suscipit. + 2. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. Ut rhoncus nec justo a porttitor. + +## Unordered list + +* Aenean eu leo eu nibh tristique posuere quis quis massa. +* Aenean imperdiet libero dui, nec malesuada dui maximus vel. Vestibulum sed dui condimentum, cursus libero in, dapibus + tortor. + * Etiam facilisis enim in egestas dictum. diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/only-headers.md b/community/document-parsers/document-parser-markdown/src/test/resources/only-headers.md new file mode 100644 index 00000000..81c770e8 --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/only-headers.md @@ -0,0 +1,20 @@ +# Header 1a + +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Curabitur diam eros, laoreet sit amet cursus vitae, varius sed +nisi. Cras sit amet quam quis velit commodo porta consectetur id nisi. Phasellus tincidunt pulvinar augue. + +# Header 1b + +Vestibulum ante ipsum primis in faucibus orci luctus et ultrices posuere cubilia curae; Etiam lobortis risus libero, sed +sollicitudin risus cursus in. Morbi enim metus, ornare vel lacinia eget, venenatis vel nibh. + +## Header 2b + +Proin vel laoreet leo, sed luctus augue. Sed et ligula commodo, commodo lacus at, consequat turpis. Maecenas eget sapien +odio. Maecenas urna lectus, pellentesque in accumsan aliquam, congue eu libero. + +# Header 1c + +## Header 2c + +Ut rhoncus nec justo a porttitor. Pellentesque auctor pharetra eros, viverra sodales lorem aliquet id. Curabitur semper nisi vel sem interdum suscipit. diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/simple.md b/community/document-parsers/document-parser-markdown/src/test/resources/simple.md new file mode 100644 index 00000000..3275c89b --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/simple.md @@ -0,0 +1,8 @@ +Lorem ipsum dolor sit amet, consectetur adipiscing elit. Donec tincidunt velit non bibendum gravida. Cras accumsan +tincidunt ornare. Donec hendrerit consequat tellus blandit accumsan. Aenean aliquam metus at arcu elementum dignissim. + +Nullam nisi dui, egestas nec sem nec, interdum lobortis enim. Pellentesque odio orci, faucibus eu luctus nec, venenatis et magna. Vestibulum nec eros non felis fermentum posuere eget ac risus. + +Aenean eu leo eu nibh tristique posuere quis quis massa.\ +Nullam lacinia luctus sem ut vehicula. + diff --git a/community/document-parsers/document-parser-markdown/src/test/resources/with-formatting.md b/community/document-parsers/document-parser-markdown/src/test/resources/with-formatting.md new file mode 100644 index 00000000..963743ec --- /dev/null +++ b/community/document-parsers/document-parser-markdown/src/test/resources/with-formatting.md @@ -0,0 +1,9 @@ +# This is a fancy header name + +Lorem ipsum dolor sit amet, **consectetur adipiscing elit**. Donec tincidunt velit non bibendum gravida. Cras accumsan +tincidunt ornare. Donec hendrerit consequat tellus *blandit* accumsan. Aenean aliquam metus at ***arcu elementum*** +dignissim. + +### Header 3 + +Aenean eu leo eu nibh tristique _posuere quis quis massa_. diff --git a/community/document-parsers/document-parser-tika/src/main/java/com/alibaba/cloud/ai/parser/tika/TikaDocumentParser.java b/community/document-parsers/document-parser-tika/src/main/java/com/alibaba/cloud/ai/parser/tika/TikaDocumentParser.java index cdd2d648..b3a8a79a 100644 --- a/community/document-parsers/document-parser-tika/src/main/java/com/alibaba/cloud/ai/parser/tika/TikaDocumentParser.java +++ b/community/document-parsers/document-parser-tika/src/main/java/com/alibaba/cloud/ai/parser/tika/TikaDocumentParser.java @@ -12,16 +12,19 @@ import org.xml.sax.ContentHandler; import java.io.InputStream; +import java.util.Collections; +import java.util.List; import java.util.Objects; import java.util.function.Supplier; /** + * Parses files into {@link Document}s using Apache Tika library, automatically detecting + * the file format. This parser supports various file formats, including PDF, DOC, PPT, + * XLS. For detailed information on supported formats, please refer to the + * Apache Tika documentation. + * * @author HeYQ - * @since 2024-12-02 11:32 Parses files into {@link Document}s using Apache Tika library, - * automatically detecting the file format. This parser supports various file formats, - * including PDF, DOC, PPT, XLS. For detailed information on supported formats, please - * refer to the Apache Tika - * documentation. + * @since 2024-12-02 11:32 */ public class TikaDocumentParser implements DocumentParser { @@ -90,7 +93,7 @@ public TikaDocumentParser(Supplier parserSupplier, Supplier parse(InputStream inputStream) { try { Parser parser = parserSupplier.get(); ContentHandler contentHandler = contentHandlerSupplier.get(); @@ -104,7 +107,7 @@ public Document parse(InputStream inputStream) { throw new ZeroByteFileException("The content is blank!"); } - return toDocument(text); + return Collections.singletonList(toDocument(text)); } catch (Exception e) { throw new RuntimeException(e); diff --git a/community/document-parsers/document-parser-tika/src/test/java/com/alibaba/cloud/ai/parser/tika/ApacheTikaDocumentParserTest.java b/community/document-parsers/document-parser-tika/src/test/java/com/alibaba/cloud/ai/parser/tika/ApacheTikaDocumentParserTest.java index e7c13664..bf98c0cf 100644 --- a/community/document-parsers/document-parser-tika/src/test/java/com/alibaba/cloud/ai/parser/tika/ApacheTikaDocumentParserTest.java +++ b/community/document-parsers/document-parser-tika/src/test/java/com/alibaba/cloud/ai/parser/tika/ApacheTikaDocumentParserTest.java @@ -22,7 +22,7 @@ void should_parse_doc_ppt_and_pdf_files(String fileName) { DocumentParser parser = new TikaDocumentParser(); InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); - Document document = parser.parse(inputStream); + Document document = parser.parse(inputStream).get(0); assertThat(document.getContent()).isEqualToIgnoringWhitespace("test content"); assertThat(document.getMetadata()).isEmpty(); @@ -35,7 +35,7 @@ void should_parse_xls_files(String fileName) { DocumentParser parser = new TikaDocumentParser(AutoDetectParser::new, null, null, null); InputStream inputStream = getClass().getClassLoader().getResourceAsStream(fileName); - Document document = parser.parse(inputStream); + Document document = parser.parse(inputStream).get(0); assertThat(document.getContent()).isEqualToIgnoringWhitespace("Sheet1\ntest content\nSheet2\ntest content"); assertThat(document.getMetadata()).isEmpty(); @@ -48,8 +48,8 @@ void should_parse_files_stateless() { InputStream inputStream1 = getClass().getClassLoader().getResourceAsStream("test-file.xls"); InputStream inputStream2 = getClass().getClassLoader().getResourceAsStream("test-file.xls"); - Document document1 = parser.parse(inputStream1); - Document document2 = parser.parse(inputStream2); + Document document1 = parser.parse(inputStream1).get(0); + Document document2 = parser.parse(inputStream2).get(0); assertThat(document1.getContent()).isEqualToIgnoringWhitespace("Sheet1\ntest content\nSheet2\ntest content"); assertThat(document2.getContent()).isEqualToIgnoringWhitespace("Sheet1\ntest content\nSheet2\ntest content"); diff --git a/community/document-readers/github-reader/src/main/java/com/alibaba/cloud/ai/reader/github/GitHubDocumentReader.java b/community/document-readers/github-reader/src/main/java/com/alibaba/cloud/ai/reader/github/GitHubDocumentReader.java index 1782fcb8..5d3253d6 100644 --- a/community/document-readers/github-reader/src/main/java/com/alibaba/cloud/ai/reader/github/GitHubDocumentReader.java +++ b/community/document-readers/github-reader/src/main/java/com/alibaba/cloud/ai/reader/github/GitHubDocumentReader.java @@ -56,19 +56,21 @@ private void processResourceList(List documents) { private void loadDocuments(List documents, GitHubResource gitHubResource) { try { - Document document = parser.parse(gitHubResource.getInputStream()); - GHContent ghContent = gitHubResource.getContent(); - Map metadata = document.getMetadata(); - metadata.put("github_git_url", ghContent.getGitUrl()); - metadata.put("github_download_url", ghContent.getDownloadUrl()); - metadata.put("github_html_url", ghContent.getHtmlUrl()); - metadata.put("github_url", ghContent.getUrl()); - metadata.put("github_file_name", ghContent.getName()); - metadata.put("github_file_path", ghContent.getPath()); - metadata.put("github_file_sha", ghContent.getSha()); - metadata.put("github_file_size", Long.toString(ghContent.getSize())); - metadata.put("github_file_encoding", ghContent.getEncoding()); - documents.add(document); + List documentList = parser.parse(gitHubResource.getInputStream()); + for (Document document : documentList) { + GHContent ghContent = gitHubResource.getContent(); + Map metadata = document.getMetadata(); + metadata.put("github_git_url", ghContent.getGitUrl()); + metadata.put("github_download_url", ghContent.getDownloadUrl()); + metadata.put("github_html_url", ghContent.getHtmlUrl()); + metadata.put("github_url", ghContent.getUrl()); + metadata.put("github_file_name", ghContent.getName()); + metadata.put("github_file_path", ghContent.getPath()); + metadata.put("github_file_sha", ghContent.getSha()); + metadata.put("github_file_size", Long.toString(ghContent.getSize())); + metadata.put("github_file_encoding", ghContent.getEncoding()); + documents.add(document); + } } catch (IOException ioException) { throw new RuntimeException("Failed to load document from GitHub: {}", ioException); diff --git a/community/document-readers/tencent-cos-reader/src/main/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentReader.java b/community/document-readers/tencent-cos-reader/src/main/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentReader.java index df17eb5f..38cb4d71 100644 --- a/community/document-readers/tencent-cos-reader/src/main/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentReader.java +++ b/community/document-readers/tencent-cos-reader/src/main/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentReader.java @@ -59,9 +59,11 @@ private void loadDocuments(List documents, TencentCosResource resource String bucket = resource.getBucket(); String source = format("cos://%s/%s", bucket, key); try { - Document document = parser.parse(resource.getInputStream()); - document.getMetadata().put(TencentCosResource.SOURCE, source); - documents.add(document); + List documentList = parser.parse(resource.getInputStream()); + for (Document document : documentList) { + document.getMetadata().put(TencentCosResource.SOURCE, source); + documents.add(document); + } } catch (Exception e) { log.warn("Failed to load an object with key '{}' from bucket '{}', skipping it. Stack trace: {}", key, diff --git a/community/document-readers/tencent-cos-reader/src/test/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentLoaderIT.java b/community/document-readers/tencent-cos-reader/src/test/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentLoaderIT.java index 7de87251..7d592f65 100644 --- a/community/document-readers/tencent-cos-reader/src/test/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentLoaderIT.java +++ b/community/document-readers/tencent-cos-reader/src/test/java/com/alibaba/cloud/ai/tencent/cos/TencentCosDocumentLoaderIT.java @@ -90,9 +90,11 @@ void should_load_multiple_documents() { // given URL url = getClass().getClassLoader().getResource("test.txt"); + assert url != null; cosClient.putObject(new PutObjectRequest(TEST_BUCKET, TEST_KEY, new File(url.getFile()))); URL url2 = getClass().getClassLoader().getResource("test2.txt"); + assert url2 != null; cosClient.putObject(new PutObjectRequest(TEST_BUCKET, TEST_KEY_2, new File(url2.getFile()))); List tencentCosResourceList = TencentCosResource.builder() @@ -126,13 +128,16 @@ void should_load_multiple_documents_with_prefix() { // given URL otherUrl = getClass().getClassLoader().getResource("other.txt"); + assert otherUrl != null; cosClient .putObject(new PutObjectRequest(TEST_BUCKET, "other_directory/file.txt", new File(otherUrl.getFile()))); URL url = getClass().getClassLoader().getResource("test.txt"); + assert url != null; cosClient.putObject(new PutObjectRequest(TEST_BUCKET, TEST_KEY, new File(url.getFile()))); URL url2 = getClass().getClassLoader().getResource("test2.txt"); + assert url2 != null; cosClient.putObject(new PutObjectRequest(TEST_BUCKET, TEST_KEY_2, new File(url2.getFile()))); List tencentCosResourceList = TencentCosResource.builder() diff --git a/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/DocumentParser.java b/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/DocumentParser.java index c5cb8cf8..c93e49e3 100644 --- a/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/DocumentParser.java +++ b/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/DocumentParser.java @@ -3,6 +3,7 @@ import org.springframework.ai.document.Document; import java.io.InputStream; +import java.util.List; /** * @author HeYQ @@ -21,6 +22,6 @@ public interface DocumentParser { * {@link Document}. * @return The parsed {@link Document}. */ - Document parse(InputStream inputStream); + List parse(InputStream inputStream); } diff --git a/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/JsonDocumentParser.java b/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/JsonDocumentParser.java new file mode 100644 index 00000000..2cdc8dfe --- /dev/null +++ b/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/JsonDocumentParser.java @@ -0,0 +1,112 @@ +package com.alibaba.cloud.ai.document; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import org.springframework.ai.document.Document; +import org.springframework.ai.reader.EmptyJsonMetadataGenerator; +import org.springframework.ai.reader.JsonMetadataGenerator; + +import java.io.IOException; +import java.io.InputStream; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.stream.StreamSupport; + +/** + * @author HeYQ + * @since 2024-12-08 21:13 + */ + +public class JsonDocumentParser implements DocumentParser { + + private final JsonMetadataGenerator jsonMetadataGenerator; + + private final ObjectMapper objectMapper = new ObjectMapper(); + + /** + * The key from the JSON that we will use as the text to parse into the Document text + */ + private final List jsonKeysToUse; + + public JsonDocumentParser(String... jsonKeysToUse) { + this(new EmptyJsonMetadataGenerator(), jsonKeysToUse); + } + + public JsonDocumentParser(JsonMetadataGenerator jsonMetadataGenerator, String... jsonKeysToUse) { + Objects.requireNonNull(jsonKeysToUse, "keys must not be null"); + Objects.requireNonNull(jsonMetadataGenerator, "jsonMetadataGenerator must not be null"); + this.jsonMetadataGenerator = jsonMetadataGenerator; + this.jsonKeysToUse = List.of(jsonKeysToUse); + } + + @Override + public List parse(InputStream inputStream) { + try { + JsonNode rootNode = this.objectMapper.readTree(inputStream); + + if (rootNode.isArray()) { + return StreamSupport.stream(rootNode.spliterator(), true) + .map(jsonNode -> parseJsonNode(jsonNode, this.objectMapper)) + .toList(); + } + else { + return Collections.singletonList(parseJsonNode(rootNode, this.objectMapper)); + } + } + catch (IOException e) { + throw new RuntimeException(e); + } + } + + private Document parseJsonNode(JsonNode jsonNode, ObjectMapper objectMapper) { + Map item = objectMapper.convertValue(jsonNode, new TypeReference>() { + + }); + var sb = new StringBuilder(); + + this.jsonKeysToUse.stream() + .filter(item::containsKey) + .forEach(key -> sb.append(key).append(": ").append(item.get(key)).append(System.lineSeparator())); + + Map metadata = this.jsonMetadataGenerator.generate(item); + String content = sb.isEmpty() ? item.toString() : sb.toString(); + return new Document(content, metadata); + } + + protected List get(JsonNode rootNode) { + if (rootNode.isArray()) { + return StreamSupport.stream(rootNode.spliterator(), true) + .map(jsonNode -> parseJsonNode(jsonNode, this.objectMapper)) + .toList(); + } + else { + return Collections.singletonList(parseJsonNode(rootNode, this.objectMapper)); + } + } + + /** + * Retrieves documents from the JSON resource using a JSON Pointer. + * @param pointer A JSON Pointer string (RFC 6901) to locate the desired element + * @return A list of Documents parsed from the located JSON element + * @throws RuntimeException if the JSON cannot be parsed or the pointer is invalid + */ + public List get(String pointer, InputStream inputStream) { + try { + JsonNode rootNode = this.objectMapper.readTree(inputStream); + JsonNode targetNode = rootNode.at(pointer); + + if (targetNode.isMissingNode()) { + throw new IllegalArgumentException("Invalid JSON Pointer: " + pointer); + } + + return get(targetNode); + } + catch (IOException e) { + throw new RuntimeException("Error reading JSON resource", e); + } + } + +} diff --git a/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/TextDocumentParser.java b/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/TextDocumentParser.java index 89ff98a6..9a895a00 100644 --- a/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/TextDocumentParser.java +++ b/spring-ai-alibaba-core/src/main/java/com/alibaba/cloud/ai/document/TextDocumentParser.java @@ -5,9 +5,15 @@ import java.io.InputStream; import java.nio.charset.Charset; +import java.util.Collections; +import java.util.List; import static java.nio.charset.StandardCharsets.UTF_8; +/** + * @author HeYQ + * @since 2024-12-08 21:13 + */ public class TextDocumentParser implements DocumentParser { private final Charset charset; @@ -22,13 +28,13 @@ public TextDocumentParser(Charset charset) { } @Override - public Document parse(InputStream inputStream) { + public List parse(InputStream inputStream) { try { String text = new String(inputStream.readAllBytes(), charset); if (text.isBlank()) { throw new Exception(); } - return new Document(text); + return Collections.singletonList(new Document(text)); } catch (Exception e) { throw new RuntimeException(e); diff --git a/spring-ai-alibaba-core/src/test/java/com/alibaba/cloud/ai/dashscope/rag/AnalyticdbVectorTest.java b/spring-ai-alibaba-core/src/test/java/com/alibaba/cloud/ai/dashscope/rag/AnalyticdbVectorTest.java index e1473930..1e2d00f5 100644 --- a/spring-ai-alibaba-core/src/test/java/com/alibaba/cloud/ai/dashscope/rag/AnalyticdbVectorTest.java +++ b/spring-ai-alibaba-core/src/test/java/com/alibaba/cloud/ai/dashscope/rag/AnalyticdbVectorTest.java @@ -40,11 +40,11 @@ void testGetInstance() throws Exception { List list = new ArrayList<>(10); Map metadata = new HashMap<>(); metadata.put("docId", "1"); // 123 //12344 - Document document = new Document("你好吗1234你是women12334444", metadata); - int length = 1536; // 数组长度 - float min = 0f; // 最小值 - float max = 1f; // 最大值 - float[] em = new float[length]; // 创建 float 数组 + Document document = new Document("hello1234you arewomen12334444", metadata); + int length = 1536; // Array length + float min = 0f; // smallest value + float max = 1f; // the largest value + float[] em = new float[length]; // create float array Random random = new Random(); for (int i = 0; i < length; i++) { em[i] = min + (max - min) * random.nextFloat(); @@ -52,7 +52,7 @@ void testGetInstance() throws Exception { document.setEmbedding(em); list.add(document); analyticdbVector.add(list); - SearchRequest searchRequest = SearchRequest.query("你好"); + SearchRequest searchRequest = SearchRequest.query("hello"); List documents = analyticdbVector.similaritySearch(searchRequest); System.out.println(documents.get(0).getContent()); @@ -62,29 +62,31 @@ void testGetInstance() throws Exception { @Test void testSearchByVector() { - // 假设我们有一个已知的向量和一些预设的参数 + // Suppose we have a known vector and some preset parameters. // List queryVector = Arrays.asList(0.1f, 0.2f, 0.3f); // Map kwargs = new HashMap<>(); // kwargs.put("score_threshold", 0.5f); - SearchRequest searchRequest = SearchRequest.query("你好"); + SearchRequest searchRequest = SearchRequest.query("hello"); searchRequest.withTopK(5); searchRequest.withSimilarityThreshold(0.5f); - // 调用方法并验证返回结果 + // Call the method and verify the return result. List results = analyticdbVector.similaritySearch(searchRequest); - // 这里应该有一些断言来验证结果是否符合预期 + // There should be some assertions here to verify that the results meet + // expectations. Assertions.assertNotNull(results); - // 更具体的断言可以根据你的需求添加 + // The more specific assertions can be added based on your needs. } @Test void testDelete() { - // 调用 delete 方法 + // Call the delete method. analyticdbVector.delete(List.of("1")); - // 根据你的实际情况，这里可以添加验证删除操作是否成功的逻辑 - // 例如，检查数据库中是否存在该集合 + // Based on your actual situation, you can add logic here to verify + // whether the delete operation was successful. + // For example, check whether the collection exists in the database. } }