From 9c7b7c90c686223970bf9ce46f2e8bfa1bf4dab7 Mon Sep 17 00:00:00 2001 From: Martijn de Boer Date: Mon, 11 Aug 2014 05:23:44 +0200 Subject: [PATCH] Updated Frisian stopwords. Should be checked but have been extracted from Fryske Akademy corpus. --- src/applications/ExtractStopwords.hx | 2 +- src/applications/ExtractStopwords.n | Bin 30445 -> 31086 bytes src/linguistics/languages/Frisian.hx | 10 ++++++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/applications/ExtractStopwords.hx b/src/applications/ExtractStopwords.hx index a954576..180235f 100644 --- a/src/applications/ExtractStopwords.hx +++ b/src/applications/ExtractStopwords.hx @@ -43,7 +43,7 @@ class ExtractStopwords for ( word in list.keys() ) { - Sys.println( word + "," + list.get( word ) ); + Sys.println( list.get( word ) + "," + word ); } diff --git a/src/applications/ExtractStopwords.n b/src/applications/ExtractStopwords.n index 814a8329d976f9f5a25e5ccc02acace2e57c3fa6..c1be5847a8f8a30f7f5fd06cccd7ebb3eddd12ce 100644 GIT binary patch delta 2625 zcmah}3vd)g8t$ISCXn|7j$|ewz1f5m#3UF<2muKoK{O2s#K(yq$s{|wJNuYfcPFl) zETfiQoJ75^TTYywqo zb$|PR|KsbwdwS->5AjEz;Ji&8on2R8gg!tB9X*gIdxpp6AtX)ALy~kvSW+oKF@P+i z4l5wF3ckJ+Vd>6NBuje)bp*UBAfpUG5wJ=?hX5+zP64B3P_{w=_)-DoOI0Mvok-J6 zq6LT+g%Q=AU7o+=lBfy`deFK-dmtp803zi&@vJ%0a8o%1vQUQ^j2g+1^g2)pRe)~` zs?|wtgw)!uQj+u=VbVWD{GjgFY~t5pfT<^lA0mDW2DmI83j8Tegdmwe4PgLq>JcJe ziNFsP0T2mTT7nD*Ea;I0wLucp!5Iu20G942!9mMULK=(&OdDp3gmm~fbTLhVF_dS`w%wd_yoGWylPPsHua`gcK8cH!Y!R zfdcGZ7ciV87MzC_fMF9WMyxnVK)*?iBy|`D%qUPJN|Iq{G6@|eLEhGgtrHuVov;Q% zM7|2yNpW^5O-uLzO&b6v^Z*o=NEt39;JKxsEM-MWYKXL?#1j$#2L`(Q@hBwlX9I*E zvA-2Z2g|c23Q{$usw)2xX$n^aLD=^yYksWK^=-3VyP1Q6=2T zn*>Z$!ksKc*KO14x38(+y10IM@AdURtCK#egzPzl{js2WxQ&2q1IVp268Yo&X}*s? z%MbA}{uJNK9e$AC&Uf&;`Mvx>ejEP<-^m$&00bWrBCiP%eTGPFn#yB*obTbk=o1hWFzAI% zZ}-C6_*XBy&c_AhSAnmw3ZhE|Yy~Rkp@0$9dj~Ajc5W#=g7KhpUGYxOtfR{j>dMWO z*~jxr9J%xf8Rt9WsuyQBRsY;fz^`dP!xi(L56U;>;~mb@y4Ol^i<7hNr+DR(N$UIb z4(X)>bYWG9p7iw?8xTdbvEe}O#MLrYQdC($eSI(Ypyx*D$q}kNLVf>zYe~j)yLXLl9hFI@4qt|UIq_H|v4zh*hzoA4QSthBziAH?f75>Yvuo-mDQf@Nm^BM(zGb*yW%u@% z6{!f+zpqd+*zAS=k$hE7^JcI@zmG)*?#s(e^JanfP~Pk`PqxT&PW#7kbB=KDdD(0g zYd#tNrBD2?eZT!rlT^(Wv7O7tpdU3)*eUInGhpYlHv&C*1!;nXtTi}VRwO)^9_!78 zoZ_@-2|F6xH)oC;EqQYib~Q?snh~nbWsimi*xkmuGTGNZa`=B#ErX<~XGT?p^%{*u z>I`o=I~}fIhpkf97A}xfmE}Y#*tSR!yE#$|t}F1A=WX!(GhBGqQ$p3M8B#uWBC@D_ zRUbmW+8tYSs5*14YNkcXag9h@zW1e#$ahEQ8}D2oZMv?qCw&(Spt`x?JmncB&!m^s zLP-rf7|s4+d;hTt-5no{t~k3HnW_ufeyfJ9j(FJWSf=;K^uJ&y%v#9wFc!P`N3-8` zvvbXs^G?5-^_Z~>cwf1oHyn4LRIYvPcE$JnZ@yXyXUDe3E1><0p#3`7%zN=BmLJc< z4Q!V&pS8ssy$#ckMs1wI7el3w?TxneZour)V$GajMpQZTA_1g^{`FhV3KBKA%`MqR)fKLBHC0!-MkiG-ag7G5t`Y|5 z)MT}x^L5&xD=(`~7UI>cIa!a_uz_SVUduMybvc&%4LL%ox{i${eK~Ped`1Lc51Qr0 ii?97d=H%{{8_uO&=BAnF_y)n9o9$H+p34s1*z_H0nMrs6 delta 1900 zcmZuw3s6*57(VANDsY zd-XcKa)s)bmXs|kp@jTS2-(x8*JwBIi6=z(BAy7sk37QD8dBnG5OO(RVkT7B zY9<=t0H+5ze9J+bfFNj;&9t`#e zKMHOMb_TZvUk&aGzJ(PIOb~YQEaSL#sptl|JIEo$iZGo+o)s6cTk+1jt@zShrN(TDnJ#7C+TclX^1IO2e*qX7<86 zNl9wFr8J5zQ)@H6Gt(S3XYo_CVD4N?Hn!*csD?fp?-U_eP71Qe>a}dp&BjznM`%VAY@7{VceSi`BpRS1$vF0{h zt)Cq!-W^H%7qXolk>b4(J_IM6AL=O%zOP6E>}=@QhmBISsMyq}*Z0yn6S%zply@E< z@!p1=HvYTtvit9ZXE=V3GmkraG$;jfbyT%HN!Bn)f+0@|^t&IAdenwQw*hosGpzC$jALykC4lTs zgf7 zOylsFgp18J z!#31z9F8nB1@ACrdSFGnXgmD+odP_`m3Q<;>7K{Cze zG(lsl{NQH#@cSP9@0blQ2U6&qP{~>*&E)~DhRi^6SQ(S%h4f4&<=$_S5}rpMZIXxQ zutHCZf#$>EmJC_|ms;jT@K5(FV^R^;%25M7bA6$&nM-yqL367%f`8InS%Au9)65k| eCt%{n2Z;-LBA2{UA&9N%>R@Y;Kn+mPmh~T|);iAs diff --git a/src/linguistics/languages/Frisian.hx b/src/linguistics/languages/Frisian.hx index 7ed601a..6ec5425 100644 --- a/src/linguistics/languages/Frisian.hx +++ b/src/linguistics/languages/Frisian.hx @@ -17,10 +17,16 @@ class Frisian implements ILanguage { var tokenizer:ITokenizer = Type.createInstance( basicTokenizer, [] ); - // TODO: Stopwords could be improved + // TODO: Stopwords could be improved and should be checked. + // Extracted from older texts using http://argyf.fryske-akademy.eu/files/tdb/ stopwords = tokenizer.tokenize( - "de fan in it en yn nei is foar troch fan op as mei jo dizze net" + "'e 'k 't al as bij by dan dat de den der dij do doe doe't dwaen dy dêr dêr't ek " + + "en er fan foar folle gau gean gjin goed guod ha har hat haw het hie hiene him hinne " + + "hja hjir hoe hoe't hy ik in is it jild jimme jit jo jou kaam ken kenne kin kinne " + + "koe komme komt lang man mar mat mear mei moast moat my myn mynhear neat nei net no " + + "oan oare oars oer of oft om op ris sa se sei sels sil sjen soe syn ta te toe troch " + + "tsjin waard want wat weard wer wier wirde woe wol wy wêr wêr't yn âlde ôf út ût" );