/h{L<SSKrSSKJr SSKJr SSKrSSKJrJrJ r J r J r J r J r Jr SSKJr \R""SS9S5r\R""SS9S 5r\R""SS9S 5r\R""SS9S 5r\R"S 5r\R.R1S SS/S4SS/S4SSS/5S5rSrSrSrSr\R"S5r\R.R1S SSS/S4SSS/5S5r\R"S5r \R.R1S SS/S 4S!S"S#/5S$5r!S%r"\R"S&5r#\R.R1S SS/S 4SS/S'4SS(S/S 4SS(S/S'4/5S)5r$\R"S*5r%\R.R1S SS/S+4S,S-S./5S/5r&S0r'S1r(\R"S25r)\R.R1S S3S4SSS(/S54SS6S(/S74SS8S(/S54/5S95r*\R"S:5r+\R.R1S S;S4SSS(/S<4SS6S(/S=4SS8S(/S<4/5S>5r,\R"S?5r-\R.R1S S3S4SSS(/S@4SS6S(/SA4SS8S(/S@4/5SB5r.\R"SC5r/\R.R1S S3S4SSS(/SD4SS6S(/SE4SS8S(/SF4/5SG5r0\R""SS9SH5r1\R.R1SISJSKSLSMSNSOSP\Rd"SQ\R.RgSRSS9ST9/5\R.R1SU/SVQ\"S5SW9SX55r4SYr5SZr6S[r7S\r8S]r9S^r:S_r;g)`N)fsum) itemgetter)MLEAbsoluteDiscountingInterpolatedKneserNeyInterpolatedLaplaceLidstone StupidBackoff VocabularyWittenBellInterpolated)padded_everygramssession)scopec[/SQSS9$)N)abcdz) unk_cutoff)r U/var/www/auris/envauris/lib/python3.13/site-packages/nltk/test/unit/lm/test_models.py vocabularyrs >1 MMrc/SQ/SQ/$)N)rrrr)egrrrrrrrr training_datar!s "@ AArc XUVs/sHn[[SU55PM sn$s snf)Nlistr r!sents rbigram_training_datar($'9F GD"1d+ , GG G'c XUVs/sHn[[SU55PM sn$s snf)Nr$r&s rtrigram_training_datar-)r)r*c<[SUS9nURU5 U$Nr#rrfit)rr(models rmle_bigram_modelr4.s j )E II"# Lrzword, context, expected_scorerrrr)rNr)rNg$I$I?)yNg۶m۶m?c\[R"URX5S5U:XdegN-C6?pytestapproxscore)r4wordcontextexpected_scores rtest_mle_bigram_scoresr@5s) ==)//> E WW Wrc`[R"URSS/55(deg)Nrr)mathisinflogscore)r4s r'test_mle_bigram_logscore_for_zero_scorerEHs' ::&//cU; < << )rHr)rrrrg(\?g_vO@r8r:r;entropy perplexity)r4trainedHrLs r'test_mle_bigram_entropy_perplexity_seenrOLs_G" AJ ==)11':D AQ FF F ==)44W=t D RR Rrc/SQn[R"URU55(de[R"URU55(deg)N)rGrr)rrrI)rBrCrKrL)r4 untraineds r)test_mle_bigram_entropy_perplexity_unseenrSesIEI ::&..y9 : :: : ::&11)< = == =rcSnSn/SQn[R"URU5S5U:Xde[R"URU5S5U:Xdeg)Ng~jt@gs @)rrr)-rrW)rr8rJ)r4rNrLtexts r+test_mle_bigram_entropy_perplexity_unigramsr[ms] AJ HD ==)11$7 >! CC C ==)44T:D AZ OO Orc<[SUS9nURU5 U$Nr,orderrr1r-rr3s rmle_trigram_modelras aJ /E II#$ Lr)r)rrr)rNgqq?)r5NUUUUUU?c\[R"URX5S5U:Xdegr7r9)rar=r>r?s rtest_mle_trigram_scoresrds)$ ==*00? F. XX Xrc>[SSUS9nURU5 U$)N皙?r#r^r r2r(rr3s rlidstone_bigram_modelris" Sj 9E II"# Lrg88?)rNg"u)?)rNgк{?)r5NgL?c\[R"URX5S5U:Xdegr7r9)rir=r>r?s rtest_lidstone_bigram_scorerks04  +11$@$G   rc/SQnSnSn[R"URU5S5U:Xde[R"URU5S5U:Xdeg)NrGrQ)rrH)rHr)rr)rrg=,Ԛ]@g鷯 1@r8rJ)rirZrNrLs r test_lidstone_entropy_perplexityrns_ D$ AJ ==.66t[SSUS9nURU5 U$)Nrfr,r^rgr`s rlidstone_trigram_modelrps" Sj 9E II#$ Lrgqq?rc\[R"URX5S5U:Xdegr7r9)rpr=r>r?s rtest_lidstone_trigram_scorerrs0  ,224A4H   rc<[SUS9nURU5 U$r/)rr2rhs rlaplace_bigram_modelrts A* -E II"# Lrgqq?)rNgtE]t?)rNgF]tE?)r5NgF]tE?c\[R"URX5S5U:Xdegr7r9)rtr=r>r?s rtest_laplace_bigram_scorervs+6  *00?F.XXrc/SQnSnSn[R"URU5S5U:Xde[R"URU5S5U:Xdeg)NrmgQ @gݓz!@r8rJ)rtrZrNrLs r&test_laplace_bigram_entropy_perplexityrxs_ D$ AJ ==-55d;T Ba GG G ==-88> E SS Src(URS:Xdeg)Nr)gamma)rts rtest_laplace_gammar{5s  % % ** *rc<[SUS9nURU5 U$)Nr,r0)r r2r`s rwittenbell_trigram_modelr}9s "1 r?s rtest_wittenbell_trigram_scorer@s1D  .44TCTJ   rc>[SSUS9nURU5 U$)Nr,g?)r_discountrrr2r`s rkneserney_trigram_modelrrs" !DZ PE II#$ Lr)rNg$I$I?gm۶m?g$I$I?c\[R"URX5S5U:Xdegr7r9)rr=r>r?s rtest_kneserney_trigram_scorerys1P  -33DBDI   rc<[SUS9nURU5 U$r])rr2r`s r"absolute_discounting_trigram_modelrs +! KE II#$ Lrrbg?c\[R"URX5S5U:Xdegr7r9)rr=r>r?s r'test_absolute_discounting_trigram_scorers1J  8>>tMtT   rc<[SUS9nURU5 U$r])r r2r`s rstupid_backoff_trigram_modelrs j 9E II#$ Lrg??g?c\[R"URX5S5U:Xdegr7r9)rr=r>r?s r!test_stupid_backoff_trigram_scorers02  288GN   rc<[SUS9nURU5 U$)Nr#r^rrhs rkneserney_bigram_modelrs !j AE II"# Lr model_fixturer4rarirtr}rrrz*Stupid Backoff is not a valid distribution)reason)marksr>) rVrWrU)rrHrY)r)r)w)idsc^^URU5m[UU4SjTR55n[R"US5S:Xdeg)Nc3H># UHnTRUT5v M g7fN)r<).0rr>r3s r !test_sums_to_1..!sJkU[[G44ks"gHz>r)getfixturevaluesumvocabr:r;)rr>requestscores_for_contextr3s ` @rtest_sums_to_1rsD0  # #M 2EJekkJJ ==+T 2c 99 9rc.URSS9S:Xdeg)Nr, random_seedrHgenerateras rtest_generate_one_no_contextr*s  % %! % 4 ?? ?rcURS/S9S:XdeURSS/S9S:XdeURSS/S9S:Xdeg)Nr text_seedrrrrrs r'test_generate_one_from_limiting_contextr.s`  % % % 6# == =  % %c % ;s BB B  % %c % ;s BB Brc0URSSS9S:Xdeg)N)rrr#rrrrrs r%test_generate_one_from_varied_contextr5s!  % % ! % LPS SS Src[UR[S55/nURU5 UR SSSS9/SQ:Xdeg)Nbdbdbd)rrr)rrrrrrr)r r_r%r2r)ramore_training_texts rtest_generate_cycler:sV+,=,C,CT(^TU,-  % %a:1 % MR  rc6URSSSS9/SQ:Xdeg)Nr)rrr,r)rHrrrrHrrs rtest_generate_with_text_seedrKs,  % %ars     4i N!Ni B!Bi H!Hi H!H # seQ seQ  XX=S2>P( #  seQ "Y#"Y # seY   '010U2 # seY seY sCj)$ sCj)$   # seW   )232 T2+ #    se34 sCjLM sCj9;3<=<( #    se-/ sCjAB sCj24? #HI#H #    se79 sCjKL sCj<>9 BC B #    seU sCj%  sCj=*!*+*i ! ",  *++##C$ $ R1  : %.:@CT "I 3r