From 368a98da0732368ac4614ecd10dfc6fc019ccd7a Mon Sep 17 00:00:00 2001 From: Feeelix <44130555+FeeeeelixWong@users.noreply.github.com> Date: Fri, 15 May 2026 18:10:32 +0800 Subject: [PATCH] Add review calibration bench --- review-calibration-bench/README.md | 35 +++ .../data/sample-calibration.json | 171 +++++++++++ review-calibration-bench/docs/demo.mp4 | Bin 0 -> 38960 bytes review-calibration-bench/docs/demo.svg | 29 ++ .../docs/requirement-map.md | 30 ++ review-calibration-bench/package.json | 18 ++ review-calibration-bench/scripts/demo.js | 16 + .../src/review-calibration.js | 274 ++++++++++++++++++ .../test/review-calibration.test.js | 49 ++++ 9 files changed, 622 insertions(+) create mode 100644 review-calibration-bench/README.md create mode 100644 review-calibration-bench/data/sample-calibration.json create mode 100644 review-calibration-bench/docs/demo.mp4 create mode 100644 review-calibration-bench/docs/demo.svg create mode 100644 review-calibration-bench/docs/requirement-map.md create mode 100644 review-calibration-bench/package.json create mode 100644 review-calibration-bench/scripts/demo.js create mode 100644 review-calibration-bench/src/review-calibration.js create mode 100644 review-calibration-bench/test/review-calibration.test.js diff --git a/review-calibration-bench/README.md b/review-calibration-bench/README.md new file mode 100644 index 0000000..95b0d70 --- /dev/null +++ b/review-calibration-bench/README.md @@ -0,0 +1,35 @@ +# Review Calibration Bench + +Dependency-free peer-review calibration and coaching signals for the Community & Reputation System bounty. + +This module focuses on the quality gate before peer-review activity increases a researcher's public reputation. It compares structured review rubric scores against consensus panels, identifies leniency/severity drift, flags reproducibility blind spots, and emits transparent trust-adjustment and coaching actions. + +## Run + +```bash +npm run check +npm test +npm run demo +``` + +## Demo Output + +```text +Program: community-review-q2-2026 +Status: coaching-needed +Reviewers calibrated: 3 +Coaching actions: 7 +Quarantined reviewers: 1 +Top reviewer: ada (trusted-reviewer) +Top action: Review practice set: rigor scores are higher than consensus. +``` + +## Files + +- `src/review-calibration.js` builds calibration reports, leaderboard scores, coaching actions, trust adjustments, dashboard summary, audit trail, and stable digest. +- `data/sample-calibration.json` contains synthetic structured reviews, consensus panels, reviewer modes, and contributor signals. +- `test/review-calibration.test.js` verifies calibration scores, drift classification, blind-spot detection, quarantine behavior, leaderboard ordering, and digest stability. +- `docs/requirement-map.md` maps this slice to issue #15. +- `docs/demo.svg` and `docs/demo.mp4` provide a short visual artifact for review. + +No real private review content, identity secret, or external service credential is used. diff --git a/review-calibration-bench/data/sample-calibration.json b/review-calibration-bench/data/sample-calibration.json new file mode 100644 index 0000000..915de21 --- /dev/null +++ b/review-calibration-bench/data/sample-calibration.json @@ -0,0 +1,171 @@ +{ + "programId": "community-review-q2-2026", + "asOf": "2026-05-15T00:00:00Z", + "rubric": { + "criteria": [ + "clarity", + "rigor", + "novelty", + "reproducibility" + ], + "weights": { + "clarity": 0.2, + "rigor": 0.3, + "novelty": 0.2, + "reproducibility": 0.3 + }, + "calibrationThreshold": 0.7 + }, + "projects": [ + { + "id": "project-organoid-benchmark", + "domain": "biology", + "consensus": { + "clarity": 4, + "rigor": 4, + "novelty": 3, + "reproducibility": 2 + } + }, + { + "id": "project-graph-protocol", + "domain": "computational-science", + "consensus": { + "clarity": 3, + "rigor": 5, + "novelty": 4, + "reproducibility": 4 + } + }, + { + "id": "project-open-dataset-release", + "domain": "data-science", + "consensus": { + "clarity": 5, + "rigor": 3, + "novelty": 2, + "reproducibility": 5 + } + } + ], + "reviews": [ + { + "id": "review-ada-1", + "reviewerId": "ada", + "projectId": "project-organoid-benchmark", + "mode": "public", + "scores": { + "clarity": 4, + "rigor": 4, + "novelty": 3, + "reproducibility": 3 + }, + "comments": [ + "Strong protocol trace, but execution container still needs one replication note." + ] + }, + { + "id": "review-ada-2", + "reviewerId": "ada", + "projectId": "project-graph-protocol", + "mode": "public", + "scores": { + "clarity": 3, + "rigor": 4, + "novelty": 4, + "reproducibility": 4 + }, + "comments": [ + "Good evidence paths and adequate reproducibility metadata." + ] + }, + { + "id": "review-bohr-1", + "reviewerId": "bohr", + "projectId": "project-organoid-benchmark", + "mode": "anonymous", + "scores": { + "clarity": 5, + "rigor": 5, + "novelty": 5, + "reproducibility": 5 + }, + "comments": [ + "Excellent across all dimensions." + ] + }, + { + "id": "review-bohr-2", + "reviewerId": "bohr", + "projectId": "project-open-dataset-release", + "mode": "anonymous", + "scores": { + "clarity": 5, + "rigor": 5, + "novelty": 4, + "reproducibility": 5 + }, + "comments": [ + "Dataset appears ready for reuse." + ] + }, + { + "id": "review-curie-1", + "reviewerId": "curie", + "projectId": "project-graph-protocol", + "mode": "double-blind", + "scores": { + "clarity": 2, + "rigor": 4, + "novelty": 3, + "reproducibility": 2 + }, + "comments": [ + "Promising, but missing a complete notebook execution trail." + ] + }, + { + "id": "review-curie-2", + "reviewerId": "curie", + "projectId": "project-open-dataset-release", + "mode": "double-blind", + "scores": { + "clarity": 4, + "rigor": 3, + "novelty": 2, + "reproducibility": 3 + }, + "comments": [ + "Good data dictionary, but independent rerun evidence is incomplete." + ] + } + ], + "contributors": [ + { + "reviewerId": "ada", + "roles": [ + "review", + "validation" + ], + "completedBounties": 2, + "endorsements": 5 + }, + { + "reviewerId": "bohr", + "roles": [ + "review" + ], + "completedBounties": 4, + "endorsements": 8 + }, + { + "reviewerId": "curie", + "roles": [ + "review", + "reproducibility" + ], + "completedBounties": 1, + "endorsements": 3 + } + ] +} diff --git a/review-calibration-bench/docs/demo.mp4 b/review-calibration-bench/docs/demo.mp4 new file mode 100644 index 0000000000000000000000000000000000000000..e9f885b643f01dee7594392e29f24562b3dce82d GIT binary patch literal 38960 zcmX_nV{|25wBU`^v2EM7la6iP*mlRZosQM9ZQEwYPRE{nZ|2QfRp)H;)UUG(000n~ zxp+EQI@{X<0HA>X_V1U)$jyY=)`5)~002OnIRSwHKs>9hsgcXKObsN&*H`7H_-WVC znp8_F-73)<>D9Fd3kMq!J&}pM6Of3RgPq8Um4$_c$e5L#gNfbfn<2&UEx;(JC@x9Q zMkK5*`YmY+H2G$T+Bz9wAio9BgUH#{_#5IRa&Y$iR{dTLoJ{$d8JND6z84}}OAnx_ z!G9K+zcmb;jO@&T{LCCgCKgWiwnhfus?0<#PCy$QOXqKh%Y)0*#N`_>akS-U`VPU! z)XUxu$j`z|$IMJ*X5{Q*;NWa+>F}S#|25#~U|?@%<_vVDig~-{)(&Rgs|0OUH**X1>6BA2YBbWaIv9xmmI@uU~3w_g#ZCssU}Y`Y%i;qyK8^1a!9eR(CQn`2Xqt$9FQ}H*qo}vNirL?0;!}Bm69^ z3`|6h|0ToE#K7?lI{at&e|94eeh%($g0lLN@@_%W(2^M2zdfm@UWp?(43C znp6_)H*@4o008WNA57JY2?(*BhN$3D!~dZP|NITgG7TUj1%QuWJve=X1mNkn7=h`K zjR~#iv&zqPvBvfA;`2K;-I?s)I8Sv~ZON>*ac=*erLZ!9wdstasj4ziAO@wCI2U3K zLDHKy5Er)~a{vWe7+ZRC$1#tgjXDSbh`e~Uo4^MVxuYe=8DQ@CVH2O^#UiwC2=w6> zy!jYIvmH7-C8^%xT(WqhcnXbJM0*PX06(nGOJ6bEHBJxOPX}^%?;L8L+4Yf=t1la5 zIXI59TT5R_?a^f}iLyiH5BT^>C;egl_e%f(ej@>{rsQ4#m|{MBArWXrM%?U{*t^j< z^s(Z}#5@);cDj(-FPN>Tu_0PQa?%8v!LT^J9_3>H@ryZ*7@yaUx=P!Kov3%rFZwt* z>P!_OaO8jU6frh(2WSe}p!V2Kq6PDMxxPpmQhm#Hg?Tlj*laP?X+p_LJZe&eW4XrE ze|r3J&Hc$~CruN^cLE-{V0vc9cMOHV07#=b_oeV*o{U(~u*fQ!Oiq~|GCv9Laf%je zgjm3D+)3ey8RrkbzJ8Z_)L#f5?^|z}QzSw!gU(=;kdS+4QYv)8Bqp~yqcASa>w(wy zVr;kS1c(u_x7(}oyL^)KX<|$ii6ETYPM&-F2wlI3e%+pd{XE4;+yWEWc}UyYZZY*} zf_@&m(Eq7qpEGT6VVg1^6TaT+H=27!EN1hyNf^NlV-FhPU z^u(Fluy-ZJ8owSfZ$15Q*R7P>KDlPQIrC6lX-b_xw05tQV&^q%KC(b;}t0fs6 zA76=}Xq|A|ygPnENux+amZ2;!b^U$S^r-AHnU~cCr6qox#EUf&i>?HPi=FE>iP_Ob zhy7osN<+BzGu|KETFW&W%%p*vr@5epiD@qEEfS`pHz~=OcGI}J&Qdzg+H%zQFtH0_ zWzIrGmECdUE;6i;Y{S#*eq2YVXS000&3Qb&BDHjAz0(nl?@K8a7PsDpU7tqmTVeVn zwCpbdDjlt7{^SxbmLP%7R2~WXvp>S^3X}6plpxX~Li1OTdq@M%U}El%zN+hE<%D0y zXr)Pe`A@{65xPFVavt)eLYB+#j;~V999PG}C$QD(+}5Dyxnz9rhL}a0S8@cq&uj%# zh;UK#iCR>qE|^uQ9h*rfEg)}6w);*X8qbWG+;XNMqd-R0vNNzvQ<8AbtJ5F_${bVT zyuD(kPHe~oAK46~k6)@l`|M-~Bc6)`@!lN&6gond8jL3W>4H!h$CG4^t<(jU#XoEO z2~`Me0Ym!1&jd20awQ_Jg2E-S&I;uvza4F`)YIw5mXUP^WZuw6Es-2RPNa^@LNbI1 zc3A3Hk4y0Gx=Skt*eqIVxGP!Ro-gpm`?|O-R)aJ~5gd>WPTe=2(6!~V|BG&B?Yi~L zoN|w4zt*?d2~c`=qhF>I#BZFsA1iwIT7L^NC*}>2E4XbWHobdx-?l)d3>GiN>jp-2 zjBHTsNlNWHz^SE^Erf1X<|j4^fU)gTss;ozp0=wXo-)0;5QQyjyE3O7rctTy%vT{@ zwwB4U->4H~;?_j(-86U#TgC%aD{US@2yO2L3Ofduce!eoM^pfoB zdx2~5@EO%luH1S!T6E8mQO9Pi!T)T&Q?Z3$ns<$b?7fiuCt)_zNRf@nw0zZ{S+G!w zL;cve7 zPP`R}MK>*2$vL!vozGT%QTCtib1_w!EeqNBQ2n{vx=71|4x*;aisA*x7m=q}4Ql<0 zbX_%;QH+$5`aV4x3&_5lFf@sfw4!oj*3EWg8=1w`qk7?5zx=lZw)c(u8n`G^+}@#O zzdb98TjEkwX7vE58(QZ6xmSFigZdgBVFkYR%>MazWfDHcZjA7Htmgn_Vcayo{x_TX zUiNDg8IK;&gO_$$Q(5@sAzg*pJ=Ou+B)tW3&>L{CG_AX)>u$vBcCo35rCk+#Iwp|N z%*WiBl-swj*VxlkVp*{{!MH3{A+Pm)Q^uFeo>dh+oe6_! zf?McU1MRHVnvm(t#{#GC_TKrJMJG5(bh(yDj1%fT&!J@pGJwF2_U=#n@CUueOR8P* z=&sUNk;uYu1;fudOWKvLpVYin{lB`XJP+P zatklXWr{b=xEpBFZ+0yW#F zI6{m7!PoBLTUJPUlG%sUw;Pc;7Mzi21^|-Nq3r1}k%S{gn~{r$bk(7QXGBmkKS-E0 z@!DVk&h6ffIoJ1UFE zTZHyNS4slK^)a9^n;n1gG*Fw^}THXN9K@fAqRkxKLe3vX!fJ+h@`oE;D9c3)HGS85(4fj2r(!Y*K|>z%`)e?xRt7Xh zDcLEY(LpQ*IU-{tS;WZ@0%b27nxc~@gpB@t_!i^clSVGys*l+; zl6!63_Fpuo@8o!YnqG(w4}~KW+E`&U z3R7WLBDC^*_ntu?l~}6as;qu8huGj@GzeOAYw;_my9E)EEDoo6(qw3+Om)EN7CHoL zK8+PKm&cUc*@V;7Ix@c?!TLMAU+o|PhUti29Jz6Bo|L3mnHgY9&mPldS*#!ET7%g# z<68Mq)Rp)H4GTKLCR27zJB@ju_2ag;)GgWcv#JjJu7XJO7u4Az{k2tF}q_-{-mn-aoX>b^Qtz|FzOJA`= zX85=+<(*Fv2c<00!&}V#esmtuF|Xkq!OS94{1GnzH@TRn635n{pa;@9rDR(|lnqa5 zQSlPJ!@brY89?T)EtBpCt$|qF#=awwYr^aR9(YKC$S#~i(|J2Elk%h1!+1Z{g785u zjY|gjKv67qS-O1fVtg+B1iDAMGSCdH=|sa0E#FZ!bC^`!4UBw(6&h&g&s-7dAS^VT zUSi3C%sfQIXm33z{@^|eBr!5{is}IwuvQ})X`1UbG0)Q#=HDyCoU2rNRN?fBAG)X? z4O%h=)qL^>xu7=F?_vd*$-`}b{YLWuj9)dC2(`=U`g)9Zm~E4Pxl75y)|m3|80tA% z3@pT1?+i)o+YsivjKNLmlAcNFQ7hed?A&2g;1CK>OAnR&^LIJ_xMLvulVfeIH&BOO z?ntE7rg^=>GzW<0z((+q%H2#S@x+^R(__RPiv%L$-URia7O^MlE^}@$p<1;s3mTNa zag*4ch>kfc{EgX#s=#anI~;|y6?(f&aEy;UUk2t4sPyGb+=fwmjhW&9U`ZHQXljhA zT!oT);lAJ1SkY&hC}hHlgVq#o2T6Io!CnEVAXu(n#0-cAABQa^$iKu;CYzSVk6(WM zK-a&ep`BLuyUr2m{pXvU)&b-8#fT?5_5svutqk6TuL4BqrDHyYOvnVI zXG!>&`w*~ZnA<)zX*G1)q^gKGcM!Y923z+93GvG^5%c^i4N^pTBP}DZM=;|MYIMn2 zRmv3jwO$GaPYLU~XWqGfV-KYq7Kn+>l6*1WG2G4E%+uTcw$R55Y0A$5&$0ytVDu1m zlVj=Zy5~5LN@LKNew~s*b~Y`4<|>Q@I{+3R54&6&Oc+y;_pIrjVdN8!bPO7uZQ4`Q zCJ9nv*b&*`Qb|<6MrT_W5>CfM??FUXF>%opXEu;3ZLK77TkMlSvGce0>82w)TRw)^ z#FCd~@wyR&cg&cA3lG@ykd`P|qssC%daTC&yCDU#j$psfG(%*LudD}ggBYfW+{UP~ z_6=QhYw~r(YZwx_0Bn8}58BugYsc}X8!^6oW--{GV4*dSsG|aJ_doD5UHm@8K#mN- zcB9i49}h%a>J*j`pn-Yg&jjy@>0+`JX?A}5?v0eoPF48Znr;$~-|FY*nKG zo|rlvYS)rr@jK1mt3Sv|m@7y1a0cF%3UBQ#KCSy%lC&1@dz!l(yJCjFQV`nhW7M16 zJ2P;*djIv*J8G(2nsMRLC_qX8GXpK~tkn_8R&(Z62d_pQzLeTT35pYiQ*2IViA-!r zJ^plc@ua>y4iK0l8xs+4Kn55_Z0Me?+UL!=e*I1SYIE~fh&-ZWh`YLUB%_y5+Zy=? z?m1=vB)wh_llBsnJQ&Drx*5e{$#M$o9f)B%{iiy)TyzwXL37UUw^jc4>kb)s_eYzH z@!SW#gU)ps9wYb)_lS05NxaBJ=FfCl?8z3_t9sglD;KJoirlP*v>{WN^rXy~ zFB2K3CjM{Cx>HLPks%On1>9wJXJH$RS0k1=&jL*VJnl0xS6Mk}@ZupbYV4Mk7j=9e z50Ao&&vwQCbzymH7P<;?qU?I3K^e*m9pSmNS%J@Ny<%^Zu$t?q8{Kn6IxCUg{e*o7 z#0CJ%3A7R5v0H=^2DP7k{%CpKG!C5sM(h2lJy`*YQKcNV zTi2TmKI#}u-6fN|ZD(Sp;5jC1jHcM9H|_Yvw>V9dt-)n91DBu{*?##^QS{R4x2ZQb zw)|iL8+jnBdJYrhE>Ovqt3T36r|SLTN-qhsc$YEW#bkia9FI@`<^cYocY6%<>Rmu~ zWW9+D)~yj!6_d&}H=A>DOMTNA+t%nJOf{@!c6T+GFa6N2B9V!p!-Os)^J>p2`e-=Q z>6~)*M|#=rNhaGJFN0{bFSTpR8=%vZpFuwRCuOLzFG8#Ua&IaE2emf+#u=LF0lQJa zo&jQ2KN^yhS!ti{BvHqu(GP!B72A>Md@c=?NA=T=yUFg?8}1Zf38YgMwdDBf6)dO` zdmLzpFQIOK{#z`Wwy@u+`5Wk6Gf=^F#UN7q8sVWm3ORbP80-ww_Y*dU`|NOQR1xIp zg=+6>KQ$`U@3&VW3UeC$n&i!r-`iv?|I(I)ytkZKYcsyTRC1{a$5=z1ik67BAzSC> zioA^>0P{TNvTiBQa5pwWH`Hq6WDm9Dqt=n9mJQT}MhAym$k!R1S5DP?d89YjHT^RL zH4c6!N^3FcWAC?Fa(SH<;cJ|ww3LIq+wH_JW1>`pmP#{PD=5;WzQ z*A#7#-am%9Y)v{qv$FJXnCO#0y@>LuExG;q@N#m_gxs~YWd3iBT?y!Dy!F?BU9JP1 zxUWwrf{aDZCkQP;^seI4N!R({v(FtI1T^u1Bgi{RQA1W0NN)y}(76kVETJy721#lY z+HiL0e3pvvn4x~@Kd)Cde;UTOt8Mje!{nT+s1yu|FURs?aW#peWrj2B3eF#YGvD-O zaWg3HW-?O8Ly4Y^7FyyhtllISaYRhq?XQ*XT<;0)Bp9{X7yHASDKSRu%Hbi>DcFJ5 zjY7LeT&p!1>**GauxC8ByYzLg&>|OoSZEw5OsGtrfK9u@o2~}FZwdtY0;ZrQqf+W9 zy6{JB0B+i-t&m_^>-tKj$2;b46c-Da8ttBc-S{hLt56bex*-+n+MyqENf^yjR~$Ib zo8NfYEpMz9`x0?h4#u3+-)>8^m5{@f8|Dmt6biN<%m~JH&$jY-+5Gxi3o-1D_a2n( z7;lqB7Kb$cu>woC$+Zu`B}YywZ&Y&~sqLJ{wZ$X<+!~`DXH1}c!U}zh zE{6vwDSxWXV|ei@A_Lcdnr-pap#81q;8D94>HhH={i=V%Baq~z7FkSIs>R(tae>ZW zB3x8QkR*X|??WaK?{X;vXaF|$7(5{R@qj5!4K_QSWHv?F+~q)HHB-_XqcEc|{2C^< z-B&2|cmMS=NdT8^{5+S4E3Wd#Z_Bu3Wx=WAWoDmi59O7P(;2MM12~$3^SSQA%orqV zHJ}T67csN34eOq*HmLDo!`AE!pG8XHTXf_aK6OgST>$94G{A8XE z_ow^F6xI!6!eB?>YaNB2z8T$}&gdO=ZYpBB9JFRt@Ul6@CDc2?-Ozf`=v0F4qzid2 z02Nq14$YXm*PDP@5QTJ{yEmPAN!uUb^+!gM(59TH@=RCqx`B|p0SbhXG?^v$NNgn) z-?&JqBNkEy#4OZ8c^}2-d;lBzt5!wkZ4o%g9M~P?eBv>?d6Atf+PqAzT<3-Qf(;NS zJ4wFZR60v+J(Jy!%dJRnX`&U(I7~D7oiApsPsbk$e}iot@~9fr_s3YZF*OEX-h3 z!R^ErZXxum9zNGh-aB@(_$sMghYEbyliPTb`BSmy{md1gS$hGZQli{yZ}+Q+DQH+k z!v>RqboYMow}t&HGG-{11fM@Ifg~uTRVsm5{b!gl48XpN|FJ-VNRfIhfOSjj@@ zG;Q<62z>=^d3}1yo9z|@99F+dRw1hqA+nCXrX?2rIbF*fvyRiG~2m`BdA5O(DRvnKWvVzX-0HR~3%eFjGlWQbG9wug3w9I?t%AfhAy zha`Ktcr5h2S&!win*_x@_t@;P4U-3_8zaBbs$tzD?VR@1rt?V~UEvW1kt5zQKQ(4l zIlWip{%rx5epxMj`kn_Nls1(t-QCt+r7w=F6$XXH`aoX@Fm1LM_92ho`(!Lyx0 zGfmi7C}*#jYQ#VgpPG-r;m_MYIx-T;xQ|&@82f(5vM$wmAWUOL^ea|tP;<6HY4QT_lNww|rP(D3c;kGg><(;W%xqOvrkpi>cJPy>02tCz| z4|yJ;E_YY~V$<63pb6QhdDNQ`lM zRZ0W&bg-oeJsz#O3N-qTEUaFUj#&GBN5-P= z>G6ElAE2ss@aA%yGABfCYeTP@ux`I%ZOW;g1%*1h!ZHR$h^ocNY-`aTJW3ByPZ}{v zX%ZnoX;Al8dezFX#O5Nez!9Dn`XO-{xL^CJI(rJS^A@=G;{Kb&Bza4Du*0)+#^Ot; z43_=SB#rM;N`gfg^I}JU+zizp3|!#13jNiH&&@mNm`FsqhtU>f@UII0ilMTB29KOV z6Zu<`Ni)rEEBV}wus~9c<;Ici9cN}EVw!0Y$V0K6f0m+6u8*11n-`RWWrX?V?<2ti zq_I2*#y8>4&VdzoG>KYI3us00T$4oM(&GyX8S&QK(DReiwwx!9;;mjJl3mp*$E6Zf zJn{==Nu1i5_~E6EDq96#EK%-Q)%UXRHmuY!RbQhAk~9}NCz7JBHG*;Aa-DU<(p)AO z2xotC3374C$*&Re?Z9Pha3~lc^c5WDcXv>BYJX;Xogqem>xC5fx*XYFUNd*_eSpR{ z#_T_A8lr_jFdCGaFrMQL$k}$5t70mnV)ZoQL>gtE`<5p1Uq5>y-Z89^O5cWl{pnJ=cOy_$lRu$rtAh4efK6LUGP@hHFa`&TmYhN zQYIQOslvnIW^c8U))=$ryHo4 zXA?%fK$Mp($u*?&zRB31HgNhzo2F0k8=5OtEnzlF!XCEnnani$)U6z3OogE##YHe2 zo#>JE41i(cRMXtvWJ?@D(E0m0e~XauS*lPt43DDs6p8Cp&S#l0h=^DDws$&U5=~^G z)PN2On_|qBud*U`Gge!KMJ+oD_C?E^p22TOSX+?m_}R+=;aZL;PB!0hM$hM!;$~IP z@z%+9qs|c4y&9Yg*j|9)XmD88HPV4TJ~;)JHaa1OgMDl8SMl}D{7&2~IvYM$>5&5` zP3AhTgt30)J>4o>nK*c)JTQjVXEE>OwyXErOJnHLVJSjyWw$Um&5bgFJ`Y;i4#8f< z4@TH==r=3ap*!)^zxLFe%+2O`MNYzQ1#$hn!~MryfdK<=juktRg+tZ_N&taOZ#B2h zup5{(8&t*eND0^s!{2bq;R=<|L@7rYj}0#~ddkeX2w9;3t)bLF&@XDnRt1IJrR7LM zCN{+61j+RSjp#zEoBFB*bqlMELR1}VEV`1op0hJkGNB#;L=1&smPLVh%WAYSydOo-8simt`5j4-c zwF}-QjFu}wME-eaV(>`Ek+Mh%0#~vF68U*%q*DQV2iNEXlFuECZeWM%c{ZYjKCJ`9 zz6R+LGo3FHhS97?`UV2{h?*gpmm9=$+wrxd+sq8VhlZTz`YAUcxJgZbYt^c9&x zO35#Ubjo|oxdCtXQ9WHE^LMYNNXUk%>(9v#Nu?iK`ers&;Q7mbL+I6+%2pmPBWu^!WCllsXC-`+PQG*5p zPCoGq7GQrBqw5)jZPeTa<@b+>B+4ZKU>sdgdDTU+bpGRfiIb3HZr<+eGowU6>1hEF z>|UD6+SwA@?$gz0wb!93Fl?ET5(@k8ST;sowcVOhgY#9x_0ex?xWjXaz0Bc9$l$h< zuoJb(4$0?mKOLp=$y5?v^9d(N4F1M-r!uYY+7Zs)bpKW1FH^3KBA(V3W% z8@1cCTOJHSv$rm(Pk80iY{^lQy5`LxO>5GPvGO>hz-v& zK5oc2^`VkwBvhtVD<}2KG*-|qJB?_gmKagR>k4bjf(P*!&nd9{9*1bmLjuP2^};SPbKLP0ZQ=t_w>rXih8ZqTSOCC-gG{ z%jMOCXiAuavdpx_SwQ?{C))%I73E5;l^$}L&Ak8Ez zB(PWw#=GYxJgRTk4mLk^5BnCn#j_D6Z;#a4+E=^^5UNd<6S6nvH>&vLvd&^6NFNi5 z-Ao}oV+7Zu`-Ra97bj@Q7ggp^3>(2Mh&OyV1`_}8$jE_@Jb%w1;Z!fI?~QQjq`Kl! z0nCaG6#*JP`mbf3&ROr(Mg9U0IhUkCrsluIsY2s5{wYyxbipjBlK>$==2rgil;KCT zPp^@?;GLPO_3zv5Mefj!g*uq8L4aF@{1TtgEkY5b)9(NS$AR=5YP!=+$#uzn<4?p4 zm(&=26EQN-Mw84tiSW^7I`9 z;chWq*;)p(Ay(wXPB-vs)27=(`!0LG&FQR6kMvA8$@U_ME~WPqxz~+ynOai_KzCsg z0yVfEjUvH?;6HFPyGj&dgCK)3X{OvFy^c{kcAI2%xO*I=(BtugV<@>6RUi#NWnoCO zQ<|pt$vf<*d?os8__KVjg6qX)4elB^3fR7Sh+odrTHG?SpyvWJH_I`rujhq8g=dvC zm`NzfXo&p`lQFzs`9b*BE=M@=e>aznH|Zb7FcDSas}JTpamTX6*ngJdS8qS5^yj$q zHY6!mHw+7@V-WA zy?JiWigAktZRJ|o=7}v!E{Np}+I7U|bNJVf4~sXhA(s{v6T3UbP#O=W@|7lw+t4CIzhg~mcB0mQz6Xl6c~tWAFEAQ`LqNzSmV(~?XPW(bUlY>e+t&fi1G zl8-4gt;}M4*{@%7EMKe;K~-Ng2bjU>hW4*3$;d{>2ba?odyu~z+lkp6q6I*XZP z0zP`xX6c_0A3J07^0l2c%~;J?Okf}T-?z!|)f@MkS`9oPL0+JHziRk(pgW@~FAex` zAAnZVa;>4}K1SS`x)~*L`mBAnm+llCu-tw`p@MY5F|Wek>~}ZZ9$Q0DN*R55GVO$} zgJW{kC*FS)4(W<(gIdyog4E}5=Kp3P;Xnp?vXee;G(1w;xv6PY1}Iw18Azw`1^-GT zqxc#pdy(<&0K#Qt#mi#sBl)yRP4;x^f$(EuUsS@{{5ZNQ9qf$TJ^!KI7p9H`9AbB2 zpNe6)jA;DRR7&+BQNyMoMLF$ysv^3_dU0G|VW8AtE9&&xyH2cB&nG$hGbu;?OgP6X z%H>wo*$x9QX+6%`fZ%#dndjVS=lOGgspV$y7cN6Pg+MN9sv9a!pp^03)lWE-1YMNV zlXy;o#Z6+EMTa)gjw9+SFlw1A;?wtIeW;nc?v*T&`8VgU8{NL{*O7rrhq<0!R7&!n ztQV6S6;Npx4>+FtN!}-~MiOk`Si=N@j&HO-A{fE9 z0v$Q{v*tgIl&8i{T@$aIp8C+N;CQ)|*>3AjMyKV)4f`ENt7WUs^&`QxFCr2+-H17b z8>|=>|G~{)G62eq#Yp>3w!sG1bR;%)y3k=RAg@YVe=G_IDx%9d*kru$ zERg^7D|O$S5qR(4g8OT4QR)aPWt;!(y~`Wr5Qs?IE>=xg%x`<3?om+3HN6;*l2c2z zqi|(HycIR)(7>mFbxs>v2-K)U%H<8o@U?Q#n=8hU1TqEp)#^{OBRhUz5>_p*eoY1* zM`KQS>0$%Xs7(b3KD|3xM5-eAo?Vugd^InRuj<(ii&u%2*O^wJz7Q0?%1@%?nx-p- zC%`byMZ9=pLmRX^iSD8C=Y;*q&UN}f7bgE+pi3blb$bQkU5ZbA;lmM0oc ziLgoF*zWNxL3^;E`mq~mA|GCUB=L=?n5z$rqo?XoD0?pe~o#hYLJ1i$VF9ZR>flI zU_U;%PKyQDU!D&eD^d-occrkXb1`(|fwwA2IEgT3RKM!(dXIUyvp?v2FNW%wnxs{`1F3p1`|2F?P);5R0Zj?JM!{MhdwxNR z;L(>xe^)hbe2NW%gu(1tFr!4m95o>nx>w!&e0Kkv9yNRjc~J=(D(y6Y{7i3reb>73 z2jF9>ZLeE0fA(Y$N;YxVLTfeyz)ChM2MzuS3|jop$SD{AaS)3L0Kl@r`>#d=U*9VT z-iJQ`gjn}KaJH@c;UBS%#yU<5U=iF$3&e&_gJIDZNBUEnR#)t8j`KmVRO^yukQRl?~tT;xrEeY^f| z9u_1Vjn#q^4FCYgtX@d<;0OEt!Nt?~j2%zbjP0KaL+L`rTjEi7Ro~BmPns2{5|=0GgPe(h6bDgw1#CEU zKctS1A=vyC#<#Aot2oYT2`9Dd;d+M1DS#GGKf&|ExMt84(PA#E@X6PKI6RgZ`QG=^ zbqI{7n!^6ay>>+T0Ml|WO=#r{YqLChlmyciCADkx6N{mgh^9Jp`p^D8_m{pXUPkTy zKQ(;SS zLnOJcurP9jphd=b_g;|R!1u`gcRUbt^E*ay)>u7LkVbhGNj*cI~S7u=_q zdTBl_s5Qexel`K~&qW&!15bKqRpq&ebfe1tWiT5r#voovH&DtScZneSRMm69MBSn{ zgE6s9A3N~CR6pj}dbcSDz-8CI!z6n9^;Gvq$GfFo=POEJ~e8a%*0_=BK%}z?(ntw*X;GI*r_%?^Mw74y)yvmGqM? z49f?HluB*Y`389#9TdZ1p_Z)}Bjabvh+UC}}Mu;WcO>vgAHx2Jo0{K66-}fjjH4`T*fe(%8`gRc(rV2iriS?XD%0^aH zGP(qyJK$8+bjc`tH|n7|$A?tI!X9donWyJ&*Qv!r$RN@hG%uVqeXRZ!scTBmZu2^# zCtw`B$&7RW3Ap+}Z>6GCAv8C+&>cXuIzEyVzJPo_+z+$BRnrGP_t1@4UElAknp zhLd8%fj%uCMMi|y zTLbY&$O56lNi#MNV;+gI{bL~PUXGC$&%aoW<;QD&%tgLOar5le#IwiubE+S&c^azo zRK*GTnR50E>R83tD-L!(w4#?`1R;?3m*?qv$kOiV7XF}dj2&~mL<%wuK~^_=0D+?` zvI^5=*R403X$#_}jK1R|k-+Bn@<3Vj*cJuIERxeCpJdF6Nb)iOK;X)}S$}7$QhpsU zI^D!Ql3uKdQX)r`-k+`Fna1eXTY*sd=VOb^lt$U-Xp7pPSa2ED;uY|8kMM~+V|x#A z*a@P)b-j!o9LH<&Q2tKwpz}1{RZC1D0Zr+PM0n3eNLKe`akfG+kTIaesPH(h_iIe5 z4S*n>nt?e;@mGNn#+PCVzpXncVkq&9C`;ph#s%UW(GSfuIWb>vhUM=kp?^`{o!SD_ zTnWh}T#oQpCZ==6#fK~!T}NW<<_4ZW%Dp#nSBvCgAdq;YKWemGVcufOtcD)?&Kye4D5jDB$_kMZim#c znCZX`_-4=cj##$R%yBCqh!I^*;tI?!P_N9oCk(J48Q^N*Wq~a!wY6vf9++aWcHyc5 z`Xqk^eF;jz_U{9V<8LfHR(>!gbjSGHBMZRt5!mi*QvnLA#_@-l>#{T!i%{5c-OJ>c zRup0l35S?ftNXE!Aa>jqA0mM+02IAb38w?@Q9!JfWDx*>;$1VFsTUqA{=k1<-SkIS zhIbLT9N?j^YKwO}N^S7OJI`UJrLwjrLRd)mFC3{<#Q@(?3^ROxzz@UuNp-tsk0|)N zP)%>ZH{?jmaHh+Kz#(vDYzubi6%zUMhL#BH(N_MvKy^E4W1^!(ILL;k3GtC?=GD5& zkg}0?N~D8aHs|^{tx{Z&{+}GyUsd6myxcC!>~PvrvdBPelc=$_xtt%2GIQJ|y-cxr z3qkww`M)@U<#`1yzrH7_+^%WFmI!4LsDvBC&{rzxMz-(SgN!BwOs6%GDVk|TZ}GBO z_+qc+fh3u+7{7Xu2MTL6MwCNAWG#&od1bilUTA`}XX4~d5VNz(^L%$KMHt58c6@l^ zFj2}w)4CaZQ(sneT>L>_Rfz2xd&(bO#^JPkt=b%JkZD(5B#b6$n|q8d_X7#sR(~0p z!)xLM-6N5UAy+Zrzh8SH^be-i4Q}#GEeNvRPDvwZtUoDImN#jzuX`VS}~#3~4sh$;_f;!PfoP?x{zC z`crh^?&I*?)`WZ#PY`%W=B}-1}tgMPe z95t?Kj3}elyKE!i4O8XC2&zMhe5scRU+w{aaq+HBME>%xhE$b;4Udod4}Ps=+hM~F|8#~nN_{{M(yH-(Ua_KC|_T@C>~O{o)8eOdgZ9vBBX16 z)d2I0Qs1_xMthKs6vQ@6rF;*rkiYsYmr4}V|MGbNb2DLg4PTSV5_bjxPm>?sSz>BQ z1xc%#1#h5SC%;<>(iJ-6);+5kQ}A9LsYIyR`hwpK3t(*?xeRGXY-I9Dv__Mv4|qb@~^l{vVGjGg;&FkA~?|4``^+)l4wEeN4PFGG3)M7?QlFHpqE_-JYM>5nH} z?SgSZnW;Y&TYO`6b^q83^xtEO2G!&f^pP6wQg6{c4Q9-xAzZhEO&LEwGr+b-e z(3|)efRJ}pPUKxLBNnYFMI@kIZur}RJ_!>E!;R$|^HaY3U^5@@ndy!i`M-6(*0MrOj z5ZUVfo?m6Xi9ha@hGnVhDBlsP$V|A2mC?T{zew#K+H_4><~PB+ERx=|ooa zb>gx0DEkV_EbqixRauf3tPMRafCm7{BI{9hXgw8{c}nFUtk6Rs)R$1zk%eQiCR!-{ zx@T;<9yTsxZzD+3`daMO-0tnHY5gzPN0Tfn#lHR_Fq{o9b7j$NEn}1o#~?y>3$RgK zwyhEeyo@v(ozc1@jr){eWbQc*(6SHwScuN=HOpFFP?0^icQMAg{U=;5Opx;eIuzTe zDVil2-ggG#R4xishxf(x&UX(yWsZ)8V4(_0Gz+_e;Y7-FJ3-pjsjwlXEH14y&6X01 zS;rkbXC%{jeBlV46t4qJlx;!dA>p?Y>%!?a^R8+d+Ot)x>RF2CX`=e4-p!=1b7uK$c60ROK~2g z|AK~}DK4ey5~<&6nncsDP5+K1*@p^ewr(-v$2t86tWcF%kUiQ2jclQ~b+bwLNykt` zb8SGZ!d5)MOtc z3=oB0&l}^&1*8sR4R`L4E1#Do7S2=~u<5*M=tiWH16BE@1o7i^<0_)qgX^?`=yi)7 zFhevqU#71Vk^5mkt2`vWnGH4du=+2Z=wb_DULP3AF9mBjA@iIH4GJ1L%r$c7 zuXs!Ij{uB`=J~1o;?H`p9ah;2G0~ggOU9M)H5{Bitr=_Q2iJHvrdeLG6W2W-g(UFK zC)pyoHEScfh&{$` z#XhG(CFwr8j~C`rHP$BNj*y?7ra|y>!k<0!OyLiT#ex8zxsM|)N&-)6MYf;PVR6Hl zc$+>bhKY~V3o`o_>cO6+OrN(TPeznhg`{d9sOvjL{3$HkD?^xMu+*S{Tn_E7O>$qcRtiYRjndVl!xv z@A@YHNdMTM!Y6*Y)#Da^MzH9-_)u_X)BO73MSoyAJ)p$4?#TIzyx?`@EpE^35ORf= zpvdJj%0l7I{aV>vlV8}Db}#ZHX-23?%~~-w%TADBPy=Sy07MB|h81Rif2&d#gSvEeBnNavR?C0X=8l zHh;OTDgkrKf+-a3VHTQnYP+(tQ%`0vZbj1%d~#IYN0yUw4K4tM_2KWNtbveBWX@+S z=ZZK;-z&Kpoq*PD`fVjGv~WsZOgpS5-mWwo2kL7}gQ>yLZ_z89@9RD$n#i5CkWY5S z?y`}_JY%vUM}R*s7IWYomtdvIhQsgkb6ANY(ixfuuE|=rf(ekz$fK=~#6u8$@-Dx@ z?7KHo9-cn$+Y;tR)zhn>8ZnMH-~ZFzTSn)xBkRIuW{T~YnVFfH*)cOSGc(7InPQHa zneD{P95XXBe{c5QbN0-9=iD`O=GVPzYqh%7lB%SqN?q0JUagk5hF@_Aa0Ib)^iw2- z6I@zuj9T6KW`S}*n(cEni<$;xVDE~{Z)j5B=ewrL6M3A$ zg~%+LIDFgCNVDR>)_eHD5&z>C)kX5f1x;pk$)OKO?wLT`?X)4yWK|1B2Qcv;d7*l6 zJ#;IlEt;PhT+Q@A7f?N?iZCS69dHiwO60yRy|V`})x+hcOQPz%im-%G7Oig&T2aTP z52|>B`cloCp9fy7q2VXZ%Y&Qw%-CZD#WJB&_SY$>i|!8lVOXa)qnp_zYsd-U-evKV9efy+P77(d~A+hP|16WCm>VO>8l z(Dp8ZpBw6CJ>y6x$EKmkJT`BY`&r!A8VB!&u4Tqylf%kn*&SDZ3wjfl=|@Ixoagmw z&`HZ58GR&B>sKnsvu5_9mH{g__u{F3{@~BL4;+@7X>kNMhb7)^QQ4&kgxNtDBqCDe z{Kcu@CEJuW+@=>2ho*}rIygT&zEH8|QQCeNa$?s0Zu@)#UT2|l)4RL)dY3Tt;jZA| z<+6*q7}~3d8oy;&9k}VNEU5#5>X98*DeuN|*0%gIEpqp}yr?fV$H_-;KPm_+bkB#T zajz@cdnrVU&5~DHNc=~J0Gi+}x(U(Z=#6il?W5CrSj|A7tLCQ@+?I&cbVYnWS%O&G z;dIaarJdq$NWEE6QJo&D7NBPO7fHmlxN`Jymue99mlHUtpSw8o>_MhW*Enl8BB)I~ zD{jCJNrQ6t%0aH^LJ6}RDE1kAu?N~mbkHO*~uUCO%G`HYFT2i59cv(RIN_U`rY4U~4p(nd;?kkjjCQ4Y-BgM019bQ>CiPpP7v-G%(BT};gIdTDseC-|Rt^Q9Zz~QxtUs)VVu9KnY;Bxl+XJAFdJu3cY}|sWKt}~h5IT3s z^T33;xD6LE1zFGDqfXPc+l@%5p3WM}$hNQdt`bTD=y+WAPts{I2#EQ#zV4DYMojAd@#&Vk&!G~WTFMv&m8TUEhriG%jC>e3r5YDSR0$)B3E23HAPpgk z7!!0OubkY3o)VI{WjY$3{Sy1IR02yVoK)6(f;f(G41wxL-B^ifP?>b~a-k$Xs(BN* zI~K7wa)J(ZK8t84;)c3!W}BSD>tGn4pMyOng1amal%59F8Du7`E;b9<1Lt|#2>1YZFEo*yJm%-Ou>i*fpN1214)=U?SmCUb zwQ=SMc~IazF9z5EAqXmgFaAlYbz8{InLU}va_+H$Ar7AgMpSj#$X?&oI%LffKObZA zfKl5Nfv}Q`yFf<(HHI@JP2kl8p+|&m_u$n`PnSX^znzzzg$_eq!agy;RBIR#zEMpF zlH%)+8-{MZ;eeE|d<<5k9-LmA2^Ilg^-Fb!$dV0NR5B;VtmiHS`gT<*TQr`i{OrEkPZQS|!`RbA1hTxPvzka21l@wDE%I>T+eWF+E-pU6ZrQ{zMv#@(|G(b%vGF z8QloY@j0inb`oThi-AbRw_IjSWUUoF>0!cNR&DMj;-Qb*3B|~uNjkMNzF-Q@OOI`LF?y&SzC&Vq_e8ligTqg*U91xyH1!9G*emqZrI;a0fX zYYZRyb!vjW40pNKgw*rm8)sZzh@O;J_AkKek2tf)nFpk>#@W$copb+8MN;WCo_DZ> zYTt#L@*D{YSu$yCF_szKAf4pck_R0^z2Stc4gRLd#oXQ#B0xr=Tb4LkF#_C9{f;HM zWrQTJ299r9@$EqVsx{ny`+e@*jNqn%l_UXU2%(9)l_fhRh^$wsc-(j3f2(}xh=TUyNijr++u)PUP4ZiA=8wY z+>mpHAguGk_hyMU;i*izVE3`NgOB9oHC$o6773P2zoUe! zOdq?J`<${3YFd|us32h$W?&nZs<-WAy1iQOI~}Y*8sZg8Ca2p&ir(%c zGf)^BVc()-{-x;{wXWjq`7A+FqPTrx4Ts&8w^AeK01@d@y!2zgJPbAJaQ)`OY-rG7 zw4D=Wjmz}mm6J&qZEq{xYRY}0r@oS$A6V;f4rZZW@~T_ibTMDR(qI;@n_E)Ud5y_? z`exUa!Yj8K?tLbz1^))g8y`pt1WA2^W&XMgVk9`Vk7G=nC^oz-1Mvdl9pc_4@Rl~+ zY5TNMq-jsDS>TmY9iC3@$7_>X89D1}(0}9JVe@1-+FX}rvF1A2CwX-go+N{?3{5t{ z8ERh55xQ1$-AW(2w4>l^|_fy2` zCvccBBE}XLd}jJKN;)cvC``I;oQfcE|C=`9EmA*9Ax>?YCc_8s?7oWRjTffFwuTo{ z_Q=e-4}O6$xa2);_GJY9snx_6pHO>vAQ6Oz&qQy*%W z1d?xB5UkX(kA|ic;%WB3acf|5j~4TtX76ypPEZ#hRS~zb3D^tz%S=oyi)9>3=V%bD zNGDBV!SophTSH?UtdfTM<}bsHR1ZwR`}wH}WOXjC1(un~Wa|b|@IG;^bSIR%dutNq z#~2FgP!t^ULmxj)>Iu118~2#&1zV0ff*Q#6JdF0Nv7Dqw_Nu3KsDJeq>u7@+cXXg|yTBJWYdl}%i8n23~OtsUF z+>EeQCJMF3grrWqVHDc8NyZ4AceR`J_8U|tO#&Nq6-T`=MTOZVeGp?_uf@zBh-_Vo z%-hcM@C1(xPZ_LN=*C`jkikvd-52{v3!R2G+O6-$$-7o#5|etw^V&$%W7)p9n1#FkUbm1AKocpARE%%s%rai{6H zQN@JUbx^@IASoR)b6J~{@hlXI=U*`Rt)|J+vf>1SLQrCr-cehQ`y6$jB-andJJ+bd zGX&vXt#d%})W6Jh2zmvtHZpta6$t5;=77%qZAJ}t|1*m4%>NirN1mlpOJ|1Tw{-fX-j1?(iKFI z&+DekjqdE|GkZ$KeXIo1;lW@2wTmA<61cvu;`_pJvJ>Ww2ac@T-v0A(`V^bYcay2I z)4P<$BKoz$j_+i;Y=}2q>*_%X7W2Ka7=bb&0*f2P^4dsV)$#mK9gwHBBiU3@>t&D0%#=03kwTgs*VM+R^3-u0;57S??B!x4=JxH15>#YSvHR$lreqZZ#E^Z z)ZH-4fb4nCjFcju95D-oY6&kZvV}p*`3yN|0*9+DYWhBm5Rw0FY%&N7v567{<9aT) zYfOe_evd}2^j?V=>g|yEqawoL)*;&$*X5lW%W#`O{f5V3DE0)HkCy~@6vMS&%H<{d z@vgc!QJHO&t{`Ri{QXAd`9kw&Ru&WqaQUi&2{$wl$=_4v$?cKAxQOUPxB_1VkC4(o zed-ucx1uktz~cdfP^t^0?}TMQ`pkifCM63IbVist$_m`3CUaA|fkKnDaZG`k&B6I_ zWFmjX;~hZfENkk0kCH`n8yD%Wu#;wNJn+QhGR&ydT}A_C^5vO7yrx+{8@(c;#MNAY zbu;F{Smh7}7&S*ZJ1krSsN?%7ZVfi{N3!C?5BXOUs5YDMR9AC8)|!M@svbvFpdeOR zUDWGtDUA2Iavgr1+?DZPVuVKFC_rl(v}5Ug@P+Hf&}zr=AX4P(-tA~C$f2wIEOEz9 zoC8!|m7hp(se^Py4K_H|cAYK>RGgd!Yxc26ip-6HIQcym^GI(y>3bio;I+pOxT|OQ zPJ(4aFe0XDzP9wK2WU+l6J1WDm*pxMrL((~5A?Uh*vNJyqLnKwvm*vnDy{=bxyjL% zei6(fguC_ayC~g~K~LuO-#L+&5GoZwpdhiW=xG_o*Hn}M%R_VqHHMU(VJFHmTP6dK zt372Ujw$0_{epM8Lk?MXJHsld<0FnG?Ck@AQX6=S6Sx(e9~2ZvlDM5I-o_ir=v5%Y z+aA6PLG1-U759`*ZKF>(YAw!=VU>3}{Pr0!`k~G&u0$IKine!LsrIZv*UxY9{wnM< zh+d^^AgRdkriTC4$Ql$^G4`nQnIRRP4y6uTRPxOTay2=4Fb!WrU~a|?BKNabs+T%w zkP>+7)oMsZX+$Kf)cx1YY!jmePdUT=#G%K_6kgFU4282wT`uH^mv&Dt5q;$z5hr<^ zGyzCJAYONyO_H1U5}1p#yQ;K5dl1)NB$V@L!(&02Xu{R$3yc;u^%(DA(-Qaveq^)| zJ%<3Et{dJ*iVTU?rO%<#5Xc+ss>!!;`#)66TVPb zLrC;e zAKD%p9d#O^UY*x7C4!oPUIV8W93iJ-ID=9hnJt45ykwD!DA&n%GRAnt@Ro08{9&%_ zm#_lo_IPfr#8uB}TfnvO*=j>QPqAvUsPtfZNg%OO&mMJ^O*Ha`@B@B~(Cdunj~GAo zm?Nd~|DjYgXhW4fvd91KiGOTxzI-yS7b+nwRDrPcC0wv+}FC3R7K+ z7(>QQwMtU<;u}mGU3Lf+idP{WsbB6nRWdQEKI)MB@{}#j&|&#e zD+!XVD^mF9{-buqv>6De+lT(jj%ShU&b@o&V*AFSmRM>I7pBFzX9J1-m8#*GF9i{1 zJNII5n(*`CHQ6@M4%nD?OvmZ+z-wur&P4d0MR!gkt3lq7d~v!(${1`B&mKa%316RQ zXi6i|Vafe&^TCkyT@~)#zAgE&Foh3H-m=WN;R*kJF^>Rb8hItLEf*4P@wiOz$gs2y00)g z9XD7ItEUCtTF)**k(F4i?bsQkv?SXh6Qc$gR2my@SWCs0oqXqtE)*RNk6{F&e255k?p11nFj}({z6w`-CMn*!~5rm zN~BAN))i9g$eE(%9M5T2go7+%zbeq{dst!F-ibZ@Wj>F5=_h;Ho3SCr&sK?PW>?*D z^GZhJxm%w0Vj&mY7 zW-3Z%59f~cW3?J$qPlD#p~5yDlB}dM=d2Q^0NGYMpY@|CcONZie}}rKBc4KxAh}et z(veu@!EZOIh`63M1RaHPjha9%`e}VRp)li*UdBo$%h?_@PJr@`41o%1?us$T!>-}AlfnW} z$nD!1NvOO0AZ$cVt_xt2q^%cuh@5qw5i_x1z621vRPI8$a;+tAGSNk{8W3!qs% z$G35;gA7x}h(&L#T0?2D1fJNUIfvMF<)?~eyr~>|C918rG3mW{Y)k5;wPk&ChSF74 zV$k^dS)WR)d>`R*UJC^5%ng+HO3d_;KR#yfqxk=lkr?aor z*RPMWR;CFe`3cTY5AD?#F~US;(n%2zIPd=hRcU)vI2r8F- zl-I;{tcI<#8+`QRkWxKp`6YDMx}s|`<@$iCW~xk=yOu3_`X$=+vYTu(>AUTdrkF zg^!}lbz%l$GCF?*7sW-YXh?Pwm?2{Oj)A>1x&HGZgD(3kF`T}oy4fB33O1BeWVn5Q zz74pX@BH0Kq?-SiDb;Lu^0@gQf|H!t5l#;2@s`lB>+~2LJ#?;9MKH6dnZBc zC|>%H!C5rgt6he0GYeO{tpuJBM6DtF4r0?=cD*x`a?6pseoNW)VPt{OcISS;A-qyE z%CNmrJ+2XLDM8}(*azbYsO78%O;!Gy&5;%SDgHiZvZm(4FML{$OaVFQV30PU4jjyl z_9`DMuB0^g(k4mAlx8_-Ku={v@b@a`lw|gPBySztH3*^`L!R*Ut2Xxxa-yA%H4G51 ztb%w@D<1n+!+l>y&f;9TdIa|Wkc3;Z(@IN40!=w<9U%%6CZLe4v#SVkPsqI4CPa!? zCf}R>V}`!RJ1+er`vzLT8p%8nvBT{PiS^^-m3ulMI}!}t39&h? z-y1~mWv%EaGUQb}XrxZLQuVHuR&(K*rw=*25h5Q%rf#9CSZ(I54G<8l@acEHLqJ58 ztp^5CV5Lgp-CJJP1cT`)8Hk_x-OppKJ)3*syx~ai|E$Py^VFSDzqBE8e0$1YX5FNm z`z}`U42>*`+>y7u8?OKMQF%ypQRIqZbR@OR3rp0U?`(gvThWLemnJW22 zR1dA)?6E@5!Q3TGmC*k@`J+!exwjm@iz&htMcyF1G9$cD7P;FYZysr79o}qI6S1Hi zND=0f$}VueB5YmZirKdUvA)j|H2T9q>qw8_ zRm7Mqaz+xcPlAZ`-SLW9PcgTOLH=ECROvBPxVn3H-w{FInU$kJ@xasjbC+f3MRjhp zu3uXP&)DeDil`U4aKG;Kb8XKKaRge#zA_F99$iR`bqXhHhmKYtSHRuPAlr=bB#)!in~9KeKw7P|56zNOCh zN{@!zGY>d)h~p8>E5n)w`p1CX_NUK3)TkNsoZ~Te9&PRjmeE%!lNg4G96bmtvS>-_ zJKRv&d9}xa*Hv9IOP*{AfPJzAqA=o_tb7c3M-B~p_g$TY;u=SeDw&1OPCwpj+srX; ze2ovH3YC4)FxO{)WjjK>OXt`;bQ&)0GnI{)`*Ja65-(IYxU_B{*VmpX31V8~D13`~ zCsG4~qm0hH$f~UOGz*GSl6(>l#odJOph!YV_hA{wGf3Y0ZMKOqC-L~YuW?12Wj8cw z-=#K8|DDv4-|fj!z75wbSn^{?qecwv`{@0V#SgYnqr|NOx)x&3D{K3JPk>%bT~~4s&7IjMw%EcrC54b92k=mF7c0Deo~c8tYBY!$>VGH8S!Q%&CFaH;Y4 z`Esyu1&$*kdyZ)rZ4tGRuY`qn^6#5BJ?$XQ(d_U$VkdA=rjc}?fi`?5_$9 z6RY>xPmU94D9|kI({eLAt(i4(5FYt-bsNhlUpYZYQRW9IL^|Km_jl(bI=W%LH(o(xKB`p>>*5QtVY~fBaKiMU* z#rw)bh2fk@6^>1&f`6>;!Ni5hpO}T53@M4$j@nhD)o;|O-fq3K-Xl|x{ltiK%P@_X z04e4Y!{S5hn}jO%qu>(U$$o+Q{)k2%Vpts?d+x&0SRRONnT0*i$J+&6NO!y=+hnQU z%(WD8+6}>2gJ}Riy<-J^O;cG@*i&v1X%ixgW3bV?=OZ3EC(o*T*kW2ID!ESFybtV9 zSI9hNShNj_zgAh6@ldV#I|d`f6m$nhMji2ntC487G zu3=o)u1$-v+g6u1O@=r4eR?J?Z2A|BB$3aC4WSnheqW8o9OH9jjBr2A;C8cC<}vKs zy!-h!7bDEmL=65ILXroG7*evZgvxjF$+5!E77zGFhqx2n>AVS_BxZpPMYYtmm4f|~ zrvR*qJd+a*nPlbDKyY9-E=pIV_DE?F_W6A5f#{>LbidnIyI7p>H2HO)uubiLX!o3i zHrR?=@IiWJe)o~>spBX@dw%G<#rx4dQAC30t5u(?c=K$^1q}?dOBbfoBKrj>m98dR zX1#M49A2VDX-Wx-!KlW>oIgc}RW8djwt=$5Mnjrmhf#3;@byujy>jb- zjhJH$e7jy$s)rYy~o<$@B%Nh?XNlL2-p-Ljp#o( z;rlYQLe;x}{>t@3z#-zZP%te@y}JE@zr9PiJCihcRL#$GbO|z3)3nuQmsXlX0k(>? zOprtO%o>rfUgE648WSwip`V9NC%tmWwzFyQ@c7$?(`) zrl5T+D#kF0viz6`yqmxPF*2qpz^&R7vFaj?nC+{kTq^Q;yT6fxE?CQs>_!_h^96DR z|D1T>WD)BDt_{|ovT%%Qa)B_-#IogqLiS}3OCC#fd@vOY^jbWp%NVmsNZZC=X?qyK zf&Yp?(qT&fsN>niaf9ych|-TYjezH^HC~6`d&r zclPRh~Qz>qrlGbNJ3?<+PJwTw0)}oAXskqh$6A)4F>qBQNj06TzS0< zxkD-BtzmD1iR7;MpjZL?vCd&5!PHOfep7g)Qj6ld<-^e7WP)8_Si&+!b~0>P$n4m5 z%yyb>-2-%E41=7wXglU1>Cm?zMO|}tQ$mI)9GcIEd8QG8eJ(A;EYnt>Ad0R+!?i1@ zbvS!h!`J0Dw^e2ALsCaA2;W7&Q$BNy%ZbCqv+jKBxTT2Rh)I74L1V|LE|*yE5BANO zK!-Zvwlt^9RWuW3UMmaLHf9ftGjK!SF3#@^=nQqYZN-X^2Uq&ajZo=}Y4@cyS|6_Y zN&!jeeJEL!Nq&@iSWdP-*!6VM`Vf`3+Df0S9>Tx?XH2D4@+mWwI zII#%Nkz^_gOZscKN|l#BC5fCN>dU*Dm}I0y ztW~sVwBv{Lb_w7M(boP*Yr141wFlyR59OxY8|a(x(hX9lRZ?+@8-@8#Dsnlm8Z(A- zjDsw*<1;5`bXHMS6;3pOCr8Ic*lkaY^8&^6yDf)HtL3G62y_k&WL}3b0w3ke2B~4v zwY{*#O48T)E7QNZDILeQS!)N21rX7GY$n^v$jmIrsEh1Ak*1psSjT$%26N~1rugxD z?kf>qh`g2Fh-Y~9o#y>_lB383h%I2Trf8`BCGe-TfRQ+z{u zJ+0o<_3ALz4~T8z+=+&(1)v&{d{vnB2wQx$Q^#0R6J6TR9vD0d2{1?%!>!d-!M>3TU`wrFC<)KgmCakN@{2iVw0H22*Q+Y* z>JQ&Ksy)u*RomI`sF=!U#TZs^M_EAiMV?OKv6q(-ODlkObc?~t6mGL;wF~RTF&D(I z=lBx$vS4sP&6!?jc~bmA41}JLULP4!tGdf%FMO3_w!HH>nmHR!3w1;-o20j`RflW8 z{vg_xR{M@GJZ^?6&9vYf=i2ASwotmMIiD=DN39Y#5ic52MjAq_3lCEcZef&#HLdhQ zcg$rxEzjJm8lAU-XA2IVey+-*+=;8LEfCga{qw$i2OBNKU;?n}4+JCpX)ioGzEy&D zixTX_6O31&Hc$3f(L(>ShO{4WT8sYfE5*L1+&h;`u+$x}M}!~ptm6cVm|zVFpxJ!_ zwn3-I#Er7vktM%jD@~v-fq0R?957$EQ!JEKV$}^nlNnWnS2bd@g&$fVAg!x$Hh;1} zV~4;Lc#I?=--|dp6OJA-}dOs5u;7-`Z|Hl3~fGA89DGxsjxaQy)6}Ma!C=+WuupZ2FQ|> zSG8bGp_MLc1(R65gv6{(IAAyN)Nx$|6AuQ&Jpmu%=^?Aj>k5MSxL)g1`l4_q2cZ?s zAZjO$#l|8XZ#Jts>VAuc&M49JEPTL*SS_!R^FA(ay+LKuzeuoNUB@Eh%Fbjm!w37& z=d~?lA~SwEJrf0ZWjNGP*lAIMrw!H*Qot4$&g!_j#^Cd4mA6@Tj`4&gsO^6%2f>Ca z&zlvbMJm6<&^aov{B7I%)ZQh9b40soXLq-YzHu?3T;ppdM7Hz-Ecug>J-DGJEG%VA zvI^G#0za{v2Ql$iCt>+dPQJ3%(Pu@nbTMAZhC#GN(BZQ>ZZWS*b`;ZL=Y20>rcOOu zMSE94#%mKZ(geNj5)EI|uaqXWpDgcEnNe2Ht-4er^%4YXaMC}dO1dJlWObXZStqly z#b3!PvMDU_Kg>uT*>*#{V`rCIh!8)6t{95g+$eTAV;MFZDBtrhusZ=`3Gbb}!d6gx zYTit>z%`CdXCKTX^}GJ=6*mhpd_`sbcnPCV@2nWHdvzp_TO$1_wYJYsaHl99FL|Gnyd|fMu}3B~47MXYGl5Wxy9JJX>R&v59^P1hlJRWul)4mDzBo zj{T0rC-vD~KgaZbXcRe@88vM2DQIs7C-`(MW=U7n&~7NU_CR=Q1m%a4V>X6H)71%G zMFTxie<3Yen72;=;;!t?5Uyj3v3CQbiOp}ruq+np9{*I z5LkQ61{lZVpT_rL8X+Qmvgv`vC4fm@`?-gl0aA*87=j-hEFKb{UW24&avkI6e)qiF z@mev~bd)|*SoJW_PiwUFv#0nJ%1dkhaZo@0C*!$k#aA*T+kIq)?OT6)0`0g*C4R~( zlR~gE>FkE!V{MGbH*LE4EM%a-#EDd6)a<0 zt~yy!&|+rssP#ciaqYMkW=6m#Sh#3f%7to_GTV4K%pI80)r77+4+1vRDX1o3=T5)@ z%1%-4jf!IJ66y6UXbgsb98SyEpu_5(D!({6tCRb*q`~sAgW<-2E%B&KbH)YlOhC3L z+{oL8IdWq_K1@l%ky!;HrUUnxKFgm=rdzbHOPQF2WuSzQz#qxOR%;KgBFkQm*RiKV za!lNfX%;X5KFcXF6>@KX=W|->?qHeMV^9Hg8$&g3mh@eV8(6B7ti6D-&7|+{6QUi} z_h%;lnb^4t4o2PiQ~X+$KPr2z+`~MPyr0PlEFS3Fqkf&Qm@z_}{Zow=nJYM?4f?nX z48A3?SJ;eYM@9}|zz-<1wb5H~g#hgFRxJCHo%u*-)#+zM7mHn=N&z(M8u)K;Zv&b+ z!GZM`f{(54!LA-Z(gGK`f^`$BC+sEhNu_Z(0sSO;zn%)3gw%Yd%* zn)1M^SQUvstJ7Uk>?Lr&!o|*h&`oqYU8n2E?;t(uiI|6+7>NY=!s@XDrdA@8kkH$h zW1gUb3!Vug!>!P!9kpd-POJtlQ&9(4%~xJg0CnT~X?J4Cx_rdB@^gNMTKnt|ECRBg z5Cq0G`=gjMeRN4Nn}Hx5>^rBBqjW>`1IGd9?k>M1g!R$SA0Uo*oQc`Nzr-k}z6a-M z<_1pP4}3&7=jC;_025~`;(b%`0IAl)N1^WHo!L2663%HYq?o1ZPk7LnsH7E8md8bb z#|EvUAdfw8n;aT5!U9Ps@1n3x)^oLb3MY;;vFCD;`t$^ z@%c<<0nt?jJ9fS;~9=U#0lrr@~JEbtYn{*m)zf100M&21SFgV z(@Cs_yWYMf|H=g{3xo?4?D>2!OFk;k6Z-8ttk_UM&S=lqb+G$xW-kAsRLXfw^#<^J zGl$|=tw_r-wmyrYMv^g>x+%Gkync+~vu|sX`I#<8MsQk>KSZ^1+(M}R+;^m_VB2@TWaRIHo;J3$Z&x@VO;m*$`mEZ!O4 zkj8eg`@KNZEm_u&V0c%I38!ve%%g>+hr36+dv>kX6)o0Jd^NyLT=l4zcJ7)cs-H?s zLWjjHSZ}b}$(L^-u0u&cjqOMl;9BxCFls08?~b1<~Z5IAjU!#Wv895Cvs+t9|Jxn9z8Ix4Mw@uELS^N zL8V7*F~Q}WF8ncYnEu-9<29rKzWv-0r|*ty9mDJ940?(5d zmlFH=i#=2DOc=?jxAZxYuP0XZ1d;*`ZNa5{@jq$y0e!rNEAB%VptzP=8^l17r58>r z9j%GM_U!nos@O*XadDHLFEvLyLx-8o>v(4qa-KD_^0@5Ic?(~-VoyzP@j4v(NODHaS{umZfR{#RgSv$b* zlO$qgVOQUByR5Lkw(5#NXnxfb1CKwNAj)dXwPhQo!`=Ljs)C~vz=w}aObMeS191ar zlodGJ$nuXy-SqCOn4dVJ>VCke&@-U+1O7k+=*!ZZk+npmn^s%V99)7ct(~6+#rJGT0U)!+XKk=1syR*mfeoCX+Rfq>Y6vy1L1~Krm;F zRp?Z`HJ-y|(RIR9{5EQTSLuh{#K=v&y*{$AYG|&HS2NsGv?GhF_cOWD_P^MoCRbP7NsfEIw5fwPbQl+{SyN_b1OG44CPX6~Pq2-xguvG6&93SM`l z-GJr(0Ay5x=@wU&&GV6ZR{E{r@BY4=fe> z>Hi!J^jQ`!(IW*=0K5vIX9mup{S#sgBna>1Lbtox*#6&<+EM|9l_fb(-qlod009~H z{7y#yzrvQm@~>fI05k%?rt}|TbNDy0b^ngd>3@gK{a?fOhu)I^Lu?KICbmEH*7R>; z16WwVn9OMZ*RcJex1awZw#R=H+aG#+{@-ES_}8%guD5Kw{|s9;@Ba#0Hvj($TRO?V zhV6H~Wefd>*#7GVp<(P_vL(Ok?N1lqr1LM?$=}E1Zw~$+9LoR6P6Ax%-#dkW+fM$y z=lqwj{eHduy}$gIu>Ii!{(sK$zsL56-u}_y{rlMd&dvQx*#6MlU;gc%efMAf?Z56z z{_=1C^F7yJ{_W5E$vne?kX7wfPR~I|KB-(Gkbr_M*1JIwf%ae!z1{A)BKH6FrE01`J4Lt z8;8LEgbm~;02;Sowjf}NL;c&+La=709KFsgTX^o6jnR*Z=-z)vN`potQT@EOSQC29 z5-1L^u(V$Rii17%Hdob ztqFclgKs%G8(IU>wNB1Xe^bb>I(Y`JM!%k5{0e{1^S2D}U$7+r$zSnT07jPq)MsO& zW8$D=WMCk$wlH*OVq^Ke@V7nYJ7AyzaR6!w0~-V3zi$G4Gy@#Vmjo2RTP;Db!Tf&> zIlvAJG8%IP7zo1rczcAX`}N@gKn8?n=<2_eKokT-EB4|xJ?pOGCP{VV*^->> + Review calibration bench demo + Dashboard-style summary of peer-review calibration, coaching queue, reviewer quarantine, and trusted reviewer tier. + + + Review Calibration Bench + Structured peer-review quality signals before reputation credit + + + Reviewers + 3 calibrated + + + + Coaching + 7 actions + + + + Quarantine + 1 reviewer + + + + Top action + Review practice set: rigor scores are higher than consensus. + Transparent trust adjustment ยท reproducibility blind spot detected + + diff --git a/review-calibration-bench/docs/requirement-map.md b/review-calibration-bench/docs/requirement-map.md new file mode 100644 index 0000000..1feda74 --- /dev/null +++ b/review-calibration-bench/docs/requirement-map.md @@ -0,0 +1,30 @@ +# Requirement Map + +This module contributes a focused structured-review quality layer for issue #15, "Community & Reputation System." + +| Issue area | Covered by this module | +| --- | --- | +| Peer reviews and comments | Scores structured reviews against discipline-neutral rubric criteria: clarity, rigor, novelty, reproducibility | +| Optional scoring quality | Compares reviewer scores against consensus panels and identifies systematic leniency, severity, and inconsistency | +| Review history on profiles | Builds reviewer calibration reports, modes used, review counts, and audit-trail events | +| Contributor credits | Includes CRediT-style reviewer roles and completed bounty / endorsement contribution signals | +| Reputation scoring | Emits transparent trust adjustments, tiers, and quarantine decisions when calibration is weak | +| Incentive tiers | Produces trusted reviewer, calibrated reviewer, coaching-needed, and mentor-required tiers | + +## Distinctness + +Existing #15 submissions cover broad community reputation ledgers, CRediT graphs, badges, leaderboards, abuse detection, and appeals. This module focuses on the quality gate before peer-review activity increases reputation: + +- Does the reviewer score close to consensus? +- Is a reviewer systematically too lenient or too severe? +- Does a reviewer overlook reproducibility? +- Should the review count toward reputation immediately, or enter a coaching queue first? + +## Verification + +```bash +cd review-calibration-bench +npm run check +npm test +npm run demo +``` diff --git a/review-calibration-bench/package.json b/review-calibration-bench/package.json new file mode 100644 index 0000000..286aa01 --- /dev/null +++ b/review-calibration-bench/package.json @@ -0,0 +1,18 @@ +{ + "name": "review-calibration-bench", + "version": "1.0.0", + "private": true, + "description": "Dependency-free peer-review calibration and coaching signals for scientific reputation systems.", + "scripts": { + "check": "node --check src/review-calibration.js && node --check scripts/demo.js && node --check test/review-calibration.test.js", + "demo": "node scripts/demo.js", + "test": "node test/review-calibration.test.js" + }, + "keywords": [ + "peer-review", + "reputation", + "calibration", + "review-quality" + ], + "license": "MIT" +} diff --git a/review-calibration-bench/scripts/demo.js b/review-calibration-bench/scripts/demo.js new file mode 100644 index 0000000..30a9312 --- /dev/null +++ b/review-calibration-bench/scripts/demo.js @@ -0,0 +1,16 @@ +const fs = require("node:fs"); +const path = require("node:path"); +const { buildCalibrationBench } = require("../src/review-calibration"); + +const samplePath = path.join(__dirname, "..", "data", "sample-calibration.json"); +const input = JSON.parse(fs.readFileSync(samplePath, "utf8")); +const report = buildCalibrationBench(input); + +console.log(`Program: ${report.programId}`); +console.log(`Status: ${report.dashboard.status}`); +console.log(`Reviewers calibrated: ${report.dashboard.reviewerCount}`); +console.log(`Coaching actions: ${report.dashboard.coachingActionCount}`); +console.log(`Quarantined reviewers: ${report.dashboard.quarantinedReviewerCount}`); +console.log(`Top reviewer: ${report.leaderboard[0].reviewerId} (${report.leaderboard[0].tier})`); +console.log(`Top action: ${report.dashboard.topAction}`); +console.log(`Digest: ${report.digest}`); diff --git a/review-calibration-bench/src/review-calibration.js b/review-calibration-bench/src/review-calibration.js new file mode 100644 index 0000000..d09a9d7 --- /dev/null +++ b/review-calibration-bench/src/review-calibration.js @@ -0,0 +1,274 @@ +const crypto = require("node:crypto"); + +function buildCalibrationBench(input) { + const validation = validateCalibrationInput(input); + const projectIndex = new Map((input.projects || []).map((project) => [project.id, project])); + const reviewerGroups = groupReviewsByReviewer(input.reviews || []); + const reviewerReports = Array.from(reviewerGroups.entries()).map(([reviewerId, reviews]) => + evaluateReviewer(reviewerId, reviews, projectIndex, input.rubric || {}, input.contributors || []) + ); + const leaderboard = buildLeaderboard(reviewerReports); + const coachingQueue = buildCoachingQueue(reviewerReports); + const trustAdjustments = buildTrustAdjustments(reviewerReports, input.rubric || {}); + const dashboard = buildDashboard(input, reviewerReports, coachingQueue, trustAdjustments); + + const report = { + programId: input.programId, + asOf: input.asOf, + validation, + reviewerReports, + leaderboard, + coachingQueue, + trustAdjustments, + dashboard, + auditTrail: buildAuditTrail(input, reviewerReports, coachingQueue, trustAdjustments) + }; + + report.digest = stableDigest(report); + return report; +} + +function validateCalibrationInput(input) { + const required = [ + ["programId", input.programId], + ["rubric.criteria", input.rubric && (input.rubric.criteria || []).length], + ["projects", (input.projects || []).length], + ["reviews", (input.reviews || []).length] + ]; + const missing = required.filter(([, value]) => !value).map(([field]) => field); + const reviewIssues = (input.reviews || []).flatMap((review) => { + const issues = []; + if (!review.id) issues.push("review.id"); + if (!review.reviewerId) issues.push( `${review.id || "unknown"}.reviewerId` + ); + if (!review.projectId) issues.push(`${review.id || "unknown"}.projectId`); + if (!review.scores) issues.push(`${review.id || "unknown"}.scores`); + return issues; + }); + + return { + status: missing.length === 0 && reviewIssues.length === 0 ? "passed" : "incomplete", + score: Math.max(0, 100 - missing.length * 15 - reviewIssues.length * 5), + missing, + reviewIssues + }; +} + +function evaluateReviewer(reviewerId, reviews, projectIndex, rubric, contributors) { + const criteria = rubric.criteria || []; + const deltas = reviews.flatMap((review) => { + const project = projectIndex.get(review.projectId); + if (!project) return []; + return criteria.map((criterion) => ({ + reviewId: review.id, + projectId: review.projectId, + criterion, + score: Number(review.scores[criterion] || 0), + consensus: Number(project.consensus[criterion] || 0), + delta: Number(review.scores[criterion] || 0) - Number(project.consensus[criterion] || 0) + })); + }); + const byCriterion = criteria.map((criterion) => { + const criterionDeltas = deltas.filter((item) => item.criterion === criterion); + const averageDelta = average(criterionDeltas.map((item) => item.delta)); + const meanAbsoluteError = average(criterionDeltas.map((item) => Math.abs(item.delta))); + return { + criterion, + averageDelta: round(averageDelta), + meanAbsoluteError: round(meanAbsoluteError), + drift: classifyDrift(averageDelta, meanAbsoluteError) + }; + }); + const weightedError = weightedMean(byCriterion, rubric.weights || {}); + const calibrationScore = round(Math.max(0, 1 - weightedError / 4)); + const reproducibilityBlindSpot = byCriterion.find((item) => item.criterion === "reproducibility" && item.averageDelta > 0.75); + const contributor = contributors.find((item) => item.reviewerId === reviewerId) || {}; + const reputationSignal = buildReputationSignal(calibrationScore, contributor, reproducibilityBlindSpot); + + return { + reviewerId, + reviewCount: reviews.length, + modes: Array.from(new Set(reviews.map((review) => review.mode))), + byCriterion, + weightedError: round(weightedError), + calibrationScore, + reproducibilityBlindSpot: Boolean(reproducibilityBlindSpot), + reputationSignal, + coachingActions: buildReviewerCoachingActions(reviewerId, byCriterion, calibrationScore, reproducibilityBlindSpot) + }; +} + +function buildReputationSignal(calibrationScore, contributor, reproducibilityBlindSpot) { + const contributionBonus = Math.min(0.12, (contributor.completedBounties || 0) * 0.02 + (contributor.endorsements || 0) * 0.005); + const blindSpotPenalty = reproducibilityBlindSpot ? 0.08 : 0; + const calibratedScore = round(Math.max(0, Math.min(1, calibrationScore + contributionBonus - blindSpotPenalty))); + return { + calibratedScore, + contributionBonus: round(contributionBonus), + blindSpotPenalty, + roles: contributor.roles || [], + tier: chooseTier(calibratedScore) + }; +} + +function buildReviewerCoachingActions(reviewerId, byCriterion, calibrationScore, reproducibilityBlindSpot) { + const actions = []; + for (const item of byCriterion) { + if (item.drift === "lenient") { + actions.push({ + reviewerId, + type: "leniency-calibration", + criterion: item.criterion, + message: `Review practice set: ${item.criterion} scores are higher than consensus.` + }); + } + if (item.drift === "severe") { + actions.push({ + reviewerId, + type: "severity-calibration", + criterion: item.criterion, + message: `Review practice set: ${item.criterion} scores are lower than consensus.` + }); + } + } + if (reproducibilityBlindSpot) { + actions.push({ + reviewerId, + type: "reproducibility-blind-spot", + criterion: "reproducibility", + message: "Add reproducibility evidence checklist before assigning Trusted Reviewer status." + }); + } + if (calibrationScore < 0.7) { + actions.push({ + reviewerId, + type: "mentor-review", + criterion: "overall", + message: "Pair with a calibrated reviewer for the next structured peer review." + }); + } + return actions; +} + +function buildLeaderboard(reviewerReports) { + return reviewerReports + .map((report) => ({ + reviewerId: report.reviewerId, + calibrationScore: report.calibrationScore, + trustScore: report.reputationSignal.calibratedScore, + tier: report.reputationSignal.tier, + reviewCount: report.reviewCount + })) + .sort((a, b) => b.trustScore - a.trustScore || b.calibrationScore - a.calibrationScore); +} + +function buildCoachingQueue(reviewerReports) { + return reviewerReports.flatMap((report) => report.coachingActions); +} + +function buildTrustAdjustments(reviewerReports, rubric) { + const threshold = Number(rubric.calibrationThreshold || 0.65); + return reviewerReports.map((report) => ({ + reviewerId: report.reviewerId, + status: report.calibrationScore >= threshold ? "eligible" : "quarantine-until-coached", + calibrationScore: report.calibrationScore, + trustScore: report.reputationSignal.calibratedScore, + tier: report.reputationSignal.tier, + reason: report.calibrationScore >= threshold + ? "Structured review scores are close enough to consensus for reputation credit." + : "Review scores require calibration before they increase public reputation." + })); +} + +function buildDashboard(input, reviewerReports, coachingQueue, trustAdjustments) { + const quarantined = trustAdjustments.filter((item) => item.status === "quarantine-until-coached"); + return { + title: `Review calibration ${input.programId}`, + status: coachingQueue.length === 0 ? "ready-for-reputation-credit" : "coaching-needed", + reviewerCount: reviewerReports.length, + coachingActionCount: coachingQueue.length, + quarantinedReviewerCount: quarantined.length, + topAction: coachingQueue[0] ? coachingQueue[0].message : "Publish calibrated reputation scores." + }; +} + +function buildAuditTrail(input, reviewerReports, coachingQueue, trustAdjustments) { + return [ + { + type: "calibration-built", + programId: input.programId, + reviewerCount: reviewerReports.length, + coachingActionCount: coachingQueue.length + }, + ...reviewerReports.map((report) => ({ + type: "reviewer-scored", + reviewerId: report.reviewerId, + calibrationScore: report.calibrationScore, + tier: report.reputationSignal.tier + })), + ...trustAdjustments.map((adjustment) => ({ + type: "trust-adjustment", + reviewerId: adjustment.reviewerId, + status: adjustment.status, + trustScore: adjustment.trustScore + })) + ]; +} + +function groupReviewsByReviewer(reviews) { + const groups = new Map(); + for (const review of reviews) { + const current = groups.get(review.reviewerId) || []; + current.push(review); + groups.set(review.reviewerId, current); + } + return groups; +} + +function weightedMean(byCriterion, weights) { + const totalWeight = byCriterion.reduce((sum, item) => sum + Number(weights[item.criterion] || 1), 0); + return byCriterion.reduce((sum, item) => sum + item.meanAbsoluteError * Number(weights[item.criterion] || 1), 0) / totalWeight; +} + +function classifyDrift(averageDelta, meanAbsoluteError) { + if (averageDelta >= 0.75 && meanAbsoluteError >= 0.75) return "lenient"; + if (averageDelta <= -0.75 && meanAbsoluteError >= 0.75) return "severe"; + if (meanAbsoluteError >= 1.25) return "inconsistent"; + return "calibrated"; +} + +function chooseTier(score) { + if (score >= 0.88) return "trusted-reviewer"; + if (score >= 0.75) return "calibrated-reviewer"; + if (score >= 0.6) return "needs-light-coaching"; + return "mentor-required"; +} + +function average(values) { + if (values.length === 0) return 0; + return values.reduce((sum, value) => sum + value, 0) / values.length; +} + +function round(value) { + return Math.round((value + Number.EPSILON) * 1000) / 1000; +} + +function stableDigest(value) { + return crypto.createHash("sha256").update(stableStringify(value)).digest("hex"); +} + +function stableStringify(value) { + if (Array.isArray(value)) return `[${value.map(stableStringify).join(",")}]`; + if (value && typeof value === "object") { + return `{${Object.keys(value).sort().map((key) => `${JSON.stringify(key)}:${stableStringify(value[key])}`).join(",")} }`.replace(' }', '}'); + } + return JSON.stringify(value); +} + +module.exports = { + buildCalibrationBench, + validateCalibrationInput, + evaluateReviewer, + classifyDrift, + stableDigest +}; diff --git a/review-calibration-bench/test/review-calibration.test.js b/review-calibration-bench/test/review-calibration.test.js new file mode 100644 index 0000000..8523310 --- /dev/null +++ b/review-calibration-bench/test/review-calibration.test.js @@ -0,0 +1,49 @@ +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const { + buildCalibrationBench, + classifyDrift, + validateCalibrationInput +} = require("../src/review-calibration"); + +const samplePath = path.join(__dirname, "..", "data", "sample-calibration.json"); +const input = JSON.parse(fs.readFileSync(samplePath, "utf8")); +const report = buildCalibrationBench(input); + +assert.equal(report.validation.status, "passed"); +assert.equal(report.reviewerReports.length, 3); +assert.equal(report.dashboard.status, "coaching-needed"); +assert.equal(report.dashboard.reviewerCount, 3); +assert.equal(report.dashboard.quarantinedReviewerCount, 1); +assert.ok(report.coachingQueue.some((action) => action.type === "reproducibility-blind-spot")); + +const ada = report.reviewerReports.find((item) => item.reviewerId === "ada"); +assert.equal(ada.calibrationScore, 0.925); +assert.equal(ada.reputationSignal.tier, "trusted-reviewer"); +assert.equal(ada.coachingActions.length, 0); + +const bohr = report.reviewerReports.find((item) => item.reviewerId === "bohr"); +assert.equal(bohr.reproducibilityBlindSpot, true); +assert.equal(bohr.reputationSignal.tier, "needs-light-coaching"); +assert.ok(bohr.coachingActions.some((action) => action.type === "leniency-calibration")); +assert.ok(bohr.coachingActions.some((action) => action.type === "reproducibility-blind-spot")); + +const curie = report.reviewerReports.find((item) => item.reviewerId === "curie"); +assert.equal(curie.reputationSignal.tier, "calibrated-reviewer"); +assert.ok(curie.coachingActions.some((action) => action.type === "severity-calibration")); + +assert.equal(report.leaderboard[0].reviewerId, "ada"); +assert.equal(report.trustAdjustments.find((item) => item.reviewerId === "bohr").status, "quarantine-until-coached"); +assert.equal(report.digest, buildCalibrationBench(input).digest); + +assert.equal(classifyDrift(1, 1), "lenient"); +assert.equal(classifyDrift(-1, 1), "severe"); +assert.equal(classifyDrift(0.2, 1.5), "inconsistent"); +assert.equal(classifyDrift(0.2, 0.3), "calibrated"); + +const incomplete = validateCalibrationInput({ programId: "draft" }); +assert.equal(incomplete.status, "incomplete"); +assert.ok(incomplete.missing.includes("rubric.criteria")); + +console.log("review-calibration-bench tests passed");