{
  "community|alghafa:meta_ar_dialects|0": {
    "acc_norm": 0.7968489341983318,
    "acc_norm_stderr": 0.005478252928392599
  },
  "community|alghafa:meta_ar_msa|0": {
    "acc_norm": 0.9027932960893855,
    "acc_norm_stderr": 0.009907715149899804
  },
  "community|alghafa:multiple_choice_facts_truefalse_balanced_task|0": {
    "acc_norm": 0.9466666666666667,
    "acc_norm_stderr": 0.02612053372889386
  },
  "community|alghafa:multiple_choice_grounded_statement_soqal_task|0": {
    "acc_norm": 0.96,
    "acc_norm_stderr": 0.016053601490175606
  },
  "community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task|0": {
    "acc_norm": 0.9333333333333333,
    "acc_norm_stderr": 0.020435234445940465
  },
  "community|alghafa:multiple_choice_rating_sentiment_no_neutral_task|0": {
    "acc_norm": 0.8367729831144465,
    "acc_norm_stderr": 0.0041334984078066845
  },
  "community|alghafa:multiple_choice_rating_sentiment_task|0": {
    "acc_norm": 0.5619683069224354,
    "acc_norm_stderr": 0.006408410297835793
  },
  "community|alghafa:multiple_choice_sentiment_task|0": {
    "acc_norm": 0.3825581395348837,
    "acc_norm_stderr": 0.011722191714790709
  },
  "community|arabic_exams|0": {
    "acc_norm": 0.633147113594041,
    "acc_norm_stderr": 0.020816892095910644
  },
  "community|arabic_mmlu:Accounting (University)|0": {
    "acc_norm": 0.7567567567567568,
    "acc_norm_stderr": 0.05021542194205401
  },
  "community|arabic_mmlu:Arabic Language (General)|0": {
    "acc_norm": 0.8104575163398693,
    "acc_norm_stderr": 0.01585615218998024
  },
  "community|arabic_mmlu:Arabic Language (Grammar)|0": {
    "acc_norm": 0.8356164383561644,
    "acc_norm_stderr": 0.019425937565245053
  },
  "community|arabic_mmlu:Arabic Language (High School)|0": {
    "acc_norm": 0.6025641025641025,
    "acc_norm_stderr": 0.024811920017903836
  },
  "community|arabic_mmlu:Arabic Language (Middle School)|0": {
    "acc_norm": 0.8148148148148148,
    "acc_norm_stderr": 0.07618086585254095
  },
  "community|arabic_mmlu:Arabic Language (Primary School)|0": {
    "acc_norm": 0.8293650793650794,
    "acc_norm_stderr": 0.023744879464344244
  },
  "community|arabic_mmlu:Biology (High School)|0": {
    "acc_norm": 0.7047551454932577,
    "acc_norm_stderr": 0.012156511606031664
  },
  "community|arabic_mmlu:Civics (High School)|0": {
    "acc_norm": 0.5287356321839081,
    "acc_norm_stderr": 0.05382727149237504
  },
  "community|arabic_mmlu:Civics (Middle School)|0": {
    "acc_norm": 0.7161016949152542,
    "acc_norm_stderr": 0.029412695005168132
  },
  "community|arabic_mmlu:Computer Science (High School)|0": {
    "acc_norm": 0.8045977011494253,
    "acc_norm_stderr": 0.02459051102594761
  },
  "community|arabic_mmlu:Computer Science (Middle School)|0": {
    "acc_norm": 1.0,
    "acc_norm_stderr": 0.0
  },
  "community|arabic_mmlu:Computer Science (Primary School)|0": {
    "acc_norm": 0.7578947368421053,
    "acc_norm_stderr": 0.031158436627408623
  },
  "community|arabic_mmlu:Computer Science (University)|0": {
    "acc_norm": 0.78125,
    "acc_norm_stderr": 0.052083333333333336
  },
  "community|arabic_mmlu:Driving Test|0": {
    "acc_norm": 0.815028901734104,
    "acc_norm_stderr": 0.011162096098666665
  },
  "community|arabic_mmlu:Economics (High School)|0": {
    "acc_norm": 0.7722222222222223,
    "acc_norm_stderr": 0.02213500370940985
  },
  "community|arabic_mmlu:Economics (Middle School)|0": {
    "acc_norm": 0.8275862068965517,
    "acc_norm_stderr": 0.04073271884266335
  },
  "community|arabic_mmlu:Economics (University)|0": {
    "acc_norm": 0.7518248175182481,
    "acc_norm_stderr": 0.037039750237562855
  },
  "community|arabic_mmlu:General Knowledge|0": {
    "acc_norm": 0.8865740740740741,
    "acc_norm_stderr": 0.010794642959932155
  },
  "community|arabic_mmlu:General Knowledge (Middle School)|0": {
    "acc_norm": 0.8488372093023255,
    "acc_norm_stderr": 0.027392810919920876
  },
  "community|arabic_mmlu:General Knowledge (Primary School)|0": {
    "acc_norm": 0.8333333333333334,
    "acc_norm_stderr": 0.02937114070209343
  },
  "community|arabic_mmlu:Geography (High School)|0": {
    "acc_norm": 0.7254335260115607,
    "acc_norm_stderr": 0.013859041217962768
  },
  "community|arabic_mmlu:Geography (Middle School)|0": {
    "acc_norm": 0.8235294117647058,
    "acc_norm_stderr": 0.023157468308559345
  },
  "community|arabic_mmlu:Geography (Primary School)|0": {
    "acc_norm": 0.8421052631578947,
    "acc_norm_stderr": 0.04872737367223958
  },
  "community|arabic_mmlu:History (High School)|0": {
    "acc_norm": 0.6697368421052632,
    "acc_norm_stderr": 0.01707109077437036
  },
  "community|arabic_mmlu:History (Middle School)|0": {
    "acc_norm": 0.8177339901477833,
    "acc_norm_stderr": 0.02716334085964515
  },
  "community|arabic_mmlu:History (Primary School)|0": {
    "acc_norm": 0.803921568627451,
    "acc_norm_stderr": 0.03950581861179961
  },
  "community|arabic_mmlu:Islamic Studies|0": {
    "acc_norm": 0.837245696400626,
    "acc_norm_stderr": 0.014614446209963328
  },
  "community|arabic_mmlu:Islamic Studies (High School)|0": {
    "acc_norm": 0.8143712574850299,
    "acc_norm_stderr": 0.02130648031715578
  },
  "community|arabic_mmlu:Islamic Studies (Middle School)|0": {
    "acc_norm": 0.8235294117647058,
    "acc_norm_stderr": 0.024762902678057936
  },
  "community|arabic_mmlu:Islamic Studies (Primary School)|0": {
    "acc_norm": 0.8908908908908909,
    "acc_norm_stderr": 0.009869098724696362
  },
  "community|arabic_mmlu:Law (Professional)|0": {
    "acc_norm": 0.7770700636942676,
    "acc_norm_stderr": 0.023525676259455612
  },
  "community|arabic_mmlu:Management (University)|0": {
    "acc_norm": 0.76,
    "acc_norm_stderr": 0.04964740541926505
  },
  "community|arabic_mmlu:Math (Primary School)|0": {
    "acc_norm": 0.8410757946210269,
    "acc_norm_stderr": 0.018100164006652885
  },
  "community|arabic_mmlu:Natural Science (Middle School)|0": {
    "acc_norm": 0.8925619834710744,
    "acc_norm_stderr": 0.019947554591049202
  },
  "community|arabic_mmlu:Natural Science (Primary School)|0": {
    "acc_norm": 0.9196428571428571,
    "acc_norm_stderr": 0.014852495890457517
  },
  "community|arabic_mmlu:Philosophy (High School)|0": {
    "acc_norm": 0.717948717948718,
    "acc_norm_stderr": 0.07299934324587597
  },
  "community|arabic_mmlu:Physics (High School)|0": {
    "acc_norm": 0.7215686274509804,
    "acc_norm_stderr": 0.02812425630369246
  },
  "community|arabic_mmlu:Political Science (University)|0": {
    "acc_norm": 0.7476190476190476,
    "acc_norm_stderr": 0.030046599156031494
  },
  "community|arabic_mmlu:Social Science (Middle School)|0": {
    "acc_norm": 0.6846473029045643,
    "acc_norm_stderr": 0.029993427997512336
  },
  "community|arabic_mmlu:Social Science (Primary School)|0": {
    "acc_norm": 0.8624113475177305,
    "acc_norm_stderr": 0.01298261595487575
  },
  "community|arabic_mmlu_ht:abstract_algebra|0": {
    "acc_norm": 0.55,
    "acc_norm_stderr": 0.05
  },
  "community|arabic_mmlu_ht:anatomy|0": {
    "acc_norm": 0.7481481481481481,
    "acc_norm_stderr": 0.03749850709174022
  },
  "community|arabic_mmlu_ht:astronomy|0": {
    "acc_norm": 0.881578947368421,
    "acc_norm_stderr": 0.026293995855474914
  },
  "community|arabic_mmlu_ht:business_ethics|0": {
    "acc_norm": 0.73,
    "acc_norm_stderr": 0.0446196043338474
  },
  "community|arabic_mmlu_ht:clinical_knowledge|0": {
    "acc_norm": 0.8226415094339623,
    "acc_norm_stderr": 0.02350873921884693
  },
  "community|arabic_mmlu_ht:college_biology|0": {
    "acc_norm": 0.8888888888888888,
    "acc_norm_stderr": 0.02628055093284808
  },
  "community|arabic_mmlu_ht:college_chemistry|0": {
    "acc_norm": 0.6,
    "acc_norm_stderr": 0.049236596391733084
  },
  "community|arabic_mmlu_ht:college_computer_science|0": {
    "acc_norm": 0.7,
    "acc_norm_stderr": 0.046056618647183814
  },
  "community|arabic_mmlu_ht:college_mathematics|0": {
    "acc_norm": 0.6,
    "acc_norm_stderr": 0.049236596391733084
  },
  "community|arabic_mmlu_ht:college_medicine|0": {
    "acc_norm": 0.7687861271676301,
    "acc_norm_stderr": 0.032147373020294696
  },
  "community|arabic_mmlu_ht:college_physics|0": {
    "acc_norm": 0.5980392156862745,
    "acc_norm_stderr": 0.04878608714466996
  },
  "community|arabic_mmlu_ht:computer_security|0": {
    "acc_norm": 0.77,
    "acc_norm_stderr": 0.04229525846816505
  },
  "community|arabic_mmlu_ht:conceptual_physics|0": {
    "acc_norm": 0.8382978723404255,
    "acc_norm_stderr": 0.024068505289695327
  },
  "community|arabic_mmlu_ht:econometrics|0": {
    "acc_norm": 0.5701754385964912,
    "acc_norm_stderr": 0.04657047260594964
  },
  "community|arabic_mmlu_ht:electrical_engineering|0": {
    "acc_norm": 0.7448275862068966,
    "acc_norm_stderr": 0.03632984052707842
  },
  "community|arabic_mmlu_ht:elementary_mathematics|0": {
    "acc_norm": 0.7328042328042328,
    "acc_norm_stderr": 0.02278967314577657
  },
  "community|arabic_mmlu_ht:formal_logic|0": {
    "acc_norm": 0.5952380952380952,
    "acc_norm_stderr": 0.04390259265377563
  },
  "community|arabic_mmlu_ht:global_facts|0": {
    "acc_norm": 0.54,
    "acc_norm_stderr": 0.05009082659620332
  },
  "community|arabic_mmlu_ht:high_school_biology|0": {
    "acc_norm": 0.919748229740362,
    "acc_norm_stderr": 0.004400330988630771
  },
  "community|arabic_mmlu_ht:high_school_chemistry|0": {
    "acc_norm": 0.916085657370518,
    "acc_norm_stderr": 0.00437565795032785
  },
  "community|arabic_mmlu_ht:high_school_computer_science|0": {
    "acc_norm": 0.81,
    "acc_norm_stderr": 0.03942772444036623
  },
  "community|arabic_mmlu_ht:high_school_european_history|0": {
    "acc_norm": 0.7672963689892051,
    "acc_norm_stderr": 0.004680343250904719
  },
  "community|arabic_mmlu_ht:high_school_geography|0": {
    "acc_norm": 0.8888888888888888,
    "acc_norm_stderr": 0.022390787638216766
  },
  "community|arabic_mmlu_ht:high_school_government_and_politics|0": {
    "acc_norm": 0.8963730569948186,
    "acc_norm_stderr": 0.02199531196364424
  },
  "community|arabic_mmlu_ht:high_school_macroeconomics|0": {
    "acc_norm": 0.937046004842615,
    "acc_norm_stderr": 0.004517968342248531
  },
  "community|arabic_mmlu_ht:high_school_mathematics|0": {
    "acc_norm": 0.5481481481481482,
    "acc_norm_stderr": 0.030343862998512623
  },
  "community|arabic_mmlu_ht:high_school_microeconomics|0": {
    "acc_norm": 0.8319327731092437,
    "acc_norm_stderr": 0.02428910211569227
  },
  "community|arabic_mmlu_ht:high_school_physics|0": {
    "acc_norm": 0.6556291390728477,
    "acc_norm_stderr": 0.03879687024073327
  },
  "community|arabic_mmlu_ht:high_school_psychology|0": {
    "acc_norm": 0.8862385321100917,
    "acc_norm_stderr": 0.013613614800232826
  },
  "community|arabic_mmlu_ht:high_school_statistics|0": {
    "acc_norm": 0.9085538752362949,
    "acc_norm_stderr": 0.004431354384952661
  },
  "community|arabic_mmlu_ht:high_school_us_history|0": {
    "acc_norm": 0.8676470588235294,
    "acc_norm_stderr": 0.02378429752091886
  },
  "community|arabic_mmlu_ht:high_school_world_history|0": {
    "acc_norm": 0.8945147679324894,
    "acc_norm_stderr": 0.019995560723758552
  },
  "community|arabic_mmlu_ht:human_aging|0": {
    "acc_norm": 0.7174887892376681,
    "acc_norm_stderr": 0.03021683101150876
  },
  "community|arabic_mmlu_ht:human_sexuality|0": {
    "acc_norm": 0.7557251908396947,
    "acc_norm_stderr": 0.03768335959728743
  },
  "community|arabic_mmlu_ht:international_law|0": {
    "acc_norm": 0.9008264462809917,
    "acc_norm_stderr": 0.027285246312758957
  },
  "community|arabic_mmlu_ht:jurisprudence|0": {
    "acc_norm": 0.8148148148148148,
    "acc_norm_stderr": 0.03755265865037181
  },
  "community|arabic_mmlu_ht:logical_fallacies|0": {
    "acc_norm": 0.7484662576687117,
    "acc_norm_stderr": 0.034089978868575295
  },
  "community|arabic_mmlu_ht:machine_learning|0": {
    "acc_norm": 0.5,
    "acc_norm_stderr": 0.04745789978762494
  },
  "community|arabic_mmlu_ht:management|0": {
    "acc_norm": 0.8640776699029126,
    "acc_norm_stderr": 0.033932957297610096
  },
  "community|arabic_mmlu_ht:marketing|0": {
    "acc_norm": 0.8760683760683761,
    "acc_norm_stderr": 0.021586494001281376
  },
  "community|arabic_mmlu_ht:medical_genetics|0": {
    "acc_norm": 0.88,
    "acc_norm_stderr": 0.03265986323710906
  },
  "community|arabic_mmlu_ht:miscellaneous|0": {
    "acc_norm": 0.952892561983471,
    "acc_norm_stderr": 0.004307732997762467
  },
  "community|arabic_mmlu_ht:moral_disputes|0": {
    "acc_norm": 0.7803468208092486,
    "acc_norm_stderr": 0.022289638852617883
  },
  "community|arabic_mmlu_ht:moral_scenarios|0": {
    "acc_norm": 0.5698324022346368,
    "acc_norm_stderr": 0.016558601636041035
  },
  "community|arabic_mmlu_ht:nutrition|0": {
    "acc_norm": 0.8333333333333334,
    "acc_norm_stderr": 0.021339479988816027
  },
  "community|arabic_mmlu_ht:philosophy|0": {
    "acc_norm": 0.8006430868167203,
    "acc_norm_stderr": 0.022691033780549656
  },
  "community|arabic_mmlu_ht:prehistory|0": {
    "acc_norm": 0.7993827160493827,
    "acc_norm_stderr": 0.02228231394977488
  },
  "community|arabic_mmlu_ht:professional_accounting|0": {
    "acc_norm": 0.8916703588834737,
    "acc_norm_stderr": 0.004626399692312275
  },
  "community|arabic_mmlu_ht:professional_law|0": {
    "acc_norm": 0.7649931138099412,
    "acc_norm_stderr": 0.004744652434682318
  },
  "community|arabic_mmlu_ht:professional_medicine|0": {
    "acc_norm": 0.9529627367135003,
    "acc_norm_stderr": 0.005234405127122951
  },
  "community|arabic_mmlu_ht:professional_psychology|0": {
    "acc_norm": 0.9192121039109334,
    "acc_norm_stderr": 0.004604929073163386
  },
  "community|arabic_mmlu_ht:public_relations|0": {
    "acc_norm": 0.7818181818181819,
    "acc_norm_stderr": 0.03955932861795833
  },
  "community|arabic_mmlu_ht:security_studies|0": {
    "acc_norm": 0.746938775510204,
    "acc_norm_stderr": 0.027833023871399677
  },
  "community|arabic_mmlu_ht:sociology|0": {
    "acc_norm": 0.8955223880597015,
    "acc_norm_stderr": 0.021628920516700637
  },
  "community|arabic_mmlu_ht:us_foreign_policy|0": {
    "acc_norm": 0.93,
    "acc_norm_stderr": 0.025643239997624294
  },
  "community|arabic_mmlu_ht:virology|0": {
    "acc_norm": 0.5542168674698795,
    "acc_norm_stderr": 0.03869543323472101
  },
  "community|arabic_mmlu_ht:world_religions|0": {
    "acc_norm": 0.8421052631578947,
    "acc_norm_stderr": 0.027966785859160907
  },
  "community|aratrust:Ethics|0": {
    "acc_norm": 0.7833333333333333,
    "acc_norm_stderr": 0.05363439040648223
  },
  "community|aratrust:Illegal|0": {
    "acc_norm": 0.9433962264150944,
    "acc_norm_stderr": 0.03204557643805462
  },
  "community|aratrust:MentalHealth|0": {
    "acc_norm": 0.9605263157894737,
    "acc_norm_stderr": 0.022484220382414568
  },
  "community|aratrust:Offensive|0": {
    "acc_norm": 0.8115942028985508,
    "acc_norm_stderr": 0.047420064740574204
  },
  "community|aratrust:PhysicalHealth|0": {
    "acc_norm": 0.8767123287671232,
    "acc_norm_stderr": 0.038745577051317655
  },
  "community|aratrust:Privacy|0": {
    "acc_norm": 0.9824561403508771,
    "acc_norm_stderr": 0.017543859649122806
  },
  "community|aratrust:Trustfulness|0": {
    "acc_norm": 0.782051282051282,
    "acc_norm_stderr": 0.047048893371093484
  },
  "community|aratrust:Unfairness|0": {
    "acc_norm": 0.9272727272727272,
    "acc_norm_stderr": 0.03533910239300182
  },
  "community|madinah_qa:Arabic Language (General)|0": {
    "acc_norm": 0.9084967320261438,
    "acc_norm_stderr": 0.01166431882942318
  },
  "community|madinah_qa:Arabic Language (Grammar)|0": {
    "acc_norm": 0.8547945205479452,
    "acc_norm_stderr": 0.018465950860407764
  },
  "community|alghafa:_average|0": {
    "acc_norm": 0.7901177074824354,
    "acc_norm_stderr": 0.01253242977046694
  },
  "community|arabic_mmlu:_average|0": {
    "acc_norm": 0.7912839996146945,
    "acc_norm_stderr": 0.02830871749479751
  },
  "community|arabic_mmlu_ht:_average|0": {
    "acc_norm": 0.7803660845353015,
    "acc_norm_stderr": 0.02753027779067881
  },
  "community|aratrust:_average|0": {
    "acc_norm": 0.8834178196098077,
    "acc_norm_stderr": 0.03678271055400767
  },
  "community|madinah_qa:_average|0": {
    "acc_norm": 0.8816456262870445,
    "acc_norm_stderr": 0.015065134844915472
  },
  "all": {
    "acc_norm": 0.7923874947069481,
    "acc_norm_stderr": 0.027129679467604573
  }
}
{
  "community|alghafa:meta_ar_dialects|0": 0,
  "community|alghafa:meta_ar_msa|0": 0,
  "community|alghafa:multiple_choice_facts_truefalse_balanced_task|0": 0,
  "community|alghafa:multiple_choice_grounded_statement_soqal_task|0": 0,
  "community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task|0": 0,
  "community|alghafa:multiple_choice_rating_sentiment_no_neutral_task|0": 0,
  "community|alghafa:multiple_choice_rating_sentiment_task|0": 0,
  "community|alghafa:multiple_choice_sentiment_task|0": 0,
  "community|arabic_exams|0": 0,
  "community|arabic_mmlu:Accounting (University)|0": 0,
  "community|arabic_mmlu:Arabic Language (General)|0": 0,
  "community|arabic_mmlu:Arabic Language (Grammar)|0": 0,
  "community|arabic_mmlu:Arabic Language (High School)|0": 0,
  "community|arabic_mmlu:Arabic Language (Middle School)|0": 0,
  "community|arabic_mmlu:Arabic Language (Primary School)|0": 0,
  "community|arabic_mmlu:Biology (High School)|0": 0,
  "community|arabic_mmlu:Civics (High School)|0": 0,
  "community|arabic_mmlu:Civics (Middle School)|0": 0,
  "community|arabic_mmlu:Computer Science (High School)|0": 0,
  "community|arabic_mmlu:Computer Science (Middle School)|0": 0,
  "community|arabic_mmlu:Computer Science (Primary School)|0": 0,
  "community|arabic_mmlu:Computer Science (University)|0": 0,
  "community|arabic_mmlu:Driving Test|0": 0,
  "community|arabic_mmlu:Economics (High School)|0": 0,
  "community|arabic_mmlu:Economics (Middle School)|0": 0,
  "community|arabic_mmlu:Economics (University)|0": 0,
  "community|arabic_mmlu:General Knowledge|0": 0,
  "community|arabic_mmlu:General Knowledge (Middle School)|0": 0,
  "community|arabic_mmlu:General Knowledge (Primary School)|0": 0,
  "community|arabic_mmlu:Geography (High School)|0": 0,
  "community|arabic_mmlu:Geography (Middle School)|0": 0,
  "community|arabic_mmlu:Geography (Primary School)|0": 0,
  "community|arabic_mmlu:History (High School)|0": 0,
  "community|arabic_mmlu:History (Middle School)|0": 0,
  "community|arabic_mmlu:History (Primary School)|0": 0,
  "community|arabic_mmlu:Islamic Studies|0": 0,
  "community|arabic_mmlu:Islamic Studies (High School)|0": 0,
  "community|arabic_mmlu:Islamic Studies (Middle School)|0": 0,
  "community|arabic_mmlu:Islamic Studies (Primary School)|0": 0,
  "community|arabic_mmlu:Law (Professional)|0": 0,
  "community|arabic_mmlu:Management (University)|0": 0,
  "community|arabic_mmlu:Math (Primary School)|0": 0,
  "community|arabic_mmlu:Natural Science (Middle School)|0": 0,
  "community|arabic_mmlu:Natural Science (Primary School)|0": 0,
  "community|arabic_mmlu:Philosophy (High School)|0": 0,
  "community|arabic_mmlu:Physics (High School)|0": 0,
  "community|arabic_mmlu:Political Science (University)|0": 0,
  "community|arabic_mmlu:Social Science (Middle School)|0": 0,
  "community|arabic_mmlu:Social Science (Primary School)|0": 0,
  "community|arabic_mmlu_ht:abstract_algebra|0": 0,
  "community|arabic_mmlu_ht:anatomy|0": 0,
  "community|arabic_mmlu_ht:astronomy|0": 0,
  "community|arabic_mmlu_ht:business_ethics|0": 0,
  "community|arabic_mmlu_ht:clinical_knowledge|0": 0,
  "community|arabic_mmlu_ht:college_biology|0": 0,
  "community|arabic_mmlu_ht:college_chemistry|0": 0,
  "community|arabic_mmlu_ht:college_computer_science|0": 0,
  "community|arabic_mmlu_ht:college_mathematics|0": 0,
  "community|arabic_mmlu_ht:college_medicine|0": 0,
  "community|arabic_mmlu_ht:college_physics|0": 0,
  "community|arabic_mmlu_ht:computer_security|0": 0,
  "community|arabic_mmlu_ht:conceptual_physics|0": 0,
  "community|arabic_mmlu_ht:econometrics|0": 0,
  "community|arabic_mmlu_ht:electrical_engineering|0": 0,
  "community|arabic_mmlu_ht:elementary_mathematics|0": 0,
  "community|arabic_mmlu_ht:formal_logic|0": 0,
  "community|arabic_mmlu_ht:global_facts|0": 0,
  "community|arabic_mmlu_ht:high_school_biology|0": 0,
  "community|arabic_mmlu_ht:high_school_chemistry|0": 0,
  "community|arabic_mmlu_ht:high_school_computer_science|0": 0,
  "community|arabic_mmlu_ht:high_school_european_history|0": 0,
  "community|arabic_mmlu_ht:high_school_geography|0": 0,
  "community|arabic_mmlu_ht:high_school_government_and_politics|0": 0,
  "community|arabic_mmlu_ht:high_school_macroeconomics|0": 0,
  "community|arabic_mmlu_ht:high_school_mathematics|0": 0,
  "community|arabic_mmlu_ht:high_school_microeconomics|0": 0,
  "community|arabic_mmlu_ht:high_school_physics|0": 0,
  "community|arabic_mmlu_ht:high_school_psychology|0": 0,
  "community|arabic_mmlu_ht:high_school_statistics|0": 0,
  "community|arabic_mmlu_ht:high_school_us_history|0": 0,
  "community|arabic_mmlu_ht:high_school_world_history|0": 0,
  "community|arabic_mmlu_ht:human_aging|0": 0,
  "community|arabic_mmlu_ht:human_sexuality|0": 0,
  "community|arabic_mmlu_ht:international_law|0": 0,
  "community|arabic_mmlu_ht:jurisprudence|0": 0,
  "community|arabic_mmlu_ht:logical_fallacies|0": 0,
  "community|arabic_mmlu_ht:machine_learning|0": 0,
  "community|arabic_mmlu_ht:management|0": 0,
  "community|arabic_mmlu_ht:marketing|0": 0,
  "community|arabic_mmlu_ht:medical_genetics|0": 0,
  "community|arabic_mmlu_ht:miscellaneous|0": 0,
  "community|arabic_mmlu_ht:moral_disputes|0": 0,
  "community|arabic_mmlu_ht:moral_scenarios|0": 0,
  "community|arabic_mmlu_ht:nutrition|0": 0,
  "community|arabic_mmlu_ht:philosophy|0": 0,
  "community|arabic_mmlu_ht:prehistory|0": 0,
  "community|arabic_mmlu_ht:professional_accounting|0": 0,
  "community|arabic_mmlu_ht:professional_law|0": 0,
  "community|arabic_mmlu_ht:professional_medicine|0": 0,
  "community|arabic_mmlu_ht:professional_psychology|0": 0,
  "community|arabic_mmlu_ht:public_relations|0": 0,
  "community|arabic_mmlu_ht:security_studies|0": 0,
  "community|arabic_mmlu_ht:sociology|0": 0,
  "community|arabic_mmlu_ht:us_foreign_policy|0": 0,
  "community|arabic_mmlu_ht:virology|0": 0,
  "community|arabic_mmlu_ht:world_religions|0": 0,
  "community|aratrust:Ethics|0": 0,
  "community|aratrust:Illegal|0": 0,
  "community|aratrust:MentalHealth|0": 0,
  "community|aratrust:Offensive|0": 0,
  "community|aratrust:PhysicalHealth|0": 0,
  "community|aratrust:Privacy|0": 0,
  "community|aratrust:Trustfulness|0": 0,
  "community|aratrust:Unfairness|0": 0,
  "community|madinah_qa:Arabic Language (General)|0": 0,
  "community|madinah_qa:Arabic Language (Grammar)|0": 0
}
{
  "community|alghafa:meta_ar_dialects": {
    "name": "alghafa:meta_ar_dialects",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "meta_ar_dialects",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 5395,
    "effective_num_docs": 5395,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:meta_ar_msa": {
    "name": "alghafa:meta_ar_msa",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "meta_ar_msa",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 895,
    "effective_num_docs": 895,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:multiple_choice_facts_truefalse_balanced_task": {
    "name": "alghafa:multiple_choice_facts_truefalse_balanced_task",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "multiple_choice_facts_truefalse_balanced_task",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 75,
    "effective_num_docs": 75,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:multiple_choice_grounded_statement_soqal_task": {
    "name": "alghafa:multiple_choice_grounded_statement_soqal_task",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "multiple_choice_grounded_statement_soqal_task",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 150,
    "effective_num_docs": 150,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task": {
    "name": "alghafa:multiple_choice_grounded_statement_xglue_mlqa_task",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "multiple_choice_grounded_statement_xglue_mlqa_task",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 150,
    "effective_num_docs": 150,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:multiple_choice_rating_sentiment_no_neutral_task": {
    "name": "alghafa:multiple_choice_rating_sentiment_no_neutral_task",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "multiple_choice_rating_sentiment_no_neutral_task",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 7995,
    "effective_num_docs": 7995,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:multiple_choice_rating_sentiment_task": {
    "name": "alghafa:multiple_choice_rating_sentiment_task",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "multiple_choice_rating_sentiment_task",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 5995,
    "effective_num_docs": 5995,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|alghafa:multiple_choice_sentiment_task": {
    "name": "alghafa:multiple_choice_sentiment_task",
    "prompt_function": "alghafa_pfn",
    "hf_repo": "OALL/AlGhafa-Arabic-LLM-Benchmark-Native",
    "hf_subset": "multiple_choice_sentiment_task",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 1720,
    "effective_num_docs": 1720,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_exams": {
    "name": "arabic_exams",
    "prompt_function": "arabic_exams_pfn",
    "hf_repo": "OALL/Arabic_EXAMS",
    "hf_subset": "default",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test",
      "validation"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": "validation",
    "few_shots_select": "sequential",
    "generation_size": null,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 537,
    "effective_num_docs": 537,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Accounting (University)": {
    "name": "arabic_mmlu:Accounting (University)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Accounting (University)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 74,
    "effective_num_docs": 74,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Arabic Language (General)": {
    "name": "arabic_mmlu:Arabic Language (General)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Arabic Language (General)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 612,
    "effective_num_docs": 612,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Arabic Language (Grammar)": {
    "name": "arabic_mmlu:Arabic Language (Grammar)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Arabic Language (Grammar)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 365,
    "effective_num_docs": 365,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Arabic Language (High School)": {
    "name": "arabic_mmlu:Arabic Language (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Arabic Language (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 390,
    "effective_num_docs": 390,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Arabic Language (Middle School)": {
    "name": "arabic_mmlu:Arabic Language (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Arabic Language (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 27,
    "effective_num_docs": 27,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Arabic Language (Primary School)": {
    "name": "arabic_mmlu:Arabic Language (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Arabic Language (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 252,
    "effective_num_docs": 252,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Biology (High School)": {
    "name": "arabic_mmlu:Biology (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Biology (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 1409,
    "effective_num_docs": 1409,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Civics (High School)": {
    "name": "arabic_mmlu:Civics (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Civics (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 87,
    "effective_num_docs": 87,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Civics (Middle School)": {
    "name": "arabic_mmlu:Civics (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Civics (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 236,
    "effective_num_docs": 236,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Computer Science (High School)": {
    "name": "arabic_mmlu:Computer Science (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Computer Science (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 261,
    "effective_num_docs": 261,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Computer Science (Middle School)": {
    "name": "arabic_mmlu:Computer Science (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Computer Science (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 27,
    "effective_num_docs": 27,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Computer Science (Primary School)": {
    "name": "arabic_mmlu:Computer Science (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Computer Science (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 190,
    "effective_num_docs": 190,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Computer Science (University)": {
    "name": "arabic_mmlu:Computer Science (University)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Computer Science (University)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 64,
    "effective_num_docs": 64,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Driving Test": {
    "name": "arabic_mmlu:Driving Test",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Driving Test",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 1211,
    "effective_num_docs": 1211,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Economics (High School)": {
    "name": "arabic_mmlu:Economics (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Economics (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 360,
    "effective_num_docs": 360,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Economics (Middle School)": {
    "name": "arabic_mmlu:Economics (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Economics (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 87,
    "effective_num_docs": 87,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Economics (University)": {
    "name": "arabic_mmlu:Economics (University)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Economics (University)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 137,
    "effective_num_docs": 137,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:General Knowledge": {
    "name": "arabic_mmlu:General Knowledge",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "General Knowledge",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 864,
    "effective_num_docs": 864,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:General Knowledge (Middle School)": {
    "name": "arabic_mmlu:General Knowledge (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "General Knowledge (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 172,
    "effective_num_docs": 172,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:General Knowledge (Primary School)": {
    "name": "arabic_mmlu:General Knowledge (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "General Knowledge (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 162,
    "effective_num_docs": 162,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Geography (High School)": {
    "name": "arabic_mmlu:Geography (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Geography (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 1038,
    "effective_num_docs": 1038,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Geography (Middle School)": {
    "name": "arabic_mmlu:Geography (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Geography (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 272,
    "effective_num_docs": 272,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Geography (Primary School)": {
    "name": "arabic_mmlu:Geography (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Geography (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 57,
    "effective_num_docs": 57,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:History (High School)": {
    "name": "arabic_mmlu:History (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "History (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 760,
    "effective_num_docs": 760,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:History (Middle School)": {
    "name": "arabic_mmlu:History (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "History (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 203,
    "effective_num_docs": 203,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:History (Primary School)": {
    "name": "arabic_mmlu:History (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "History (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 102,
    "effective_num_docs": 102,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Islamic Studies": {
    "name": "arabic_mmlu:Islamic Studies",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Islamic Studies",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 639,
    "effective_num_docs": 639,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Islamic Studies (High School)": {
    "name": "arabic_mmlu:Islamic Studies (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Islamic Studies (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 334,
    "effective_num_docs": 334,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Islamic Studies (Middle School)": {
    "name": "arabic_mmlu:Islamic Studies (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Islamic Studies (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 238,
    "effective_num_docs": 238,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Islamic Studies (Primary School)": {
    "name": "arabic_mmlu:Islamic Studies (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Islamic Studies (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 999,
    "effective_num_docs": 999,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Law (Professional)": {
    "name": "arabic_mmlu:Law (Professional)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Law (Professional)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 314,
    "effective_num_docs": 314,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Management (University)": {
    "name": "arabic_mmlu:Management (University)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Management (University)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 75,
    "effective_num_docs": 75,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Math (Primary School)": {
    "name": "arabic_mmlu:Math (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Math (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 409,
    "effective_num_docs": 409,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Natural Science (Middle School)": {
    "name": "arabic_mmlu:Natural Science (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Natural Science (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 242,
    "effective_num_docs": 242,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Natural Science (Primary School)": {
    "name": "arabic_mmlu:Natural Science (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Natural Science (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 336,
    "effective_num_docs": 336,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Philosophy (High School)": {
    "name": "arabic_mmlu:Philosophy (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Philosophy (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 39,
    "effective_num_docs": 39,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Physics (High School)": {
    "name": "arabic_mmlu:Physics (High School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Physics (High School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 255,
    "effective_num_docs": 255,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Political Science (University)": {
    "name": "arabic_mmlu:Political Science (University)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Political Science (University)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 210,
    "effective_num_docs": 210,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Social Science (Middle School)": {
    "name": "arabic_mmlu:Social Science (Middle School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Social Science (Middle School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 241,
    "effective_num_docs": 241,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu:Social Science (Primary School)": {
    "name": "arabic_mmlu:Social Science (Primary School)",
    "prompt_function": "arabic_mmlu_pfn",
    "hf_repo": "MBZUAI/ArabicMMLU",
    "hf_subset": "Social Science (Primary School)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 705,
    "effective_num_docs": 705,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:abstract_algebra": {
    "name": "arabic_mmlu_ht:abstract_algebra",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "abstract_algebra",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:anatomy": {
    "name": "arabic_mmlu_ht:anatomy",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "anatomy",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 135,
    "effective_num_docs": 135,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:astronomy": {
    "name": "arabic_mmlu_ht:astronomy",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "astronomy",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 152,
    "effective_num_docs": 152,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:business_ethics": {
    "name": "arabic_mmlu_ht:business_ethics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "business_ethics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:clinical_knowledge": {
    "name": "arabic_mmlu_ht:clinical_knowledge",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "clinical_knowledge",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 265,
    "effective_num_docs": 265,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:college_biology": {
    "name": "arabic_mmlu_ht:college_biology",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "college_biology",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 144,
    "effective_num_docs": 144,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:college_chemistry": {
    "name": "arabic_mmlu_ht:college_chemistry",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "college_chemistry",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:college_computer_science": {
    "name": "arabic_mmlu_ht:college_computer_science",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "college_computer_science",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:college_mathematics": {
    "name": "arabic_mmlu_ht:college_mathematics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "college_mathematics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:college_medicine": {
    "name": "arabic_mmlu_ht:college_medicine",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "college_medicine",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 173,
    "effective_num_docs": 173,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:college_physics": {
    "name": "arabic_mmlu_ht:college_physics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "college_physics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 102,
    "effective_num_docs": 102,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:computer_security": {
    "name": "arabic_mmlu_ht:computer_security",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "computer_security",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:conceptual_physics": {
    "name": "arabic_mmlu_ht:conceptual_physics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "conceptual_physics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 235,
    "effective_num_docs": 235,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:econometrics": {
    "name": "arabic_mmlu_ht:econometrics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "econometrics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 114,
    "effective_num_docs": 114,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:electrical_engineering": {
    "name": "arabic_mmlu_ht:electrical_engineering",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "electrical_engineering",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 145,
    "effective_num_docs": 145,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:elementary_mathematics": {
    "name": "arabic_mmlu_ht:elementary_mathematics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "elementary_mathematics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 378,
    "effective_num_docs": 378,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:formal_logic": {
    "name": "arabic_mmlu_ht:formal_logic",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "formal_logic",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 126,
    "effective_num_docs": 126,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:global_facts": {
    "name": "arabic_mmlu_ht:global_facts",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "global_facts",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_biology": {
    "name": "arabic_mmlu_ht:high_school_biology",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_biology",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 3813,
    "effective_num_docs": 3813,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_chemistry": {
    "name": "arabic_mmlu_ht:high_school_chemistry",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_chemistry",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 4016,
    "effective_num_docs": 4016,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_computer_science": {
    "name": "arabic_mmlu_ht:high_school_computer_science",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_computer_science",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_european_history": {
    "name": "arabic_mmlu_ht:high_school_european_history",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_european_history",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 8152,
    "effective_num_docs": 8152,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_geography": {
    "name": "arabic_mmlu_ht:high_school_geography",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_geography",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 198,
    "effective_num_docs": 198,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_government_and_politics": {
    "name": "arabic_mmlu_ht:high_school_government_and_politics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_government_and_politics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 193,
    "effective_num_docs": 193,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_macroeconomics": {
    "name": "arabic_mmlu_ht:high_school_macroeconomics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_macroeconomics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 2891,
    "effective_num_docs": 2891,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_mathematics": {
    "name": "arabic_mmlu_ht:high_school_mathematics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_mathematics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 270,
    "effective_num_docs": 270,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_microeconomics": {
    "name": "arabic_mmlu_ht:high_school_microeconomics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_microeconomics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 238,
    "effective_num_docs": 238,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_physics": {
    "name": "arabic_mmlu_ht:high_school_physics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_physics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 151,
    "effective_num_docs": 151,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_psychology": {
    "name": "arabic_mmlu_ht:high_school_psychology",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_psychology",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 545,
    "effective_num_docs": 545,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_statistics": {
    "name": "arabic_mmlu_ht:high_school_statistics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_statistics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 4232,
    "effective_num_docs": 4232,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_us_history": {
    "name": "arabic_mmlu_ht:high_school_us_history",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_us_history",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 204,
    "effective_num_docs": 204,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:high_school_world_history": {
    "name": "arabic_mmlu_ht:high_school_world_history",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "high_school_world_history",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 237,
    "effective_num_docs": 237,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:human_aging": {
    "name": "arabic_mmlu_ht:human_aging",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "human_aging",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 223,
    "effective_num_docs": 223,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:human_sexuality": {
    "name": "arabic_mmlu_ht:human_sexuality",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "human_sexuality",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 131,
    "effective_num_docs": 131,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:international_law": {
    "name": "arabic_mmlu_ht:international_law",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "international_law",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 121,
    "effective_num_docs": 121,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:jurisprudence": {
    "name": "arabic_mmlu_ht:jurisprudence",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "jurisprudence",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 108,
    "effective_num_docs": 108,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:logical_fallacies": {
    "name": "arabic_mmlu_ht:logical_fallacies",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "logical_fallacies",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 163,
    "effective_num_docs": 163,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:machine_learning": {
    "name": "arabic_mmlu_ht:machine_learning",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "machine_learning",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 112,
    "effective_num_docs": 112,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:management": {
    "name": "arabic_mmlu_ht:management",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "management",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 103,
    "effective_num_docs": 103,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:marketing": {
    "name": "arabic_mmlu_ht:marketing",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "marketing",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 234,
    "effective_num_docs": 234,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:medical_genetics": {
    "name": "arabic_mmlu_ht:medical_genetics",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "medical_genetics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:miscellaneous": {
    "name": "arabic_mmlu_ht:miscellaneous",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "miscellaneous",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 2420,
    "effective_num_docs": 2420,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:moral_disputes": {
    "name": "arabic_mmlu_ht:moral_disputes",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "moral_disputes",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 346,
    "effective_num_docs": 346,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:moral_scenarios": {
    "name": "arabic_mmlu_ht:moral_scenarios",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "moral_scenarios",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 895,
    "effective_num_docs": 895,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:nutrition": {
    "name": "arabic_mmlu_ht:nutrition",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "nutrition",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 306,
    "effective_num_docs": 306,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:philosophy": {
    "name": "arabic_mmlu_ht:philosophy",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "philosophy",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 311,
    "effective_num_docs": 311,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:prehistory": {
    "name": "arabic_mmlu_ht:prehistory",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "prehistory",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 324,
    "effective_num_docs": 324,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:professional_accounting": {
    "name": "arabic_mmlu_ht:professional_accounting",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "professional_accounting",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 4514,
    "effective_num_docs": 4514,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:professional_law": {
    "name": "arabic_mmlu_ht:professional_law",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "professional_law",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 7987,
    "effective_num_docs": 7987,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:professional_medicine": {
    "name": "arabic_mmlu_ht:professional_medicine",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "professional_medicine",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 1637,
    "effective_num_docs": 1637,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:professional_psychology": {
    "name": "arabic_mmlu_ht:professional_psychology",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "professional_psychology",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 3503,
    "effective_num_docs": 3503,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:public_relations": {
    "name": "arabic_mmlu_ht:public_relations",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "public_relations",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 110,
    "effective_num_docs": 110,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:security_studies": {
    "name": "arabic_mmlu_ht:security_studies",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "security_studies",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 245,
    "effective_num_docs": 245,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:sociology": {
    "name": "arabic_mmlu_ht:sociology",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "sociology",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 201,
    "effective_num_docs": 201,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:us_foreign_policy": {
    "name": "arabic_mmlu_ht:us_foreign_policy",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "us_foreign_policy",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 100,
    "effective_num_docs": 100,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:virology": {
    "name": "arabic_mmlu_ht:virology",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "virology",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 166,
    "effective_num_docs": 166,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|arabic_mmlu_ht:world_religions": {
    "name": "arabic_mmlu_ht:world_religions",
    "prompt_function": "arabic_mmlu_ht_pfn",
    "hf_repo": "MBZUAI/human_translated_arabic_mmlu",
    "hf_subset": "world_religions",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 171,
    "effective_num_docs": 171,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:Ethics": {
    "name": "aratrust:Ethics",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "Ethics",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 60,
    "effective_num_docs": 60,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:Illegal": {
    "name": "aratrust:Illegal",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "Illegal",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 53,
    "effective_num_docs": 53,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:MentalHealth": {
    "name": "aratrust:MentalHealth",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "MentalHealth",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 76,
    "effective_num_docs": 76,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:Offensive": {
    "name": "aratrust:Offensive",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "Offensive",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 69,
    "effective_num_docs": 69,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:PhysicalHealth": {
    "name": "aratrust:PhysicalHealth",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "PhysicalHealth",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 73,
    "effective_num_docs": 73,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:Privacy": {
    "name": "aratrust:Privacy",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "Privacy",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 57,
    "effective_num_docs": 57,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:Trustfulness": {
    "name": "aratrust:Trustfulness",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "Trustfulness",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 78,
    "effective_num_docs": 78,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|aratrust:Unfairness": {
    "name": "aratrust:Unfairness",
    "prompt_function": "aratrust_pfn",
    "hf_repo": "asas-ai/AraTrust-categorized",
    "hf_subset": "Unfairness",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "train"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "train"
    ],
    "few_shots_split": null,
    "few_shots_select": null,
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 55,
    "effective_num_docs": 55,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|madinah_qa:Arabic Language (General)": {
    "name": "madinah_qa:Arabic Language (General)",
    "prompt_function": "madinah_qa_pfn",
    "hf_repo": "MBZUAI/MadinahQA",
    "hf_subset": "Arabic Language (General)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 612,
    "effective_num_docs": 612,
    "must_remove_duplicate_docs": false,
    "version": 0
  },
  "community|madinah_qa:Arabic Language (Grammar)": {
    "name": "madinah_qa:Arabic Language (Grammar)",
    "prompt_function": "madinah_qa_pfn",
    "hf_repo": "MBZUAI/MadinahQA",
    "hf_subset": "Arabic Language (Grammar)",
    "metric": [
      {
        "metric_name": "acc_norm",
        "higher_is_better": true,
        "category": "8",
        "use_case": "1",
        "sample_level_fn": "compute",
        "corpus_level_fn": "mean"
      }
    ],
    "hf_revision": null,
    "hf_filter": null,
    "hf_avail_splits": [
      "test"
    ],
    "trust_dataset": true,
    "evaluation_splits": [
      "test"
    ],
    "few_shots_split": [
      "dev"
    ],
    "few_shots_select": "sequential",
    "generation_size": -1,
    "generation_grammar": null,
    "stop_sequence": [],
    "num_samples": null,
    "suite": [
      "community"
    ],
    "original_num_docs": 365,
    "effective_num_docs": 365,
    "must_remove_duplicate_docs": false,
    "version": 0
  }
}
{
  "community|alghafa:meta_ar_dialects|0": {
    "hashes": {
      "hash_examples": "c0b6081f83e14064",
      "hash_full_prompts": "697d1eedfdb9c04c",
      "hash_input_tokens": "d4a0db365411644e",
      "hash_cont_tokens": "0787d6e25363632e"
    },
    "truncated": 0,
    "non_truncated": 5395,
    "padded": 0,
    "non_padded": 21580,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:meta_ar_msa|0": {
    "hashes": {
      "hash_examples": "64eb78a7c5b7484b",
      "hash_full_prompts": "3e3d8fb5dcd581d1",
      "hash_input_tokens": "d990861a06693abf",
      "hash_cont_tokens": "9419419f52941664"
    },
    "truncated": 0,
    "non_truncated": 895,
    "padded": 0,
    "non_padded": 3580,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:multiple_choice_facts_truefalse_balanced_task|0": {
    "hashes": {
      "hash_examples": "54fc3502c1c02c06",
      "hash_full_prompts": "8d202b92b5bbc680",
      "hash_input_tokens": "0fb80c9443bf5241",
      "hash_cont_tokens": "6b8a184674b81d9e"
    },
    "truncated": 0,
    "non_truncated": 75,
    "padded": 0,
    "non_padded": 150,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:multiple_choice_grounded_statement_soqal_task|0": {
    "hashes": {
      "hash_examples": "46572d83696552ae",
      "hash_full_prompts": "abea4c437d96afea",
      "hash_input_tokens": "3d681520d78d4836",
      "hash_cont_tokens": "081e673a9b5034d1"
    },
    "truncated": 0,
    "non_truncated": 150,
    "padded": 0,
    "non_padded": 750,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:multiple_choice_grounded_statement_xglue_mlqa_task|0": {
    "hashes": {
      "hash_examples": "f430d97ff715bc1c",
      "hash_full_prompts": "a0bcf40e918e6155",
      "hash_input_tokens": "55cffc968c3ffac6",
      "hash_cont_tokens": "081e673a9b5034d1"
    },
    "truncated": 0,
    "non_truncated": 150,
    "padded": 0,
    "non_padded": 750,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:multiple_choice_rating_sentiment_no_neutral_task|0": {
    "hashes": {
      "hash_examples": "6b70a7416584f98c",
      "hash_full_prompts": "439c16f6cb92a3c0",
      "hash_input_tokens": "e5536f539dc949bb",
      "hash_cont_tokens": "fbec465fdd3933cd"
    },
    "truncated": 0,
    "non_truncated": 7995,
    "padded": 0,
    "non_padded": 15990,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:multiple_choice_rating_sentiment_task|0": {
    "hashes": {
      "hash_examples": "bc2005cc9d2f436e",
      "hash_full_prompts": "1030e11d7cf70b2d",
      "hash_input_tokens": "5060f63674a767b0",
      "hash_cont_tokens": "4a2b3f8ddebfb48d"
    },
    "truncated": 0,
    "non_truncated": 5995,
    "padded": 0,
    "non_padded": 17985,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|alghafa:multiple_choice_sentiment_task|0": {
    "hashes": {
      "hash_examples": "6fb0e254ea5945d8",
      "hash_full_prompts": "890f5941deb3c0f8",
      "hash_input_tokens": "bb6df780b215a6af",
      "hash_cont_tokens": "edaa2e057c5deb01"
    },
    "truncated": 0,
    "non_truncated": 1720,
    "padded": 0,
    "non_padded": 5160,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_exams|0": {
    "hashes": {
      "hash_examples": "6d721df351722656",
      "hash_full_prompts": "55e0657bdc4b57ec",
      "hash_input_tokens": "c37229aa6ac70d58",
      "hash_cont_tokens": "a4ef65bc7bab8dcf"
    },
    "truncated": 0,
    "non_truncated": 537,
    "padded": 0,
    "non_padded": 2148,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Accounting (University)|0": {
    "hashes": {
      "hash_examples": "30e09697562ff9e7",
      "hash_full_prompts": "4741945dc05c404c",
      "hash_input_tokens": "f5e5d86273b34891",
      "hash_cont_tokens": "587bf4caea1658f4"
    },
    "truncated": 0,
    "non_truncated": 74,
    "padded": 0,
    "non_padded": 256,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Arabic Language (General)|0": {
    "hashes": {
      "hash_examples": "bef69fb8b3b75f28",
      "hash_full_prompts": "defb906f381cfcd9",
      "hash_input_tokens": "5a05b0da3f02e6f9",
      "hash_cont_tokens": "95a234c727b7b43c"
    },
    "truncated": 0,
    "non_truncated": 612,
    "padded": 0,
    "non_padded": 2403,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Arabic Language (Grammar)|0": {
    "hashes": {
      "hash_examples": "bd066a9e6a140a4b",
      "hash_full_prompts": "070068c411fb3997",
      "hash_input_tokens": "02e16347fc37ae64",
      "hash_cont_tokens": "98f874e7446c544b"
    },
    "truncated": 0,
    "non_truncated": 365,
    "padded": 0,
    "non_padded": 1588,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Arabic Language (High School)|0": {
    "hashes": {
      "hash_examples": "a9c2cd9a9929292a",
      "hash_full_prompts": "b876f89bc9dcafa1",
      "hash_input_tokens": "934afcec25d3575c",
      "hash_cont_tokens": "3d676d0d2f081e05"
    },
    "truncated": 0,
    "non_truncated": 390,
    "padded": 0,
    "non_padded": 1525,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Arabic Language (Middle School)|0": {
    "hashes": {
      "hash_examples": "2f8a77bbbd0e21ff",
      "hash_full_prompts": "e677edecf0bd75d5",
      "hash_input_tokens": "863315292beaedaf",
      "hash_cont_tokens": "322ea7667dfc2c2d"
    },
    "truncated": 0,
    "non_truncated": 27,
    "padded": 0,
    "non_padded": 105,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Arabic Language (Primary School)|0": {
    "hashes": {
      "hash_examples": "5eed3da47822539b",
      "hash_full_prompts": "94d49814387fbb4e",
      "hash_input_tokens": "125f6290472105af",
      "hash_cont_tokens": "f3c78f80ddea1519"
    },
    "truncated": 0,
    "non_truncated": 252,
    "padded": 0,
    "non_padded": 926,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Biology (High School)|0": {
    "hashes": {
      "hash_examples": "91ae6d22a0f0213d",
      "hash_full_prompts": "16e71c6d0e42d330",
      "hash_input_tokens": "ac756f900da51586",
      "hash_cont_tokens": "aaa20fdc3c06d2c3"
    },
    "truncated": 0,
    "non_truncated": 1409,
    "padded": 0,
    "non_padded": 5056,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Civics (High School)|0": {
    "hashes": {
      "hash_examples": "f27bf8791bea2bb9",
      "hash_full_prompts": "8094808a6f0fa9b9",
      "hash_input_tokens": "48e749cfe9fa2906",
      "hash_cont_tokens": "e02c7ebfec7f8df8"
    },
    "truncated": 0,
    "non_truncated": 87,
    "padded": 0,
    "non_padded": 316,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Civics (Middle School)|0": {
    "hashes": {
      "hash_examples": "74f5bb0098c8916f",
      "hash_full_prompts": "47c41a36aa6573c3",
      "hash_input_tokens": "7756c4ebae2a2557",
      "hash_cont_tokens": "1ffdd9a463183bfa"
    },
    "truncated": 0,
    "non_truncated": 236,
    "padded": 0,
    "non_padded": 944,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Computer Science (High School)|0": {
    "hashes": {
      "hash_examples": "a4278d7b525d46fe",
      "hash_full_prompts": "4191fc5811855ee4",
      "hash_input_tokens": "43b63720cc350cc7",
      "hash_cont_tokens": "821feca3d9004c98"
    },
    "truncated": 0,
    "non_truncated": 261,
    "padded": 0,
    "non_padded": 1006,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Computer Science (Middle School)|0": {
    "hashes": {
      "hash_examples": "0cb6c07e4b80dfd4",
      "hash_full_prompts": "bb0869b87598266f",
      "hash_input_tokens": "b7dd67a213ea4059",
      "hash_cont_tokens": "8b4f299b6f012a83"
    },
    "truncated": 0,
    "non_truncated": 27,
    "padded": 0,
    "non_padded": 100,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Computer Science (Primary School)|0": {
    "hashes": {
      "hash_examples": "d96fc1bc32473533",
      "hash_full_prompts": "7ed3631dd316f004",
      "hash_input_tokens": "c71023cdb189eb90",
      "hash_cont_tokens": "1bc67f97b48b9ece"
    },
    "truncated": 0,
    "non_truncated": 190,
    "padded": 0,
    "non_padded": 476,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Computer Science (University)|0": {
    "hashes": {
      "hash_examples": "8835587e436cbaff",
      "hash_full_prompts": "6250444a7b9604b2",
      "hash_input_tokens": "a249aba6b8d9770d",
      "hash_cont_tokens": "e9d871459bc85f62"
    },
    "truncated": 0,
    "non_truncated": 64,
    "padded": 0,
    "non_padded": 255,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Driving Test|0": {
    "hashes": {
      "hash_examples": "7a4c38a2c451d075",
      "hash_full_prompts": "45b7b936d82c1150",
      "hash_input_tokens": "cf578483b42d6305",
      "hash_cont_tokens": "cd411982b0f12d43"
    },
    "truncated": 0,
    "non_truncated": 1211,
    "padded": 0,
    "non_padded": 3685,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Economics (High School)|0": {
    "hashes": {
      "hash_examples": "c04c252836601279",
      "hash_full_prompts": "214ea55ed2547a5e",
      "hash_input_tokens": "af033a4fe342a437",
      "hash_cont_tokens": "4bda66df90f2d4d8"
    },
    "truncated": 0,
    "non_truncated": 360,
    "padded": 0,
    "non_padded": 1390,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Economics (Middle School)|0": {
    "hashes": {
      "hash_examples": "18fba1579406b3cc",
      "hash_full_prompts": "bf2e646324d90429",
      "hash_input_tokens": "01f15416dbc52be4",
      "hash_cont_tokens": "3ea283b0f50a72f5"
    },
    "truncated": 0,
    "non_truncated": 87,
    "padded": 0,
    "non_padded": 348,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Economics (University)|0": {
    "hashes": {
      "hash_examples": "7c9e86fba8151562",
      "hash_full_prompts": "cb280cc805670549",
      "hash_input_tokens": "d47e59c553c042e3",
      "hash_cont_tokens": "91cdb256248a5bdf"
    },
    "truncated": 0,
    "non_truncated": 137,
    "padded": 0,
    "non_padded": 544,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:General Knowledge|0": {
    "hashes": {
      "hash_examples": "acfbe4e1f0314b85",
      "hash_full_prompts": "755f37afe33f7559",
      "hash_input_tokens": "582a23425090181e",
      "hash_cont_tokens": "76d704fbedbe5ab8"
    },
    "truncated": 0,
    "non_truncated": 864,
    "padded": 0,
    "non_padded": 3213,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:General Knowledge (Middle School)|0": {
    "hashes": {
      "hash_examples": "03cd0ecf10224316",
      "hash_full_prompts": "900b57bd55d23559",
      "hash_input_tokens": "1cb2e25dae471a2d",
      "hash_cont_tokens": "aff2aed9268be2e2"
    },
    "truncated": 0,
    "non_truncated": 172,
    "padded": 0,
    "non_padded": 628,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:General Knowledge (Primary School)|0": {
    "hashes": {
      "hash_examples": "c3ee30196e05e122",
      "hash_full_prompts": "a0a424f0a3b0eac3",
      "hash_input_tokens": "618c51af21b712b4",
      "hash_cont_tokens": "6c8978669cdc11fb"
    },
    "truncated": 0,
    "non_truncated": 162,
    "padded": 0,
    "non_padded": 637,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Geography (High School)|0": {
    "hashes": {
      "hash_examples": "e2e329d2bdd9fb7b",
      "hash_full_prompts": "5e3f92a804772548",
      "hash_input_tokens": "868e9b261ab576d0",
      "hash_cont_tokens": "37e2e9c548d8c904"
    },
    "truncated": 0,
    "non_truncated": 1038,
    "padded": 0,
    "non_padded": 4116,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Geography (Middle School)|0": {
    "hashes": {
      "hash_examples": "420b161444291989",
      "hash_full_prompts": "7b0ccab5b80729bd",
      "hash_input_tokens": "9f2927ec7e80ce2e",
      "hash_cont_tokens": "5e24bb4c8be23901"
    },
    "truncated": 0,
    "non_truncated": 272,
    "padded": 0,
    "non_padded": 975,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Geography (Primary School)|0": {
    "hashes": {
      "hash_examples": "5bc5ca48a4210899",
      "hash_full_prompts": "492ea8c528d1a4ce",
      "hash_input_tokens": "cf2abd8deef720ad",
      "hash_cont_tokens": "b9f45957a97d1ecf"
    },
    "truncated": 0,
    "non_truncated": 57,
    "padded": 0,
    "non_padded": 216,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:History (High School)|0": {
    "hashes": {
      "hash_examples": "c7cc37f29311bea1",
      "hash_full_prompts": "3b1781c2884844f2",
      "hash_input_tokens": "37c029b0ca69c67e",
      "hash_cont_tokens": "b16e65544485acae"
    },
    "truncated": 0,
    "non_truncated": 760,
    "padded": 0,
    "non_padded": 2962,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:History (Middle School)|0": {
    "hashes": {
      "hash_examples": "5b9f1973337153a2",
      "hash_full_prompts": "61496c59903567ab",
      "hash_input_tokens": "4186706fbdbdf44e",
      "hash_cont_tokens": "e3b355a58a286ee4"
    },
    "truncated": 0,
    "non_truncated": 203,
    "padded": 0,
    "non_padded": 746,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:History (Primary School)|0": {
    "hashes": {
      "hash_examples": "af2469847007c1fe",
      "hash_full_prompts": "bef057b12f592695",
      "hash_input_tokens": "8a90678a7dbba4f2",
      "hash_cont_tokens": "a52a22630c3cb3f7"
    },
    "truncated": 0,
    "non_truncated": 102,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Islamic Studies|0": {
    "hashes": {
      "hash_examples": "c8da9b2f16a5ea0f",
      "hash_full_prompts": "0e5f1eb074dc0576",
      "hash_input_tokens": "a0d4449f6b0c0385",
      "hash_cont_tokens": "1866597a67ff4424"
    },
    "truncated": 0,
    "non_truncated": 639,
    "padded": 0,
    "non_padded": 2529,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Islamic Studies (High School)|0": {
    "hashes": {
      "hash_examples": "efb11bc8ef398117",
      "hash_full_prompts": "f744eb11c6b54d16",
      "hash_input_tokens": "adce96a4f351e6da",
      "hash_cont_tokens": "6b678abb2fd451bd"
    },
    "truncated": 0,
    "non_truncated": 334,
    "padded": 0,
    "non_padded": 1285,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Islamic Studies (Middle School)|0": {
    "hashes": {
      "hash_examples": "9e33ab030eebdb99",
      "hash_full_prompts": "7813be863908efee",
      "hash_input_tokens": "ad906fd4ce3a9946",
      "hash_cont_tokens": "e0c922e595ad51cd"
    },
    "truncated": 0,
    "non_truncated": 238,
    "padded": 0,
    "non_padded": 883,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Islamic Studies (Primary School)|0": {
    "hashes": {
      "hash_examples": "4167565d878b20eb",
      "hash_full_prompts": "4aca4836da321f21",
      "hash_input_tokens": "205e1781dd6e264f",
      "hash_cont_tokens": "97e2182a63c5686c"
    },
    "truncated": 0,
    "non_truncated": 999,
    "padded": 0,
    "non_padded": 3024,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Law (Professional)|0": {
    "hashes": {
      "hash_examples": "e77f52c8fe4352b3",
      "hash_full_prompts": "a3be12bf791af8de",
      "hash_input_tokens": "becb9d69cb250e37",
      "hash_cont_tokens": "324cc46c561b417c"
    },
    "truncated": 0,
    "non_truncated": 314,
    "padded": 0,
    "non_padded": 1232,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Management (University)|0": {
    "hashes": {
      "hash_examples": "09682649b04b7327",
      "hash_full_prompts": "78d4da7370157ec4",
      "hash_input_tokens": "aaf1b9399f2efdbb",
      "hash_cont_tokens": "1e98e1e2cd19a5e3"
    },
    "truncated": 0,
    "non_truncated": 75,
    "padded": 0,
    "non_padded": 200,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Math (Primary School)|0": {
    "hashes": {
      "hash_examples": "edb027bfae7e76f1",
      "hash_full_prompts": "178f2eb46937183b",
      "hash_input_tokens": "c61dc12cee88d842",
      "hash_cont_tokens": "632401a080490684"
    },
    "truncated": 0,
    "non_truncated": 409,
    "padded": 0,
    "non_padded": 1296,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Natural Science (Middle School)|0": {
    "hashes": {
      "hash_examples": "96e72c9094c2364c",
      "hash_full_prompts": "239009c63aea5254",
      "hash_input_tokens": "8686a68aa785c50e",
      "hash_cont_tokens": "17e42af5dbb9eee1"
    },
    "truncated": 0,
    "non_truncated": 242,
    "padded": 0,
    "non_padded": 940,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Natural Science (Primary School)|0": {
    "hashes": {
      "hash_examples": "69e35bad3dec5a4d",
      "hash_full_prompts": "4944591b3477da56",
      "hash_input_tokens": "cfd47431ff1515f1",
      "hash_cont_tokens": "a7423721c9837336"
    },
    "truncated": 0,
    "non_truncated": 336,
    "padded": 0,
    "non_padded": 1228,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Philosophy (High School)|0": {
    "hashes": {
      "hash_examples": "dc6ebd484a02fca5",
      "hash_full_prompts": "2e02d39134ca27b9",
      "hash_input_tokens": "73107f364adb6bcf",
      "hash_cont_tokens": "69b31fc6977897bf"
    },
    "truncated": 0,
    "non_truncated": 39,
    "padded": 0,
    "non_padded": 156,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Physics (High School)|0": {
    "hashes": {
      "hash_examples": "58a1722472c9e644",
      "hash_full_prompts": "8775cd855e0037a0",
      "hash_input_tokens": "28d4c0baf14b4b54",
      "hash_cont_tokens": "f9506aa86f66954d"
    },
    "truncated": 0,
    "non_truncated": 255,
    "padded": 0,
    "non_padded": 1020,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Political Science (University)|0": {
    "hashes": {
      "hash_examples": "07a4ed6aabbdfd1e",
      "hash_full_prompts": "37e2532e45ac8d56",
      "hash_input_tokens": "3a4df3364e93d261",
      "hash_cont_tokens": "4799b66f49438465"
    },
    "truncated": 0,
    "non_truncated": 210,
    "padded": 0,
    "non_padded": 710,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Social Science (Middle School)|0": {
    "hashes": {
      "hash_examples": "8ca955902f304664",
      "hash_full_prompts": "d16202ba6df674e8",
      "hash_input_tokens": "b98151c1160dfec8",
      "hash_cont_tokens": "4602cb88db99312d"
    },
    "truncated": 0,
    "non_truncated": 241,
    "padded": 0,
    "non_padded": 929,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu:Social Science (Primary School)|0": {
    "hashes": {
      "hash_examples": "934025ab3738123c",
      "hash_full_prompts": "978eed85ae542f82",
      "hash_input_tokens": "f70c0a524035a6e6",
      "hash_cont_tokens": "19e973e9f05c9c82"
    },
    "truncated": 0,
    "non_truncated": 705,
    "padded": 0,
    "non_padded": 2043,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:abstract_algebra|0": {
    "hashes": {
      "hash_examples": "0b557911f2f6d919",
      "hash_full_prompts": "b9b74df23feeb3d4",
      "hash_input_tokens": "8f2754b76dc3cafb",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:anatomy|0": {
    "hashes": {
      "hash_examples": "a552d8a0ef294061",
      "hash_full_prompts": "b06f5aaf0f85df9f",
      "hash_input_tokens": "030bdc7e69aab14e",
      "hash_cont_tokens": "96c000fa61c3bd55"
    },
    "truncated": 0,
    "non_truncated": 135,
    "padded": 0,
    "non_padded": 540,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:astronomy|0": {
    "hashes": {
      "hash_examples": "c4a372d0af7da098",
      "hash_full_prompts": "3a822121db18d697",
      "hash_input_tokens": "f8b593caec1e9c19",
      "hash_cont_tokens": "b13cc32205751d90"
    },
    "truncated": 0,
    "non_truncated": 152,
    "padded": 0,
    "non_padded": 608,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:business_ethics|0": {
    "hashes": {
      "hash_examples": "9f71d816abf8af7a",
      "hash_full_prompts": "9c3db18f134ba3a8",
      "hash_input_tokens": "01eb411d7c7ceeae",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:clinical_knowledge|0": {
    "hashes": {
      "hash_examples": "38303cd765589ef3",
      "hash_full_prompts": "b5da720810123a41",
      "hash_input_tokens": "195b866953484587",
      "hash_cont_tokens": "c771582839d4f30c"
    },
    "truncated": 0,
    "non_truncated": 265,
    "padded": 0,
    "non_padded": 1060,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:college_biology|0": {
    "hashes": {
      "hash_examples": "dbd9b5d318e60b04",
      "hash_full_prompts": "35dc7c41ef73879a",
      "hash_input_tokens": "f2f2d4b2043e56f2",
      "hash_cont_tokens": "ec774ac0d0ad658b"
    },
    "truncated": 0,
    "non_truncated": 144,
    "padded": 0,
    "non_padded": 576,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:college_chemistry|0": {
    "hashes": {
      "hash_examples": "6f88491d03db8a4c",
      "hash_full_prompts": "60538c856b87e6f8",
      "hash_input_tokens": "b11ad967aa853641",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:college_computer_science|0": {
    "hashes": {
      "hash_examples": "ebfdee5ef2ed5e17",
      "hash_full_prompts": "f5c6bef4fe087efc",
      "hash_input_tokens": "91447268eab049f5",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:college_mathematics|0": {
    "hashes": {
      "hash_examples": "e3f22cd7712aae2f",
      "hash_full_prompts": "23654f332df33413",
      "hash_input_tokens": "d55424a3c07a769f",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:college_medicine|0": {
    "hashes": {
      "hash_examples": "51a5501373afb5a7",
      "hash_full_prompts": "cd2cef8530d399a0",
      "hash_input_tokens": "d86e48bb624769c8",
      "hash_cont_tokens": "1823a754e6394181"
    },
    "truncated": 0,
    "non_truncated": 173,
    "padded": 0,
    "non_padded": 692,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:college_physics|0": {
    "hashes": {
      "hash_examples": "2d3e015989b108db",
      "hash_full_prompts": "e410e1b05e941a97",
      "hash_input_tokens": "10c3e2c656ed19e5",
      "hash_cont_tokens": "ee5dc873d27b9e10"
    },
    "truncated": 0,
    "non_truncated": 102,
    "padded": 0,
    "non_padded": 408,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:computer_security|0": {
    "hashes": {
      "hash_examples": "f8810eddc38dfee4",
      "hash_full_prompts": "42cc7d49956effc3",
      "hash_input_tokens": "b014b9028b81b403",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:conceptual_physics|0": {
    "hashes": {
      "hash_examples": "211e32cc43c6b1dc",
      "hash_full_prompts": "ff91fa0a4e67f9a8",
      "hash_input_tokens": "d8b5909ca4bd3881",
      "hash_cont_tokens": "b7b580bbcf7e0afa"
    },
    "truncated": 0,
    "non_truncated": 235,
    "padded": 0,
    "non_padded": 940,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:econometrics|0": {
    "hashes": {
      "hash_examples": "810023786b2484d2",
      "hash_full_prompts": "eb28f6258f649706",
      "hash_input_tokens": "2d0b44e853599df1",
      "hash_cont_tokens": "d44932b2a931e093"
    },
    "truncated": 0,
    "non_truncated": 114,
    "padded": 0,
    "non_padded": 456,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:electrical_engineering|0": {
    "hashes": {
      "hash_examples": "a222760c93eaa1ee",
      "hash_full_prompts": "68e10a631a57f618",
      "hash_input_tokens": "3715e9d50d40a0c1",
      "hash_cont_tokens": "159f4cb1232d2a3c"
    },
    "truncated": 0,
    "non_truncated": 145,
    "padded": 0,
    "non_padded": 580,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:elementary_mathematics|0": {
    "hashes": {
      "hash_examples": "4c069aeee64dc227",
      "hash_full_prompts": "012257987ce861de",
      "hash_input_tokens": "71ac9ddc9537c52f",
      "hash_cont_tokens": "2bf44b70baf49dfa"
    },
    "truncated": 0,
    "non_truncated": 378,
    "padded": 0,
    "non_padded": 1512,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:formal_logic|0": {
    "hashes": {
      "hash_examples": "3cb0ccbf8e8a77ae",
      "hash_full_prompts": "1092202f23e61e35",
      "hash_input_tokens": "3ab8052fb4e0b808",
      "hash_cont_tokens": "8765c45f4711ebb8"
    },
    "truncated": 0,
    "non_truncated": 126,
    "padded": 0,
    "non_padded": 504,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:global_facts|0": {
    "hashes": {
      "hash_examples": "c1d039e64ea321b9",
      "hash_full_prompts": "7bfe9d88e40df3ee",
      "hash_input_tokens": "65a2a63ff3d30945",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_biology|0": {
    "hashes": {
      "hash_examples": "ddcb8237bb4ba08a",
      "hash_full_prompts": "db5ce7a5a6f1dbc3",
      "hash_input_tokens": "51db364da8bcdb9e",
      "hash_cont_tokens": "49908817551a4513"
    },
    "truncated": 0,
    "non_truncated": 3813,
    "padded": 0,
    "non_padded": 15252,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_chemistry|0": {
    "hashes": {
      "hash_examples": "07061b55c5c436d9",
      "hash_full_prompts": "742ff8decec1118f",
      "hash_input_tokens": "c35ba60d264d9c5f",
      "hash_cont_tokens": "a7f16a586e1cfe0f"
    },
    "truncated": 0,
    "non_truncated": 4016,
    "padded": 0,
    "non_padded": 16064,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_computer_science|0": {
    "hashes": {
      "hash_examples": "8d3405483d5fdcff",
      "hash_full_prompts": "544efc214e04a185",
      "hash_input_tokens": "0d5889dbb5e4eb5a",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_european_history|0": {
    "hashes": {
      "hash_examples": "031c49a430356414",
      "hash_full_prompts": "36062098867d2fdf",
      "hash_input_tokens": "26c34aeb69f969c4",
      "hash_cont_tokens": "5420388845898571"
    },
    "truncated": 0,
    "non_truncated": 8152,
    "padded": 0,
    "non_padded": 32608,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_geography|0": {
    "hashes": {
      "hash_examples": "d0ce2b019a66c1de",
      "hash_full_prompts": "fc69d2030755028c",
      "hash_input_tokens": "3b84bab864459f99",
      "hash_cont_tokens": "fa4a2c8384dfaaa5"
    },
    "truncated": 0,
    "non_truncated": 198,
    "padded": 0,
    "non_padded": 792,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_government_and_politics|0": {
    "hashes": {
      "hash_examples": "7d7c6d476d0576b1",
      "hash_full_prompts": "ea01912a5b3de0d4",
      "hash_input_tokens": "bf4b3a0ca3e41d71",
      "hash_cont_tokens": "682709d2fa91c75e"
    },
    "truncated": 0,
    "non_truncated": 193,
    "padded": 0,
    "non_padded": 772,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_macroeconomics|0": {
    "hashes": {
      "hash_examples": "694d3a01c6144ddb",
      "hash_full_prompts": "067fc8d5b4c2f558",
      "hash_input_tokens": "c6f75d3085f76384",
      "hash_cont_tokens": "4f2f97c723cb220f"
    },
    "truncated": 0,
    "non_truncated": 2891,
    "padded": 0,
    "non_padded": 11564,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_mathematics|0": {
    "hashes": {
      "hash_examples": "004f9c0a40b5ec10",
      "hash_full_prompts": "1baa89b754304381",
      "hash_input_tokens": "54868b0c06f0336f",
      "hash_cont_tokens": "8130a825e5a2ee3d"
    },
    "truncated": 0,
    "non_truncated": 270,
    "padded": 0,
    "non_padded": 1080,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_microeconomics|0": {
    "hashes": {
      "hash_examples": "80cf03d462e6ccbc",
      "hash_full_prompts": "45f2c96cec5cd28c",
      "hash_input_tokens": "ca47a408404428d1",
      "hash_cont_tokens": "4f6974070ef28d29"
    },
    "truncated": 0,
    "non_truncated": 238,
    "padded": 0,
    "non_padded": 952,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_physics|0": {
    "hashes": {
      "hash_examples": "92218def5b383845",
      "hash_full_prompts": "9d8ce8051351cf76",
      "hash_input_tokens": "d338ac26612d8fa0",
      "hash_cont_tokens": "5d32bcd7ba8252ba"
    },
    "truncated": 0,
    "non_truncated": 151,
    "padded": 0,
    "non_padded": 604,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_psychology|0": {
    "hashes": {
      "hash_examples": "323f7848fee32e58",
      "hash_full_prompts": "63e76443cb7e9247",
      "hash_input_tokens": "c2f999dd7bd988c5",
      "hash_cont_tokens": "1512a6938229952b"
    },
    "truncated": 0,
    "non_truncated": 545,
    "padded": 0,
    "non_padded": 2180,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_statistics|0": {
    "hashes": {
      "hash_examples": "d7bbe0d037cf31ec",
      "hash_full_prompts": "31e18f289d3ff6bd",
      "hash_input_tokens": "d27b1c315445b55f",
      "hash_cont_tokens": "95cb29e5c31221c8"
    },
    "truncated": 0,
    "non_truncated": 4232,
    "padded": 0,
    "non_padded": 16928,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_us_history|0": {
    "hashes": {
      "hash_examples": "722ec9207e3b0e04",
      "hash_full_prompts": "a8049bd5bf8ffccf",
      "hash_input_tokens": "583a446bd537f78e",
      "hash_cont_tokens": "0c31c2de1e3429bf"
    },
    "truncated": 0,
    "non_truncated": 204,
    "padded": 0,
    "non_padded": 816,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:high_school_world_history|0": {
    "hashes": {
      "hash_examples": "b5eb675d3b578584",
      "hash_full_prompts": "51d86b4490ac7e71",
      "hash_input_tokens": "781dcfe1c4af3774",
      "hash_cont_tokens": "5e704d9d54138833"
    },
    "truncated": 0,
    "non_truncated": 237,
    "padded": 0,
    "non_padded": 948,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:human_aging|0": {
    "hashes": {
      "hash_examples": "713ac79cd2dd2d7b",
      "hash_full_prompts": "f863ba5d24da8adc",
      "hash_input_tokens": "19ce4cadfa46d838",
      "hash_cont_tokens": "e5a3e63957647f04"
    },
    "truncated": 0,
    "non_truncated": 223,
    "padded": 0,
    "non_padded": 892,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:human_sexuality|0": {
    "hashes": {
      "hash_examples": "47551ab4e5dcf6c5",
      "hash_full_prompts": "eb92874ade3ce5f1",
      "hash_input_tokens": "515bef28478bed18",
      "hash_cont_tokens": "90a9b6d1231332f4"
    },
    "truncated": 0,
    "non_truncated": 131,
    "padded": 0,
    "non_padded": 524,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:international_law|0": {
    "hashes": {
      "hash_examples": "da360336943398d5",
      "hash_full_prompts": "03f5f8d05ff8b8d3",
      "hash_input_tokens": "347d82dc9b55b22b",
      "hash_cont_tokens": "9ab33ab519d55748"
    },
    "truncated": 0,
    "non_truncated": 121,
    "padded": 0,
    "non_padded": 484,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:jurisprudence|0": {
    "hashes": {
      "hash_examples": "661d161a486fb035",
      "hash_full_prompts": "a6713089705fd8ee",
      "hash_input_tokens": "cea59ad1babfe009",
      "hash_cont_tokens": "2cecb6db3790a23b"
    },
    "truncated": 0,
    "non_truncated": 108,
    "padded": 0,
    "non_padded": 432,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:logical_fallacies|0": {
    "hashes": {
      "hash_examples": "5c3926384758bda7",
      "hash_full_prompts": "12f9bceb00903955",
      "hash_input_tokens": "55abf397ffb00e8c",
      "hash_cont_tokens": "f5c60e363dd9fc3d"
    },
    "truncated": 0,
    "non_truncated": 163,
    "padded": 0,
    "non_padded": 652,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:machine_learning|0": {
    "hashes": {
      "hash_examples": "3ce756e6a22ffc48",
      "hash_full_prompts": "11f5a6e1e0c98e08",
      "hash_input_tokens": "b405e043dd5d9a7a",
      "hash_cont_tokens": "d41e7e44237c0a16"
    },
    "truncated": 0,
    "non_truncated": 112,
    "padded": 0,
    "non_padded": 448,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:management|0": {
    "hashes": {
      "hash_examples": "20fe769bb3276832",
      "hash_full_prompts": "7062687337a3a595",
      "hash_input_tokens": "38a9503006fb7377",
      "hash_cont_tokens": "372864196dbb4cad"
    },
    "truncated": 0,
    "non_truncated": 103,
    "padded": 0,
    "non_padded": 412,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:marketing|0": {
    "hashes": {
      "hash_examples": "6b19449559d987ce",
      "hash_full_prompts": "9d23a14914d796f3",
      "hash_input_tokens": "43d5b1d347a4a823",
      "hash_cont_tokens": "ad74b6b4e88f6100"
    },
    "truncated": 0,
    "non_truncated": 234,
    "padded": 0,
    "non_padded": 936,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:medical_genetics|0": {
    "hashes": {
      "hash_examples": "cbb0fa9df0f5435a",
      "hash_full_prompts": "9891ab73be820b06",
      "hash_input_tokens": "4b9303a985def007",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:miscellaneous|0": {
    "hashes": {
      "hash_examples": "0a4134046c23cff9",
      "hash_full_prompts": "cb347937c22a0fcb",
      "hash_input_tokens": "68dc36db9204d0d2",
      "hash_cont_tokens": "f2831dc319b7001c"
    },
    "truncated": 0,
    "non_truncated": 2420,
    "padded": 0,
    "non_padded": 9680,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:moral_disputes|0": {
    "hashes": {
      "hash_examples": "1ac8a0967c82caa0",
      "hash_full_prompts": "49fadb27be0ce7aa",
      "hash_input_tokens": "8e4e96d444df39d4",
      "hash_cont_tokens": "d6a32c4f89ec0e43"
    },
    "truncated": 0,
    "non_truncated": 346,
    "padded": 0,
    "non_padded": 1384,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:moral_scenarios|0": {
    "hashes": {
      "hash_examples": "2c0670188bc5a789",
      "hash_full_prompts": "c4129c0b4487975b",
      "hash_input_tokens": "3f41c4a3a8cffcf0",
      "hash_cont_tokens": "0abad6841e9b5dc1"
    },
    "truncated": 0,
    "non_truncated": 895,
    "padded": 0,
    "non_padded": 3580,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:nutrition|0": {
    "hashes": {
      "hash_examples": "658628c0dcdfe201",
      "hash_full_prompts": "dc303807e16966cd",
      "hash_input_tokens": "a1558476f989f99d",
      "hash_cont_tokens": "1947ff415070dfa5"
    },
    "truncated": 0,
    "non_truncated": 306,
    "padded": 0,
    "non_padded": 1224,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:philosophy|0": {
    "hashes": {
      "hash_examples": "8b6707b322affafd",
      "hash_full_prompts": "7a7cfceedd3d2ea9",
      "hash_input_tokens": "5e05dee63f4fd651",
      "hash_cont_tokens": "566ed263a8423f58"
    },
    "truncated": 0,
    "non_truncated": 311,
    "padded": 0,
    "non_padded": 1244,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:prehistory|0": {
    "hashes": {
      "hash_examples": "0c85ffcdc9a7b367",
      "hash_full_prompts": "438b0c8c256bd402",
      "hash_input_tokens": "474a6cb853a01f35",
      "hash_cont_tokens": "69725bb3099f23d0"
    },
    "truncated": 0,
    "non_truncated": 324,
    "padded": 0,
    "non_padded": 1296,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:professional_accounting|0": {
    "hashes": {
      "hash_examples": "cce1ea2d5f544b2f",
      "hash_full_prompts": "0b347aee49687cec",
      "hash_input_tokens": "11d732a2fb0766c6",
      "hash_cont_tokens": "25802ac32c51a7f7"
    },
    "truncated": 0,
    "non_truncated": 4514,
    "padded": 0,
    "non_padded": 18056,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:professional_law|0": {
    "hashes": {
      "hash_examples": "1c654b024b54eb4b",
      "hash_full_prompts": "5eaef9276cbb03a1",
      "hash_input_tokens": "0e7f1050ab751db1",
      "hash_cont_tokens": "7f2b1b7218a1ef40"
    },
    "truncated": 0,
    "non_truncated": 7987,
    "padded": 0,
    "non_padded": 31948,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:professional_medicine|0": {
    "hashes": {
      "hash_examples": "c621eaacfa662ebc",
      "hash_full_prompts": "827b2000a1b95969",
      "hash_input_tokens": "d10264fa5974dfa0",
      "hash_cont_tokens": "e1afe1503a5d02c5"
    },
    "truncated": 0,
    "non_truncated": 1637,
    "padded": 0,
    "non_padded": 6548,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:professional_psychology|0": {
    "hashes": {
      "hash_examples": "bc14a28eaec87dc4",
      "hash_full_prompts": "ac74eed216ed2aa4",
      "hash_input_tokens": "3e7877b790bcd40e",
      "hash_cont_tokens": "1c41f3eeadeec685"
    },
    "truncated": 0,
    "non_truncated": 3503,
    "padded": 0,
    "non_padded": 14012,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:public_relations|0": {
    "hashes": {
      "hash_examples": "de4989d9375885c4",
      "hash_full_prompts": "b2a1a4d76e245a88",
      "hash_input_tokens": "cc23a3ef57a6fdda",
      "hash_cont_tokens": "3914ab4a5d5b69e8"
    },
    "truncated": 0,
    "non_truncated": 110,
    "padded": 0,
    "non_padded": 440,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:security_studies|0": {
    "hashes": {
      "hash_examples": "3f84bfeec717c6de",
      "hash_full_prompts": "2bdaa60ae6cdb812",
      "hash_input_tokens": "9af6d4fa28520458",
      "hash_cont_tokens": "1d2b199df736dea4"
    },
    "truncated": 0,
    "non_truncated": 245,
    "padded": 0,
    "non_padded": 980,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:sociology|0": {
    "hashes": {
      "hash_examples": "10d7c2fae10bfcbc",
      "hash_full_prompts": "6dc64bcb9a8e74b3",
      "hash_input_tokens": "238273a55fde267b",
      "hash_cont_tokens": "3556cb090eda6dec"
    },
    "truncated": 0,
    "non_truncated": 201,
    "padded": 0,
    "non_padded": 804,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:us_foreign_policy|0": {
    "hashes": {
      "hash_examples": "bb05f02c38ddaf1a",
      "hash_full_prompts": "2c78f15085fe5d18",
      "hash_input_tokens": "ab9318807b63e6c5",
      "hash_cont_tokens": "d35519013f781909"
    },
    "truncated": 0,
    "non_truncated": 100,
    "padded": 0,
    "non_padded": 400,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:virology|0": {
    "hashes": {
      "hash_examples": "290915a48884ede2",
      "hash_full_prompts": "6ac0831aaf7baaa6",
      "hash_input_tokens": "09a250f9f32ac5f0",
      "hash_cont_tokens": "cbf93f8f3bd5c82c"
    },
    "truncated": 0,
    "non_truncated": 166,
    "padded": 0,
    "non_padded": 664,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|arabic_mmlu_ht:world_religions|0": {
    "hashes": {
      "hash_examples": "91cc5451c7284f75",
      "hash_full_prompts": "bc7e3635516606d1",
      "hash_input_tokens": "e179572d39fbceb9",
      "hash_cont_tokens": "b5fbc024ac54a858"
    },
    "truncated": 0,
    "non_truncated": 171,
    "padded": 0,
    "non_padded": 684,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:Ethics|0": {
    "hashes": {
      "hash_examples": "b77354655caca219",
      "hash_full_prompts": "3b6c35af9dd55ba5",
      "hash_input_tokens": "077529e82d06fc5b",
      "hash_cont_tokens": "67fe5dc315ef723c"
    },
    "truncated": 0,
    "non_truncated": 60,
    "padded": 0,
    "non_padded": 180,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:Illegal|0": {
    "hashes": {
      "hash_examples": "daa90cfb03dd9ed8",
      "hash_full_prompts": "a1a797d500db526f",
      "hash_input_tokens": "6b8509a338fe1c01",
      "hash_cont_tokens": "2cc82a58b4d87abc"
    },
    "truncated": 0,
    "non_truncated": 53,
    "padded": 0,
    "non_padded": 159,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:MentalHealth|0": {
    "hashes": {
      "hash_examples": "ca046355c96d95d9",
      "hash_full_prompts": "f0b0bf58012fc511",
      "hash_input_tokens": "46b83034d3854773",
      "hash_cont_tokens": "7b399d0f0a9124f1"
    },
    "truncated": 0,
    "non_truncated": 76,
    "padded": 0,
    "non_padded": 228,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:Offensive|0": {
    "hashes": {
      "hash_examples": "6ff77d23c0f3113d",
      "hash_full_prompts": "e44f96f8e6f119af",
      "hash_input_tokens": "0bd14975b5ba3734",
      "hash_cont_tokens": "0cd5015bc3370adf"
    },
    "truncated": 0,
    "non_truncated": 69,
    "padded": 0,
    "non_padded": 207,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:PhysicalHealth|0": {
    "hashes": {
      "hash_examples": "085db2421f8abf29",
      "hash_full_prompts": "76b6bac76dbaf06c",
      "hash_input_tokens": "53364ea1ad410b57",
      "hash_cont_tokens": "cb8655dcad91858d"
    },
    "truncated": 0,
    "non_truncated": 73,
    "padded": 0,
    "non_padded": 219,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:Privacy|0": {
    "hashes": {
      "hash_examples": "78f4d16753b18c49",
      "hash_full_prompts": "63c7263b4ad8a155",
      "hash_input_tokens": "9bac5956cf324d23",
      "hash_cont_tokens": "7f23416c661e2ee5"
    },
    "truncated": 0,
    "non_truncated": 57,
    "padded": 0,
    "non_padded": 171,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:Trustfulness|0": {
    "hashes": {
      "hash_examples": "373f72b4e30243c4",
      "hash_full_prompts": "9b519b28f08eebb6",
      "hash_input_tokens": "56fa0e407be90e01",
      "hash_cont_tokens": "ff874dba360c1ede"
    },
    "truncated": 0,
    "non_truncated": 78,
    "padded": 0,
    "non_padded": 234,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|aratrust:Unfairness|0": {
    "hashes": {
      "hash_examples": "51fa7940e42ffcc6",
      "hash_full_prompts": "d5ca4d44e29290d9",
      "hash_input_tokens": "d62c109dd60f2e07",
      "hash_cont_tokens": "3e990fe3a474dbc5"
    },
    "truncated": 0,
    "non_truncated": 55,
    "padded": 0,
    "non_padded": 165,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|madinah_qa:Arabic Language (General)|0": {
    "hashes": {
      "hash_examples": "25bf94d05f737b63",
      "hash_full_prompts": "5c56043c8155627b",
      "hash_input_tokens": "c43c9cb443d82598",
      "hash_cont_tokens": "95a234c727b7b43c"
    },
    "truncated": 0,
    "non_truncated": 612,
    "padded": 0,
    "non_padded": 2403,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  },
  "community|madinah_qa:Arabic Language (Grammar)|0": {
    "hashes": {
      "hash_examples": "e65fe4df843f4380",
      "hash_full_prompts": "454a1d8530051200",
      "hash_input_tokens": "8ed81cfab877b4a1",
      "hash_cont_tokens": "98f874e7446c544b"
    },
    "truncated": 0,
    "non_truncated": 365,
    "padded": 0,
    "non_padded": 1588,
    "effective_few_shots": 0.0,
    "num_truncated_few_shots": 0
  }
}