File size: 5,363 Bytes
dfbcab4 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 | paradigm,Gen-3.6 (13.9M),GPT-2-large (774M),Transformer-XL (~257M),LSTM LM (Gulordava et al.),5-gram LM,Human (individual),gen36_beats_gpt2_large,gen36_beats_transformer_xl,gen36_beats_lstm,gen36_beats_ngram_5gram,gen36_beats_human
adjunct_island,73.3%,89.4%,64.5%,67.4%,47.9%,94.1%,,YES,YES,YES,
anaphor_gender_agreement,83.3%,99.4%,91.2%,88.2%,43.9%,96.0%,,,,YES,
anaphor_number_agreement,86.7%,99.2%,97.0%,95.2%,51.9%,99.0%,,,,YES,
animate_subject_passive,60.0%,76.6%,74.3%,71.5%,70.3%,86.1%,,,,,
animate_subject_trans,76.7%,84.9%,88.5%,86.8%,90.7%,87.0%,,,,,
causative,76.7%,78.2%,57.5%,68.4%,53.5%,98.0%,,YES,YES,YES,
complex_NP_island,53.3%,72.2%,57.5%,47.4%,49.7%,80.0%,,,YES,YES,
coordinate_structure_constraint_complex_left_branch,20.0%,81.0%,35.8%,30.1%,32.0%,90.0%,,,,,
coordinate_structure_constraint_object_extraction,60.0%,85.4%,74.4%,70.5%,59.3%,90.9%,,,,YES,
determiner_noun_agreement_1,93.3%,98.8%,91.7%,91.8%,88.3%,95.0%,,YES,YES,YES,
determiner_noun_agreement_2,100.0%,97.7%,81.1%,91.9%,86.3%,95.0%,YES,YES,YES,YES,YES
determiner_noun_agreement_irregular_1,76.7%,95.7%,88.3%,81.5%,85.4%,92.0%,,,,,
determiner_noun_agreement_irregular_2,80.0%,95.3%,81.6%,86.0%,90.4%,84.2%,,,,,
determiner_noun_agreement_with_adj_2,90.0%,94.9%,78.1%,86.2%,49.7%,96.0%,,YES,YES,YES,
determiner_noun_agreement_with_adj_irregular_1,80.0%,92.7%,80.8%,76.2%,53.0%,94.0%,,,YES,YES,
determiner_noun_agreement_with_adj_irregular_2,76.7%,94.0%,76.6%,82.9%,55.4%,85.0%,,YES,,YES,
determiner_noun_agreement_with_adjective_1,80.0%,97.5%,85.9%,86.7%,51.9%,95.0%,,,,YES,
distractor_agreement_relational_noun,73.3%,79.5%,77.2%,76.0%,23.5%,81.0%,,,,YES,
distractor_agreement_relative_clause,50.0%,65.7%,59.8%,62.7%,21.7%,86.0%,,,,YES,
drop_argument,66.7%,80.7%,70.3%,78.9%,71.6%,85.1%,,,,,
ellipsis_n_bar_1,53.3%,91.5%,65.1%,67.8%,23.4%,91.1%,,,,YES,
ellipsis_n_bar_2,96.7%,87.2%,89.2%,67.4%,50.4%,78.0%,YES,YES,YES,YES,YES
existential_there_object_raising,60.0%,78.4%,75.9%,65.9%,83.5%,90.0%,,,,,
existential_there_quantifiers_1,100.0%,99.5%,93.7%,96.3%,90.5%,94.0%,YES,YES,YES,YES,YES
existential_there_quantifiers_2,13.3%,42.4%,14.0%,16.2%,62.3%,76.0%,,,,,
existential_there_subject_raising,73.3%,91.1%,78.7%,79.5%,77.4%,88.0%,,,,,
expletive_it_object_raising,66.7%,79.2%,71.5%,63.0%,71.5%,86.0%,,,YES,,
inchoative,53.3%,65.9%,54.3%,65.3%,50.9%,82.0%,,,,YES,
intransitive,63.3%,84.2%,67.2%,78.5%,67.8%,90.0%,,,,,
irregular_past_participle_adjectives,100.0%,97.7%,90.5%,92.8%,79.4%,99.0%,YES,YES,YES,YES,YES
irregular_past_participle_verbs,90.0%,86.1%,65.8%,85.3%,79.6%,95.0%,YES,YES,YES,YES,
irregular_plural_subject_verb_agreement_1,76.7%,92.8%,77.6%,81.1%,72.6%,94.0%,,,,YES,
irregular_plural_subject_verb_agreement_2,86.7%,92.4%,82.7%,89.1%,87.6%,95.0%,,YES,,,
left_branch_island_echo_question,20.0%,52.3%,63.0%,31.6%,95.8%,91.0%,,,,,
left_branch_island_simple_question,13.3%,87.1%,35.5%,36.1%,56.5%,99.0%,,,,,
matrix_question_npi_licensor_present,10.0%,65.4%,0.7%,1.6%,1.2%,98.0%,,YES,YES,YES,
npi_present_1,16.7%,64.8%,60.5%,53.6%,46.8%,83.0%,,,,,
npi_present_2,40.0%,64.3%,48.2%,53.9%,46.9%,96.1%,,,,,
only_npi_licensor_present,73.3%,94.5%,79.8%,92.9%,56.6%,91.1%,,,,YES,
only_npi_scope,86.7%,78.5%,45.3%,36.4%,29.8%,72.0%,YES,YES,YES,YES,YES
passive_1,80.0%,89.3%,80.6%,71.9%,89.2%,95.0%,,,YES,,
passive_2,83.3%,90.2%,80.9%,72.5%,82.2%,86.0%,,YES,YES,YES,
principle_A_c_command,76.7%,73.7%,60.8%,58.8%,57.8%,86.0%,YES,YES,YES,YES,
principle_A_case_1,100.0%,100.0%,100.0%,100.0%,100.0%,98.0%,,,,,YES
principle_A_case_2,90.0%,94.8%,95.3%,87.3%,48.9%,96.0%,,,YES,YES,
principle_A_domain_1,100.0%,98.4%,99.1%,98.3%,95.2%,95.0%,YES,YES,YES,YES,YES
principle_A_domain_2,46.7%,77.5%,69.7%,68.3%,56.4%,75.0%,,,,,
principle_A_domain_3,56.7%,75.4%,60.0%,55.2%,52.2%,83.0%,,,YES,YES,
principle_A_reconstruction,46.7%,46.8%,37.9%,46.4%,40.4%,78.2%,,YES,YES,YES,
regular_plural_subject_verb_agreement_1,80.0%,96.7%,73.1%,89.0%,75.5%,94.1%,,YES,,YES,
regular_plural_subject_verb_agreement_2,96.7%,91.0%,85.3%,82.5%,80.9%,95.0%,YES,YES,YES,YES,YES
sentential_negation_npi_licensor_present,100.0%,97.1%,98.8%,100.0%,92.7%,93.1%,YES,YES,,YES,YES
sentential_negation_npi_scope,43.3%,73.2%,53.3%,23.2%,44.7%,81.0%,,,YES,,
sentential_subject_island,43.3%,35.5%,36.5%,42.8%,60.5%,60.0%,YES,YES,YES,,
superlative_quantifiers_1,76.7%,87.0%,84.3%,62.8%,44.7%,91.0%,,,YES,YES,
superlative_quantifiers_2,66.7%,87.2%,85.2%,82.6%,16.5%,85.0%,,,,YES,
tough_vs_raising_1,30.0%,72.0%,45.2%,33.6%,32.6%,75.2%,,,,,
tough_vs_raising_2,73.3%,88.9%,86.1%,93.0%,77.4%,81.0%,,,,,
transitive,76.7%,86.0%,75.8%,64.7%,70.8%,98.0%,,YES,YES,YES,
wh_island,70.0%,78.9%,19.5%,46.7%,56.2%,73.3%,,YES,YES,YES,
wh_questions_object_gap,70.0%,84.2%,60.7%,79.4%,52.7%,85.0%,,YES,,YES,
wh_questions_subject_gap,93.3%,95.6%,82.5%,91.5%,82.0%,98.0%,,YES,YES,YES,
wh_questions_subject_gap_long_distance,90.0%,87.1%,86.2%,95.6%,85.6%,85.0%,YES,YES,,YES,YES
wh_vs_that_no_gap,96.7%,96.8%,86.4%,96.9%,82.5%,97.0%,,YES,,YES,
wh_vs_that_no_gap_long_distance,96.7%,93.8%,91.2%,97.3%,81.0%,92.0%,YES,YES,,YES,YES
wh_vs_that_with_gap,36.7%,55.1%,42.0%,42.8%,17.6%,77.0%,,,,YES,
wh_vs_that_with_gap_long_distance,3.3%,56.3%,17.2%,13.7%,20.0%,74.0%,,,,,
OVERALL AVERAGE,68.3%,83.0%,69.6%,69.8%,61.2%,88.4%
Gen-3.6 wins (count/67),,13/67,28/67,28/67,43/67,11/67
|