Upload 8 files

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+training_curves.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,15 @@
+---
+language: en
+tags:
+  - ielts
+  - automated-essay-scoring
+  - deberta-v3
+  - regression
+  - nlp
+library_name: transformers
+license: mit
+metrics:
+  - qwk
+  - rmse
+pipeline_tag: text-classification
+---

aes_meta.json

@@ -0,0 +1,30 @@
+{
+  "model_type": "PromptAwareIELTSScorer",
+  "backbone": "microsoft/deberta-v3-base",
+  "trait_cols": [
+    "TA",
+    "CC",
+    "LR",
+    "GRA"
+  ],
+  "score_min": 0.0,
+  "score_max": 9.0,
+  "max_len": 512,
+  "head_hidden": 256,
+  "head_dropout": 0.3,
+  "msd_k": 5,
+  "pool_dropout": 0.1,
+  "prompt_col": "prompt",
+  "essay_col": "essay",
+  "best_val_qwk": 0.6583,
+  "best_epoch": 4,
+  "final_metrics": {
+    "TA": 0.6032,
+    "CC": 0.5727,
+    "LR": 0.7023,
+    "GRA": 0.7462,
+    "OverallBand": 0.6671,
+    "avg_qwk": 0.6583,
+    "val_loss": 0.02651
+  }
+}

config.json

@@ -0,0 +1,39 @@
+{
+  "architectures": [
+    "PromptAwareIELTSScorer"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "bos_token_id": null,
+  "dtype": "float32",
+  "eos_token_id": null,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 3072,
+  "layer_norm_eps": 1e-07,
+  "legacy": true,
+  "max_position_embeddings": 512,
+  "max_relative_positions": -1,
+  "model_type": "deberta-v2",
+  "norm_rel_ebd": "layer_norm",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 12,
+  "pad_token_id": 0,
+  "pooler_dropout": 0,
+  "pooler_hidden_act": "gelu",
+  "pooler_hidden_size": 768,
+  "pos_att_type": [
+    "p2c",
+    "c2p"
+  ],
+  "position_biased_input": false,
+  "position_buckets": 256,
+  "relative_attention": true,
+  "sep_token_id": 2,
+  "share_att_key": true,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.0.0",
+  "type_vocab_size": 0,
+  "vocab_size": 128100
+}

modeling_ielts.py

@@ -0,0 +1,166 @@
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+import numpy as np
+from transformers import DebertaV2Model, DebertaV2PreTrainedModel, DebertaV2Config
+
+TRAIT_COLS      = ["TA", "CC", "LR", "GRA"]
+SCORE_MIN       = 0.0
+SCORE_MAX       = 9.0
+HEAD_HIDDEN     = 256
+HEAD_DROPOUT    = 0.3
+MSD_K           = 5
+POOL_DROPOUT    = 0.1
+
+BAND_GROUP_RULES = [
+    (-float("inf"), 4.0,  3.5),
+    (4.0,           5.5,  4.5),
+    (5.5,           6.0,  5.5),
+    (6.0,           6.5,  6.0),
+    (6.5,           7.0,  6.5),
+    (7.0,           7.5,  7.0),
+    (7.5,           8.0,  7.5),
+    (8.0,           float("inf"), 8.5),
+]
+
+def map_score_to_continuous(raw_score: float) -> float:
+    for lo, hi, cont in BAND_GROUP_RULES:
+        if lo <= raw_score < hi:
+            return cont
+    return BAND_GROUP_RULES[-1][2]
+
+class AttentionPooling(nn.Module):
+    def __init__(self, hidden_size: int, dropout: float = POOL_DROPOUT):
+        super().__init__()
+        self.attention = nn.Linear(hidden_size, 1)
+        self.dropout   = nn.Dropout(dropout)
+
+    def forward(self, hidden_states, pool_mask):
+        hidden_states = self.dropout(hidden_states)
+        scores  = self.attention(hidden_states).squeeze(-1)
+        scores  = scores.masked_fill(pool_mask == 0, -1e4)
+        weights = torch.softmax(scores, dim=-1)
+        pooled  = torch.bmm(weights.unsqueeze(1), hidden_states)
+        return pooled.squeeze(1)
+
+class TraitHead(nn.Module):
+    def __init__(self, hidden_size, mid_size=HEAD_HIDDEN, dropout=HEAD_DROPOUT):
+        super().__init__()
+        self.net = nn.Sequential(
+            nn.Linear(hidden_size, mid_size),
+            nn.GELU(),
+            nn.Dropout(dropout),
+            nn.Linear(mid_size, 1),
+            nn.Sigmoid(),
+        )
+
+    def forward(self, x):
+        return self.net(x).squeeze(-1)
+
+class PromptAwareIELTSScorer(DebertaV2PreTrainedModel):
+    config_class = DebertaV2Config
+
+    def __init__(self, config):
+        super().__init__(config)
+        self.deberta = DebertaV2Model(config)
+        H = config.hidden_size
+
+        self.attention_pooling = AttentionPooling(H)
+        self.shared_proj = nn.Sequential(
+            nn.Linear(H, H),
+            nn.GELU(),
+        )
+        self.msd_dropouts = nn.ModuleList([nn.Dropout(HEAD_DROPOUT) for _ in range(MSD_K)])
+
+        self.head_ta  = TraitHead(H)
+        self.head_cc  = TraitHead(H)
+        self.head_lr  = TraitHead(H)
+        self.head_gra = TraitHead(H)
+        self.post_init()
+
+    def get_backbone_params(self):
+        return list(self.deberta.parameters())
+
+    def get_head_params(self):
+        return (list(self.attention_pooling.parameters()) +
+                list(self.shared_proj.parameters()) +
+                list(self.head_ta.parameters()) +
+                list(self.head_cc.parameters()) +
+                list(self.head_lr.parameters()) +
+                list(self.head_gra.parameters()))
+
+    def _essay_only_mask(self, input_ids, attention_mask):
+        sep_token_id = getattr(self.config, 'sep_token_id', None)
+        if sep_token_id is None:
+            sep_token_id = 2
+
+        essay_mask = attention_mask.clone()
+        is_sep = (input_ids == sep_token_id)
+        has_sep = is_sep.any(dim=1)
+
+        sep_cumsum = is_sep.cumsum(dim=1)
+        before_first_sep = (sep_cumsum == 0)
+        at_first_sep = is_sep & (sep_cumsum == 1)
+        prompt_region = (before_first_sep | at_first_sep) & has_sep.unsqueeze(1)
+
+        essay_mask = essay_mask.masked_fill(prompt_region, 0)
+
+        all_zero = (essay_mask.sum(dim=1) == 0)
+        if all_zero.any():
+            essay_mask = essay_mask.clone()
+            essay_mask[all_zero] = attention_mask[all_zero]
+
+        return essay_mask
+
+    def encode(self, input_ids, attention_mask):
+        out = self.deberta(input_ids=input_ids, attention_mask=attention_mask,
+                           output_hidden_states=False)
+        last_hidden = out.last_hidden_state
+        essay_mask = self._essay_only_mask(input_ids, attention_mask)
+        context = self.attention_pooling(last_hidden, essay_mask)
+        shared = self.shared_proj(context)
+        return shared
+
+    def forward(self, input_ids=None, attention_mask=None, labels=None, **kwargs):
+        shared = self.encode(input_ids, attention_mask)
+
+        logits_sum = None
+        for dropout in self.msd_dropouts:
+            h = dropout(shared)
+            logits_k = torch.stack([
+                self.head_ta(h),
+                self.head_cc(h),
+                self.head_lr(h),
+                self.head_gra(h),
+            ], dim=1)
+            logits_sum = logits_k if logits_sum is None else logits_sum + logits_k
+
+        logits = logits_sum / len(self.msd_dropouts)
+
+        loss = None
+        if labels is not None:
+            loss = F.smooth_l1_loss(logits, labels, beta=0.3)
+        return {"loss": loss, "logits": logits}
+
+def predict_essay(model, tokenizer, prompt: str, essay: str,
+                  max_length: int = 512, device: str = "cpu") -> dict:
+    model.eval()
+    enc = tokenizer(prompt, essay, return_tensors="pt",
+                    truncation=True, max_length=max_length, padding=False)
+    enc = {k: v.to(device) for k, v in enc.items() if k in ['input_ids', 'attention_mask']}
+    with torch.no_grad():
+        out = model(**enc)
+
+    preds_9 = out["logits"][0].cpu().numpy() * (SCORE_MAX - SCORE_MIN) + SCORE_MIN
+    ta, cc, lr, gra = preds_9
+
+    overall = round(float((ta + cc + lr + gra) / 4.0) * 2) / 2.0
+
+    return {
+        "TA":          round(float(ta),  2),
+        "CC":          round(float(cc),  2),
+        "LR":          round(float(lr),  2),
+        "GRA":         round(float(gra), 2),
+        "OverallBand": overall,
+    }

tokenizer_config.json

@@ -0,0 +1,23 @@
+{
+  "add_prefix_space": true,
+  "backend": "tokenizers",
+  "bos_token": "[CLS]",
+  "cls_token": "[CLS]",
+  "do_lower_case": false,
+  "eos_token": "[SEP]",
+  "extra_special_tokens": [
+    "[PAD]",
+    "[CLS]",
+    "[SEP]"
+  ],
+  "is_local": false,
+  "mask_token": "[MASK]",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "split_by_punct": false,
+  "tokenizer_class": "DebertaV2Tokenizer",
+  "unk_id": 3,
+  "unk_token": "[UNK]",
+  "vocab_type": "spm"
+}

training_history.csv

@@ -0,0 +1,7 @@
+epoch,train_loss,TA,CC,LR,GRA,OverallBand,avg_qwk,val_loss
+1,0.042457533743399514,0.3564,0.338,0.5395,0.6175,0.4654,0.4634,0.03586
+2,0.026793291574542383,0.5092,0.4707,0.6512,0.7061,0.6022,0.5879,0.0284
+3,0.02471122434183812,0.5824,0.5452,0.7001,0.7378,0.6509,0.6433,0.02702
+4,0.022730398019798117,0.6032,0.5727,0.7023,0.7462,0.6671,0.6583,0.02651
+5,0.02116386474276467,0.525,0.4968,0.6399,0.6876,0.5933,0.5885,0.03349
+6,0.019647491585986734,0.5396,0.5142,0.6452,0.6987,0.6057,0.6007,0.03209