Add Tiny Aya ZeroGPU sidecar

README.md

@@ -0,0 +1,19 @@
+---
+title: Tiny Army Tiny Aya ZeroGPU
+emoji: 🌍
+colorFrom: green
+colorTo: blue
+sdk: gradio
+sdk_version: 6.15.2
+app_file: app.py
+python_version: "3.11"
+suggested_hardware: zero-a10g
+pinned: false
+license: apache-2.0
+models:
+  - CohereLabs/tiny-aya-global
+---
+
+# Tiny Army Tiny Aya ZeroGPU
+
+ZeroGPU sidecar for Tiny Army text generation using Cohere Labs Tiny Aya Global.

app.py

@@ -0,0 +1,74 @@
+from __future__ import annotations
+
+import os
+import threading
+
+os.environ.setdefault("OPENBLAS_NUM_THREADS", "4")
+os.environ.setdefault("OMP_NUM_THREADS", "4")
+os.environ.setdefault("MKL_NUM_THREADS", "4")
+os.environ.setdefault("TOKENIZERS_PARALLELISM", "false")
+os.environ.setdefault("GRADIO_SSR_MODE", "false")
+
+import gradio as gr
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+MODEL_ID = os.environ.get("TINY_AYA_MODEL", "CohereLabs/tiny-aya-global")
+DEFAULT_MAX_TOKENS = int(os.environ.get("TINY_AYA_MAX_TOKENS", "400"))
+
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
+model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype="auto")
+if torch.cuda.is_available():
+    model.to("cuda")
+model.eval()
+
+_lock = threading.Lock()
+
+
+def _messages(system: str, user: str):
+    messages = []
+    if system and system.strip():
+        messages.append({"role": "system", "content": system.strip()})
+    messages.append({"role": "user", "content": (user or "").strip()})
+    return messages
+
+
+@spaces.GPU(duration=120)
+def generate(system: str, user: str, max_tokens: int = DEFAULT_MAX_TOKENS, temperature: float = 0.8):
+    if not user or not user.strip():
+        raise gr.Error("user prompt required")
+    max_tokens = max(1, min(int(max_tokens or DEFAULT_MAX_TOKENS), 1024))
+    temperature = max(0.0, min(float(temperature if temperature is not None else 0.8), 2.0))
+    inputs = tokenizer.apply_chat_template(
+        _messages(system, user),
+        tokenize=True,
+        add_generation_prompt=True,
+        return_dict=True,
+        return_tensors="pt",
+    ).to(model.device)
+    with _lock, torch.inference_mode():
+        outputs = model.generate(
+            **inputs,
+            max_new_tokens=max_tokens,
+            do_sample=temperature > 0,
+            temperature=max(temperature, 1e-5),
+            top_p=0.95,
+            pad_token_id=tokenizer.eos_token_id,
+        )
+    return tokenizer.decode(outputs[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True).strip()
+
+
+with gr.Blocks(title="Tiny Army Tiny Aya ZeroGPU") as demo:
+    gr.Markdown("# Tiny Army Tiny Aya ZeroGPU")
+    system = gr.Textbox(label="System", lines=5)
+    user = gr.Textbox(label="User", lines=5)
+    max_tokens = gr.Slider(1, 1024, value=DEFAULT_MAX_TOKENS, step=1, label="Max new tokens")
+    temperature = gr.Slider(0, 2, value=0.8, step=0.05, label="Temperature")
+    btn = gr.Button("Generate")
+    out = gr.Textbox(label="Output", lines=10)
+    btn.click(generate, inputs=[system, user, max_tokens, temperature], outputs=out, api_name="generate")
+
+
+if __name__ == "__main__":
+    demo.launch()

requirements.txt

@@ -0,0 +1,8 @@
+gradio==6.15.2
+spaces
+torch
+transformers>=5.4.0
+accelerate
+sentencepiece
+protobuf
+huggingface_hub