prithivMLmods commited on
Commit
efe9200
·
verified ·
1 Parent(s): dc24e06

incl. — a GPU Duration slider that dynamically allocates ZeroGPU time. [@v2]

Browse files
Files changed (1) hide show
  1. app.py +180 -111
app.py CHANGED
@@ -1,16 +1,19 @@
1
  import os
2
  import gc
3
- import base64
4
  import json
5
  import ast
6
  import re
 
7
  from io import BytesIO
8
- from typing import Iterable
9
  from threading import Thread
 
10
 
11
- import numpy as np
12
- import torch
 
13
  import spaces
 
 
14
  from PIL import Image, ImageDraw, ImageFont
15
  import supervision as sv
16
  from transformers import (
@@ -19,19 +22,16 @@ from transformers import (
19
  TextIteratorStreamer,
20
  )
21
 
22
- import gradio as gr
23
- from gradio import Server
24
- from fastapi.responses import HTMLResponse
25
-
26
- # --- Config ---
27
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
28
  DTYPE = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float16
29
- MODEL_NAME = "Qwen/Qwen3.8-27B"
30
 
31
  GPU_DURATIONS = [60, 90, 120, 150, 180, 250, 300]
32
  DEFAULT_GPU_DURATION_IDX = 1
33
 
34
- # --- Colors for Annotations ---
35
  BRIGHT_YELLOW = sv.Color(r=255, g=230, b=0)
36
  DARK_OUTLINE = sv.Color(r=40, g=40, b=40)
37
  BLACK = sv.Color(r=0, g=0, b=0)
@@ -40,11 +40,19 @@ WHITE = sv.Color(r=255, g=255, b=255)
40
  SPATIAL_LINE = (255, 69, 0) # OrangeRed
41
  SPATIAL_DOT = (255, 69, 0)
42
  SPATIAL_RING = (255, 255, 255)
43
- SPATIAL_LABEL_BG = (80, 25, 0)
44
  SPATIAL_LABEL_TXT = (255, 255, 255)
45
- SPATIAL_ARROW = (230, 149, 0)
46
 
47
- # --- Model Loading ---
 
 
 
 
 
 
 
 
48
  print(f"Loading model: {MODEL_NAME} ...")
49
  qwen_model = Qwen3_5ForConditionalGeneration.from_pretrained(
50
  MODEL_NAME, torch_dtype=DTYPE, device_map=DEVICE, attn_implementation="kernels-community/flash-attn2@v3",
@@ -52,33 +60,81 @@ qwen_model = Qwen3_5ForConditionalGeneration.from_pretrained(
52
  qwen_processor = AutoProcessor.from_pretrained(MODEL_NAME)
53
  print("Model loaded.")
54
 
55
- # --- JSON Parsing & Extraction ---
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
56
  def safe_parse_json(text: str):
57
  text = re.sub(r"```(json)?", "", text).strip()
58
  match = re.search(r'(\[.*\]|\{.*\})', text, re.DOTALL)
59
  if match:
60
  json_str = match.group(1)
61
  json_str_clean = re.sub(r',\s*([}\]])', r'\1', json_str)
62
- try: return json.loads(json_str_clean)
 
63
  except json.JSONDecodeError:
64
- try: return ast.literal_eval(json_str_clean)
65
- except Exception: pass
 
 
66
  text_clean = re.sub(r',\s*([}\]])', r'\1', text)
67
- try: return json.loads(text_clean)
68
- except json.JSONDecodeError: pass
69
- try: return ast.literal_eval(text_clean)
70
- except Exception: pass
 
 
 
 
71
  return []
72
 
73
  def _extract_point(item: dict):
74
- if not isinstance(item, dict): return None
 
75
  for k in ["point_2d", "point", "points", "coordinate", "coordinates", "xy"]:
76
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 2:
77
  return item[k]
78
  return None
79
 
80
  def _extract_bbox(item: dict):
81
- if not isinstance(item, dict): return None
 
82
  for k in ["bbox_2d", "bbox", "box", "bounding_box", "xyxy"]:
83
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 4:
84
  return item[k]
@@ -86,14 +142,22 @@ def _extract_bbox(item: dict):
86
 
87
  def _load_font(size: int = 16):
88
  size = max(6, int(size))
89
- try: return ImageFont.truetype("arial.ttf", size)
 
90
  except (IOError, OSError):
91
- try: return ImageFont.truetype("DejaVuSans.ttf", size)
92
- except (IOError, OSError): return ImageFont.load_default()
 
 
 
 
 
 
 
93
 
94
- # --- Annotation Functions ---
95
  def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_thickness: int = 2, text_scale: float = 0.5):
96
- if not isinstance(image, Image.Image) or not isinstance(result, dict): return image
 
97
  image = image.convert("RGB")
98
  ow, oh = image.size
99
 
@@ -104,7 +168,8 @@ def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_
104
 
105
  if "points" in result and result["points"]:
106
  pts = [[int(p["x"] * ow), int(p["y"] * oh)] for p in result["points"]]
107
- if not pts: return image
 
108
  kp = sv.KeyPoints(xy=np.array(pts).reshape(1, -1, 2))
109
  scene = np.array(image.copy())
110
  scene = sv.VertexAnnotator(radius=point_radius + 3, color=DARK_OUTLINE).annotate(scene=scene, key_points=kp)
@@ -134,7 +199,8 @@ def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_
134
  obj.get("x_max", 0.0) * ow, obj.get("y_max", 0.0) * oh,
135
  ])
136
  labels.append(obj.get("label", "object"))
137
- if not boxes: return image
 
138
  scene = np.array(image.copy())
139
  h, w = scene.shape[:2]
140
  masks = np.zeros((len(boxes), h, w), dtype=bool)
@@ -143,7 +209,8 @@ def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_
143
  x2, y2 = min(w, int(box[2])), min(h, int(box[3]))
144
  masks[i, y1:y2, x1:x2] = True
145
  dets = sv.Detections(xyxy=np.array(boxes), mask=masks)
146
- if len(dets) == 0: return image
 
147
  scene = sv.MaskAnnotator(color=BRIGHT_YELLOW, opacity=0.18, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
148
  scene = sv.BoxAnnotator(color=BRIGHT_YELLOW, thickness=box_thickness, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
149
  scene = sv.LabelAnnotator(
@@ -154,10 +221,12 @@ def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_
154
  return image
155
 
156
  def annotate_spatial_path(image: Image.Image, result: dict, dot_radius: int = 6, line_width: int = 4, text_scale: float = 0.5):
157
- if not isinstance(image, Image.Image) or not isinstance(result, dict): return image
 
158
  image = image.convert("RGB")
159
  w, h = image.size
160
- if "points" not in result or not result["points"]: return image
 
161
 
162
  dot_radius = max(1, int(dot_radius))
163
  line_width = max(1, int(line_width))
@@ -184,7 +253,8 @@ def annotate_spatial_path(image: Image.Image, result: dict, dot_radius: int = 6,
184
  x2, y2 = pts[i+1]
185
  dx, dy = x2 - x1, y2 - y1
186
  length = (dx * dx + dy * dy) ** 0.5
187
- if length < 12: continue
 
188
  ux, uy = dx / length, dy / length
189
  offset = 18 * scale_ratio
190
  bx, by = x2 - ux * offset, y2 - uy * offset
@@ -235,60 +305,15 @@ def annotate_spatial_path(image: Image.Image, result: dict, dot_radius: int = 6,
235
 
236
  return image
237
 
238
- def pil_to_b64_png(image: Image.Image) -> str:
239
- buf = BytesIO()
240
- image.save(buf, format="PNG")
241
- return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode()}"
242
-
243
- def get_gpu_duration(image_b64, prompt, task, point_radius, box_thickness, text_scale, gpu_duration_seconds):
244
- """Dynamically returns GPU duration based on the UI slider value."""
245
  try:
246
  return int(gpu_duration_seconds)
247
  except (TypeError, ValueError):
248
  return GPU_DURATIONS[DEFAULT_GPU_DURATION_IDX]
249
 
250
- # --- Config Examples ---
251
- EXAMPLES_CONFIG = [
252
- {"image": "examples/1.jpg", "prompt": "Detect the yellow car that is parked.", "task": "Detect"},
253
- {"image": "examples/2.jpg", "prompt": "Point to all the red cars.", "task": "Point"},
254
- {"image": "examples/3.jpg", "prompt": "Map a path from the door to the lamp.", "task": "Spatial"},
255
- ]
256
-
257
- def make_thumb_b64(path, max_dim=220):
258
- if not os.path.exists(path): return ""
259
- try:
260
- img = Image.open(path).convert("RGB")
261
- img.thumbnail((max_dim, max_dim), Image.LANCZOS)
262
- buf = BytesIO()
263
- img.save(buf, format="JPEG", quality=65)
264
- return f"data:image/jpeg;base64,{base64.b64encode(buf.getvalue()).decode()}"
265
- except Exception as e:
266
- return ""
267
-
268
- def encode_full_image(path):
269
- if not os.path.exists(path): return ""
270
- try:
271
- with open(path, "rb") as f: data = f.read()
272
- ext = path.rsplit(".", 1)[-1].lower()
273
- mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", "webp": "image/webp"}.get(ext, "image/jpeg")
274
- return f"data:{mime};base64,{base64.b64encode(data).decode()}"
275
- except Exception as e:
276
- return ""
277
-
278
- def build_client_config():
279
- examples = []
280
- for i, ex in enumerate(EXAMPLES_CONFIG):
281
- examples.append({
282
- "idx": i,
283
- "thumb": make_thumb_b64(ex["image"]),
284
- "prompt": ex["prompt"],
285
- "task": ex["task"],
286
- })
287
- return {"examples": examples}
288
-
289
- CLIENT_CONFIG = build_client_config()
290
-
291
- # --- Gradio Server ---
292
  app = Server(title="Qwen3.8-27B-Object-Detection")
293
 
294
  @app.mcp.tool(name="run_inference")
@@ -297,18 +322,20 @@ app = Server(title="Qwen3.8-27B-Object-Detection")
297
  def infer(
298
  image_b64: str,
299
  prompt: str,
300
- task: str,
301
  point_radius: int,
302
  box_thickness: int,
303
  text_scale: float,
304
  gpu_duration_seconds: int,
305
  ) -> dict:
306
- """Runs Qwen3.8 vision model for detection, point localization, or spatial mapping."""
307
  gc.collect()
308
  torch.cuda.empty_cache()
309
 
310
- if not image_b64: raise gr.Error("Please upload an image.")
311
- if not prompt or not prompt.strip(): raise gr.Error("Please provide a prompt.")
 
 
312
 
313
  try:
314
  header, data = image_b64.split(",", 1)
@@ -318,19 +345,19 @@ def infer(
318
 
319
  pil_image.thumbnail((512, 512))
320
 
321
- if task == "Detect":
322
  full_prompt = (
323
  f"Provide bounding box coordinates for {prompt}. "
324
  f"Report strictly in JSON format as a list of objects with 'label' and "
325
  f"'bbox_2d' (xmin, ymin, xmax, ymax in 0-1000 scale)."
326
  )
327
- elif task == "Point":
328
  full_prompt = (
329
  f"Provide 2d point coordinates for {prompt}. "
330
  f"Report strictly in JSON format as a list of objects with 'label' and "
331
  f"'point_2d' (x, y in 0-1000 scale)."
332
  )
333
- elif task == "Spatial":
334
  full_prompt = (
335
  f"Identify the key spatial waypoints to map a path/route for: {prompt}. "
336
  f"Return the points in the order they should be connected along the path, "
@@ -348,7 +375,6 @@ def infer(
348
  {"type": "text", "text": full_prompt},
349
  ],
350
  }]
351
-
352
  text = qwen_processor.apply_chat_template(
353
  messages, tokenize=False, add_generation_prompt=True
354
  )
@@ -377,16 +403,22 @@ def infer(
377
  full_text += tok
378
  thread.join()
379
 
380
- # --- Post-process ---
381
- if task == "Point":
382
  parsed = safe_parse_json(full_text)
383
  if isinstance(parsed, dict):
 
384
  for k in ["points", "keypoints", "point"]:
385
  if k in parsed and isinstance(parsed[k], list):
386
  parsed = parsed[k]
 
387
  break
388
- else:
389
- parsed = next((v for v in parsed.values() if isinstance(v, list)), [])
 
 
 
 
 
390
 
391
  result = {"points": []}
392
  if isinstance(parsed, list):
@@ -394,23 +426,36 @@ def infer(
394
  pt = _extract_point(item)
395
  if pt:
396
  x, y = pt
397
- result["points"].append({"label": item.get("label", ""), "x": x / 1000.0, "y": y / 1000.0})
 
 
 
398
 
399
  if result["points"]:
400
- annotated_img = annotate_image(pil_image.copy(), result, point_radius=point_radius, box_thickness=box_thickness, text_scale=text_scale)
 
 
 
401
  return {"image": pil_to_b64_png(annotated_img), "text": json.dumps(result, indent=2)}
402
  else:
403
  return {"image": pil_to_b64_png(pil_image), "text": f"Could not extract any points.\nRaw model output:\n{full_text}"}
404
 
405
- elif task == "Detect":
406
  parsed = safe_parse_json(full_text)
407
  if isinstance(parsed, dict):
 
408
  for k in ["objects", "detections", "bboxes", "boxes", "results"]:
409
  if k in parsed and isinstance(parsed[k], list):
410
  parsed = parsed[k]
 
411
  break
412
- else:
413
- parsed = next((v for v in parsed.values() if isinstance(v, list)), [])
 
 
 
 
 
414
 
415
  result = {"objects": []}
416
  if isinstance(parsed, list):
@@ -425,20 +470,30 @@ def infer(
425
  })
426
 
427
  if result["objects"]:
428
- annotated_img = annotate_image(pil_image.copy(), result, point_radius=point_radius, box_thickness=box_thickness, text_scale=text_scale)
 
 
 
429
  return {"image": pil_to_b64_png(annotated_img), "text": json.dumps(result, indent=2)}
430
  else:
431
  return {"image": pil_to_b64_png(pil_image), "text": f"Could not extract any objects.\nRaw model output:\n{full_text}"}
432
 
433
- elif task == "Spatial":
434
  parsed = safe_parse_json(full_text)
435
  if isinstance(parsed, dict):
 
436
  for k in ["points", "waypoints", "path", "route", "nodes", "map"]:
437
  if k in parsed and isinstance(parsed[k], list):
438
  parsed = parsed[k]
 
439
  break
440
- else:
441
- parsed = next((v for v in parsed.values() if isinstance(v, list)), [])
 
 
 
 
 
442
 
443
  result = {"points": []}
444
  if isinstance(parsed, list):
@@ -446,15 +501,30 @@ def infer(
446
  pt = _extract_point(item)
447
  if pt:
448
  x, y = pt
449
- result["points"].append({"label": item.get("label", "waypoint"), "x": x / 1000.0, "y": y / 1000.0})
 
 
 
450
 
451
  if result["points"]:
452
- annotated_img = annotate_spatial_path(pil_image.copy(), result, dot_radius=point_radius, line_width=box_thickness * 2, text_scale=text_scale)
453
- return {"image": pil_to_b64_png(annotated_img), "text": json.dumps(result, indent=2)}
 
 
 
 
 
 
 
 
 
 
 
 
454
  else:
455
  return {"image": pil_to_b64_png(pil_image), "text": f"Could not extract any spatial waypoints.\nRaw model output:\n{full_text}"}
456
 
457
- return {"image": pil_to_b64_png(pil_image), "text": full_text}
458
 
459
 
460
  @app.api(name="load_example", queue=False)
@@ -464,14 +534,13 @@ def load_example(idx: float) -> dict:
464
  except (ValueError, TypeError):
465
  i = -1
466
  if i < 0 or i >= len(EXAMPLES_CONFIG):
467
- return {"image": "", "prompt": "", "task": "", "name": "", "status": "error"}
468
  ex = EXAMPLES_CONFIG[i]
469
  b64 = encode_full_image(ex["image"])
470
  return {
471
  "image": b64,
472
  "prompt": ex["prompt"],
473
  "task": ex["task"],
474
- "name": os.path.basename(ex["image"]),
475
  "status": "ok" if b64 else "error"
476
  }
477
 
 
1
  import os
2
  import gc
 
3
  import json
4
  import ast
5
  import re
6
+ import base64
7
  from io import BytesIO
 
8
  from threading import Thread
9
+ from typing import Tuple
10
 
11
+ import gradio as gr
12
+ from gradio import Server
13
+ from fastapi.responses import HTMLResponse
14
  import spaces
15
+ import torch
16
+ import numpy as np
17
  from PIL import Image, ImageDraw, ImageFont
18
  import supervision as sv
19
  from transformers import (
 
22
  TextIteratorStreamer,
23
  )
24
 
25
+ # ------------------------------------------------------------------
26
+ # Config & Constants
27
+ # ------------------------------------------------------------------
28
+ MODEL_NAME = "Qwen/Qwen3.8-27B"
 
29
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
30
  DTYPE = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float16
 
31
 
32
  GPU_DURATIONS = [60, 90, 120, 150, 180, 250, 300]
33
  DEFAULT_GPU_DURATION_IDX = 1
34
 
 
35
  BRIGHT_YELLOW = sv.Color(r=255, g=230, b=0)
36
  DARK_OUTLINE = sv.Color(r=40, g=40, b=40)
37
  BLACK = sv.Color(r=0, g=0, b=0)
 
40
  SPATIAL_LINE = (255, 69, 0) # OrangeRed
41
  SPATIAL_DOT = (255, 69, 0)
42
  SPATIAL_RING = (255, 255, 255)
43
+ SPATIAL_LABEL_BG = (50, 10, 0)
44
  SPATIAL_LABEL_TXT = (255, 255, 255)
45
+ SPATIAL_ARROW = (255, 140, 0)
46
 
47
+ EXAMPLES_CONFIG = [
48
+ {"image": "examples/1.jpg", "prompt": "Detect the yellow car that is parked.", "task": "Detect"},
49
+ {"image": "examples/2.jpg", "prompt": "Point to all the red cars.", "task": "Point"},
50
+ {"image": "examples/3.jpg", "prompt": "Map a path from the door to the lamp.", "task": "Spatial"},
51
+ ]
52
+
53
+ # ------------------------------------------------------------------
54
+ # Model Loading
55
+ # ------------------------------------------------------------------
56
  print(f"Loading model: {MODEL_NAME} ...")
57
  qwen_model = Qwen3_5ForConditionalGeneration.from_pretrained(
58
  MODEL_NAME, torch_dtype=DTYPE, device_map=DEVICE, attn_implementation="kernels-community/flash-attn2@v3",
 
60
  qwen_processor = AutoProcessor.from_pretrained(MODEL_NAME)
61
  print("Model loaded.")
62
 
63
+ # ------------------------------------------------------------------
64
+ # Helper Functions
65
+ # ------------------------------------------------------------------
66
+ def make_thumb_b64(path, max_dim=220):
67
+ if not os.path.exists(path):
68
+ return ""
69
+ try:
70
+ img = Image.open(path).convert("RGB")
71
+ img.thumbnail((max_dim, max_dim), Image.LANCZOS)
72
+ buf = BytesIO()
73
+ img.save(buf, format="JPEG", quality=65)
74
+ return f"data:image/jpeg;base64,{base64.b64encode(buf.getvalue()).decode()}"
75
+ except Exception:
76
+ return ""
77
+
78
+ def encode_full_image(path):
79
+ if not os.path.exists(path):
80
+ return ""
81
+ try:
82
+ with open(path, "rb") as f:
83
+ data = f.read()
84
+ ext = path.rsplit(".", 1)[-1].lower()
85
+ mime = {"jpg": "image/jpeg", "jpeg": "image/jpeg", "png": "image/png", "webp": "image/webp"}.get(ext, "image/jpeg")
86
+ return f"data:{mime};base64,{base64.b64encode(data).decode()}"
87
+ except Exception:
88
+ return ""
89
+
90
+ def build_client_config():
91
+ examples = []
92
+ for i, ex in enumerate(EXAMPLES_CONFIG):
93
+ examples.append({
94
+ "idx": i,
95
+ "thumb": make_thumb_b64(ex["image"]),
96
+ "prompt": ex["prompt"],
97
+ "task": ex["task"],
98
+ })
99
+ return {"examples": examples, "gpu_durations": GPU_DURATIONS, "default_gpu_idx": DEFAULT_GPU_DURATION_IDX}
100
+
101
+ CLIENT_CONFIG = build_client_config()
102
+
103
  def safe_parse_json(text: str):
104
  text = re.sub(r"```(json)?", "", text).strip()
105
  match = re.search(r'(\[.*\]|\{.*\})', text, re.DOTALL)
106
  if match:
107
  json_str = match.group(1)
108
  json_str_clean = re.sub(r',\s*([}\]])', r'\1', json_str)
109
+ try:
110
+ return json.loads(json_str_clean)
111
  except json.JSONDecodeError:
112
+ try:
113
+ return ast.literal_eval(json_str_clean)
114
+ except Exception:
115
+ pass
116
  text_clean = re.sub(r',\s*([}\]])', r'\1', text)
117
+ try:
118
+ return json.loads(text_clean)
119
+ except json.JSONDecodeError:
120
+ pass
121
+ try:
122
+ return ast.literal_eval(text_clean)
123
+ except Exception:
124
+ pass
125
  return []
126
 
127
  def _extract_point(item: dict):
128
+ if not isinstance(item, dict):
129
+ return None
130
  for k in ["point_2d", "point", "points", "coordinate", "coordinates", "xy"]:
131
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 2:
132
  return item[k]
133
  return None
134
 
135
  def _extract_bbox(item: dict):
136
+ if not isinstance(item, dict):
137
+ return None
138
  for k in ["bbox_2d", "bbox", "box", "bounding_box", "xyxy"]:
139
  if k in item and isinstance(item[k], (list, tuple)) and len(item[k]) == 4:
140
  return item[k]
 
142
 
143
  def _load_font(size: int = 16):
144
  size = max(6, int(size))
145
+ try:
146
+ return ImageFont.truetype("arial.ttf", size)
147
  except (IOError, OSError):
148
+ try:
149
+ return ImageFont.truetype("DejaVuSans.ttf", size)
150
+ except (IOError, OSError):
151
+ return ImageFont.load_default()
152
+
153
+ def pil_to_b64_png(image: Image.Image) -> str:
154
+ buf = BytesIO()
155
+ image.save(buf, format="PNG")
156
+ return f"data:image/png;base64,{base64.b64encode(buf.getvalue()).decode()}"
157
 
 
158
  def annotate_image(image: Image.Image, result: dict, point_radius: int = 6, box_thickness: int = 2, text_scale: float = 0.5):
159
+ if not isinstance(image, Image.Image) or not isinstance(result, dict):
160
+ return image
161
  image = image.convert("RGB")
162
  ow, oh = image.size
163
 
 
168
 
169
  if "points" in result and result["points"]:
170
  pts = [[int(p["x"] * ow), int(p["y"] * oh)] for p in result["points"]]
171
+ if not pts:
172
+ return image
173
  kp = sv.KeyPoints(xy=np.array(pts).reshape(1, -1, 2))
174
  scene = np.array(image.copy())
175
  scene = sv.VertexAnnotator(radius=point_radius + 3, color=DARK_OUTLINE).annotate(scene=scene, key_points=kp)
 
199
  obj.get("x_max", 0.0) * ow, obj.get("y_max", 0.0) * oh,
200
  ])
201
  labels.append(obj.get("label", "object"))
202
+ if not boxes:
203
+ return image
204
  scene = np.array(image.copy())
205
  h, w = scene.shape[:2]
206
  masks = np.zeros((len(boxes), h, w), dtype=bool)
 
209
  x2, y2 = min(w, int(box[2])), min(h, int(box[3]))
210
  masks[i, y1:y2, x1:x2] = True
211
  dets = sv.Detections(xyxy=np.array(boxes), mask=masks)
212
+ if len(dets) == 0:
213
+ return image
214
  scene = sv.MaskAnnotator(color=BRIGHT_YELLOW, opacity=0.18, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
215
  scene = sv.BoxAnnotator(color=BRIGHT_YELLOW, thickness=box_thickness, color_lookup=sv.ColorLookup.INDEX).annotate(scene=scene, detections=dets)
216
  scene = sv.LabelAnnotator(
 
221
  return image
222
 
223
  def annotate_spatial_path(image: Image.Image, result: dict, dot_radius: int = 6, line_width: int = 4, text_scale: float = 0.5):
224
+ if not isinstance(image, Image.Image) or not isinstance(result, dict):
225
+ return image
226
  image = image.convert("RGB")
227
  w, h = image.size
228
+ if "points" not in result or not result["points"]:
229
+ return image
230
 
231
  dot_radius = max(1, int(dot_radius))
232
  line_width = max(1, int(line_width))
 
253
  x2, y2 = pts[i+1]
254
  dx, dy = x2 - x1, y2 - y1
255
  length = (dx * dx + dy * dy) ** 0.5
256
+ if length < 12:
257
+ continue
258
  ux, uy = dx / length, dy / length
259
  offset = 18 * scale_ratio
260
  bx, by = x2 - ux * offset, y2 - uy * offset
 
305
 
306
  return image
307
 
308
+ def get_gpu_duration(image_b64, prompt, task_type, point_radius, box_thickness, text_scale, gpu_duration_seconds):
 
 
 
 
 
 
309
  try:
310
  return int(gpu_duration_seconds)
311
  except (TypeError, ValueError):
312
  return GPU_DURATIONS[DEFAULT_GPU_DURATION_IDX]
313
 
314
+ # ------------------------------------------------------------------
315
+ # Gradio Server (Server mode)
316
+ # ------------------------------------------------------------------
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
317
  app = Server(title="Qwen3.8-27B-Object-Detection")
318
 
319
  @app.mcp.tool(name="run_inference")
 
322
  def infer(
323
  image_b64: str,
324
  prompt: str,
325
+ task_type: str,
326
  point_radius: int,
327
  box_thickness: int,
328
  text_scale: float,
329
  gpu_duration_seconds: int,
330
  ) -> dict:
331
+ """Runs object detection, point localization, or spatial mapping."""
332
  gc.collect()
333
  torch.cuda.empty_cache()
334
 
335
+ if not image_b64:
336
+ raise gr.Error("Please upload an image.")
337
+ if not prompt or prompt.strip() == "":
338
+ raise gr.Error("Please provide a prompt.")
339
 
340
  try:
341
  header, data = image_b64.split(",", 1)
 
345
 
346
  pil_image.thumbnail((512, 512))
347
 
348
+ if task_type == "Detect":
349
  full_prompt = (
350
  f"Provide bounding box coordinates for {prompt}. "
351
  f"Report strictly in JSON format as a list of objects with 'label' and "
352
  f"'bbox_2d' (xmin, ymin, xmax, ymax in 0-1000 scale)."
353
  )
354
+ elif task_type == "Point":
355
  full_prompt = (
356
  f"Provide 2d point coordinates for {prompt}. "
357
  f"Report strictly in JSON format as a list of objects with 'label' and "
358
  f"'point_2d' (x, y in 0-1000 scale)."
359
  )
360
+ elif task_type == "Spatial":
361
  full_prompt = (
362
  f"Identify the key spatial waypoints to map a path/route for: {prompt}. "
363
  f"Return the points in the order they should be connected along the path, "
 
375
  {"type": "text", "text": full_prompt},
376
  ],
377
  }]
 
378
  text = qwen_processor.apply_chat_template(
379
  messages, tokenize=False, add_generation_prompt=True
380
  )
 
403
  full_text += tok
404
  thread.join()
405
 
406
+ if task_type == "Point":
 
407
  parsed = safe_parse_json(full_text)
408
  if isinstance(parsed, dict):
409
+ list_found = False
410
  for k in ["points", "keypoints", "point"]:
411
  if k in parsed and isinstance(parsed[k], list):
412
  parsed = parsed[k]
413
+ list_found = True
414
  break
415
+ if not list_found:
416
+ for v in parsed.values():
417
+ if isinstance(v, list):
418
+ parsed = v
419
+ break
420
+ else:
421
+ parsed = []
422
 
423
  result = {"points": []}
424
  if isinstance(parsed, list):
 
426
  pt = _extract_point(item)
427
  if pt:
428
  x, y = pt
429
+ result["points"].append({
430
+ "label": item.get("label", ""),
431
+ "x": x / 1000.0, "y": y / 1000.0,
432
+ })
433
 
434
  if result["points"]:
435
+ annotated_img = annotate_image(
436
+ pil_image.copy(), result,
437
+ point_radius=point_radius, box_thickness=box_thickness, text_scale=text_scale,
438
+ )
439
  return {"image": pil_to_b64_png(annotated_img), "text": json.dumps(result, indent=2)}
440
  else:
441
  return {"image": pil_to_b64_png(pil_image), "text": f"Could not extract any points.\nRaw model output:\n{full_text}"}
442
 
443
+ elif task_type == "Detect":
444
  parsed = safe_parse_json(full_text)
445
  if isinstance(parsed, dict):
446
+ list_found = False
447
  for k in ["objects", "detections", "bboxes", "boxes", "results"]:
448
  if k in parsed and isinstance(parsed[k], list):
449
  parsed = parsed[k]
450
+ list_found = True
451
  break
452
+ if not list_found:
453
+ for v in parsed.values():
454
+ if isinstance(v, list):
455
+ parsed = v
456
+ break
457
+ else:
458
+ parsed = []
459
 
460
  result = {"objects": []}
461
  if isinstance(parsed, list):
 
470
  })
471
 
472
  if result["objects"]:
473
+ annotated_img = annotate_image(
474
+ pil_image.copy(), result,
475
+ point_radius=point_radius, box_thickness=box_thickness, text_scale=text_scale,
476
+ )
477
  return {"image": pil_to_b64_png(annotated_img), "text": json.dumps(result, indent=2)}
478
  else:
479
  return {"image": pil_to_b64_png(pil_image), "text": f"Could not extract any objects.\nRaw model output:\n{full_text}"}
480
 
481
+ elif task_type == "Spatial":
482
  parsed = safe_parse_json(full_text)
483
  if isinstance(parsed, dict):
484
+ list_found = False
485
  for k in ["points", "waypoints", "path", "route", "nodes", "map"]:
486
  if k in parsed and isinstance(parsed[k], list):
487
  parsed = parsed[k]
488
+ list_found = True
489
  break
490
+ if not list_found:
491
+ for v in parsed.values():
492
+ if isinstance(v, list):
493
+ parsed = v
494
+ break
495
+ else:
496
+ parsed = []
497
 
498
  result = {"points": []}
499
  if isinstance(parsed, list):
 
501
  pt = _extract_point(item)
502
  if pt:
503
  x, y = pt
504
+ result["points"].append({
505
+ "label": item.get("label", "waypoint"),
506
+ "x": x / 1000.0, "y": y / 1000.0,
507
+ })
508
 
509
  if result["points"]:
510
+ wp_lines = "\n".join(
511
+ f" {i+1}. {p['label']} → ({p['x']:.3f}, {p['y']:.3f})"
512
+ for i, p in enumerate(result["points"])
513
+ )
514
+ summary = (
515
+ f"Spatial map generated.\n"
516
+ f"Waypoints ({len(result['points'])}):\n{wp_lines}\n"
517
+ f"Path segments: {max(0, len(result['points']) - 1)}"
518
+ )
519
+ annotated_img = annotate_spatial_path(
520
+ pil_image.copy(), result,
521
+ dot_radius=point_radius, line_width=box_thickness * 2, text_scale=text_scale,
522
+ )
523
+ return {"image": pil_to_b64_png(annotated_img), "text": summary}
524
  else:
525
  return {"image": pil_to_b64_png(pil_image), "text": f"Could not extract any spatial waypoints.\nRaw model output:\n{full_text}"}
526
 
527
+ return {"image": pil_to_b64_png(pil_image), "text": "Unknown task type."}
528
 
529
 
530
  @app.api(name="load_example", queue=False)
 
534
  except (ValueError, TypeError):
535
  i = -1
536
  if i < 0 or i >= len(EXAMPLES_CONFIG):
537
+ return {"image": "", "prompt": "", "task": "", "status": "error"}
538
  ex = EXAMPLES_CONFIG[i]
539
  b64 = encode_full_image(ex["image"])
540
  return {
541
  "image": b64,
542
  "prompt": ex["prompt"],
543
  "task": ex["task"],
 
544
  "status": "ok" if b64 else "error"
545
  }
546