Spaces:
Paused
Paused
Update sys7_miner.py
Browse files- sys7_miner.py +2 -2
sys7_miner.py
CHANGED
|
@@ -288,8 +288,8 @@ def fuse_text(description: Optional[str], hashtags: Optional[str | Sequence[str]
|
|
| 288 |
ht_tokens = normalize_hashtags(hashtags, slang_map)
|
| 289 |
hashtag_text = " ".join(ht_tokens)
|
| 290 |
fused = " ".join(filter(None, [desc, hashtag_text])).lower()
|
| 291 |
-
if ftfy:
|
| 292 |
-
|
| 293 |
fused = CONTROL_CHARS_RE.sub(" ", fused)
|
| 294 |
fused = REPLACEMENT_RUN_RE.sub(" ", fused)
|
| 295 |
fused = standardize_quotes(fused)
|
|
|
|
| 288 |
ht_tokens = normalize_hashtags(hashtags, slang_map)
|
| 289 |
hashtag_text = " ".join(ht_tokens)
|
| 290 |
fused = " ".join(filter(None, [desc, hashtag_text])).lower()
|
| 291 |
+
# if ftfy:
|
| 292 |
+
# fused = ftfy.fix_text(fused, normalization="NFC")
|
| 293 |
fused = CONTROL_CHARS_RE.sub(" ", fused)
|
| 294 |
fused = REPLACEMENT_RUN_RE.sub(" ", fused)
|
| 295 |
fused = standardize_quotes(fused)
|