Imaginethat commited on
Commit
3a5b833
·
verified ·
1 Parent(s): 8cd4720

Update sys7_miner.py

Browse files
Files changed (1) hide show
  1. sys7_miner.py +2 -2
sys7_miner.py CHANGED
@@ -288,8 +288,8 @@ def fuse_text(description: Optional[str], hashtags: Optional[str | Sequence[str]
288
  ht_tokens = normalize_hashtags(hashtags, slang_map)
289
  hashtag_text = " ".join(ht_tokens)
290
  fused = " ".join(filter(None, [desc, hashtag_text])).lower()
291
- if ftfy:
292
- fused = ftfy.fix_text(fused, normalization="NFC")
293
  fused = CONTROL_CHARS_RE.sub(" ", fused)
294
  fused = REPLACEMENT_RUN_RE.sub(" ", fused)
295
  fused = standardize_quotes(fused)
 
288
  ht_tokens = normalize_hashtags(hashtags, slang_map)
289
  hashtag_text = " ".join(ht_tokens)
290
  fused = " ".join(filter(None, [desc, hashtag_text])).lower()
291
+ # if ftfy:
292
+ # fused = ftfy.fix_text(fused, normalization="NFC")
293
  fused = CONTROL_CHARS_RE.sub(" ", fused)
294
  fused = REPLACEMENT_RUN_RE.sub(" ", fused)
295
  fused = standardize_quotes(fused)