From 2345588d383a9115e8e36098caefc7b5bae077f3 Mon Sep 17 00:00:00 2001
From: abc <98614666+xtekky@users.noreply.github.com>
Date: Sun, 19 Nov 2023 22:59:18 +0000
Subject: ~ | improve compatibility with lower python versions

remove tiktoken
---
 g4f/api/_tokenizer.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

(limited to 'g4f/api/_tokenizer.py')

diff --git a/g4f/api/_tokenizer.py b/g4f/api/_tokenizer.py
index fd8f9d5a..de5877c4 100644
--- a/g4f/api/_tokenizer.py
+++ b/g4f/api/_tokenizer.py
@@ -1,9 +1,9 @@
-import tiktoken
-from typing import Union
+# import tiktoken
+# from typing import Union
 
-def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
-    encoding   = tiktoken.encoding_for_model(model)
-    encoded    = encoding.encode(text)
-    num_tokens = len(encoded)
+# def tokenize(text: str, model: str = 'gpt-3.5-turbo') -> Union[int, str]:
+#     encoding   = tiktoken.encoding_for_model(model)
+#     encoded    = encoding.encode(text)
+#     num_tokens = len(encoded)
     
-    return num_tokens, encoded
\ No newline at end of file
+#     return num_tokens, encoded
\ No newline at end of file
-- 
cgit v1.2.3