generated from dellevin/template
63 lines
1.7 KiB
Python
63 lines
1.7 KiB
Python
"""
|
|
分词接口调用示例
|
|
"""
|
|
import requests
|
|
import json
|
|
|
|
BASE_URL = "http://127.0.0.1:27056"
|
|
# BASE_URL = "http://fenci.iletter.top/"
|
|
|
|
|
|
def tokenize_single(text):
|
|
"""单文本分词"""
|
|
url = f"{BASE_URL}/tokenize"
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer DdzBgb8GEkBpA8gtCJXP24hGJz9bXpEOi6z91fHm25X59q5968XqtLxPi1MfiTHJ"
|
|
}
|
|
payload = {"input_text": text}
|
|
|
|
response = requests.post(url, headers=headers, json=payload)
|
|
print(response.json())
|
|
return response.json()
|
|
|
|
|
|
def tokenize_batch(texts):
|
|
"""批量分词"""
|
|
url = f"{BASE_URL}/batch-tokenize"
|
|
headers = {
|
|
"Content-Type": "application/json",
|
|
"Authorization": f"Bearer DdzBgb8GEkBpA8gtCJXP24hGJz9bXpEOi6z91fHm25X59q5968XqtLxPi1MfiTHJ"
|
|
}
|
|
payload = {"texts": texts}
|
|
|
|
response = requests.post(url, headers=headers, json=payload)
|
|
return response.json()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
# 示例 1: 单文本分词
|
|
print("=" * 50)
|
|
print("示例 1: 单文本分词")
|
|
print("=" * 50)
|
|
|
|
text = "MemoFlowMookNoteVoceChat"
|
|
result = tokenize_single(text)
|
|
|
|
if result["success"]:
|
|
print(f"原文: {result['original_text']}")
|
|
print(f"\n分词结果:")
|
|
for token in result["tokens"]:
|
|
print(f" [{token['type']:12}] {token['text']}")
|
|
|
|
print(f"\n统计信息:")
|
|
stats = result["stats"]
|
|
print(f" 总计: {stats['total']} 个单词")
|
|
print(f" 中文: {stats['chinese']} 个")
|
|
print(f" 英文: {stats['english']} 个")
|
|
print(f" 数字: {stats['number']} 个")
|
|
print(f" 标点: {stats['punctuation']} 个")
|
|
else:
|
|
print(f"错误: {result['error']}")
|
|
|