feat: complete local version to overwrite remote

This commit is contained in:
DelLevin-Home
2026-06-16 03:30:57 +08:00
parent 1735c19f48
commit 3c78293f4d
129 changed files with 22814 additions and 0 deletions

View File

@@ -0,0 +1 @@
# utils package

View File

@@ -0,0 +1,316 @@
# -*- coding: utf-8 -*-
"""
down-video 视频下载工具函数
"""
import os
import re
import subprocess
from urllib.parse import urlparse
from config import TWITTER_COOKIE, BILIBILI_COOKIE, INSTAGRAM_COOKIE, YOUTUBE_COOKIE
# 可选依赖
try:
import yt_dlp
YTDLP_AVAILABLE = True
except ImportError:
YTDLP_AVAILABLE = False
def check_ffmpeg():
"""检查 FFmpeg 是否可用"""
try:
result = subprocess.run(['ffmpeg', '-version'], capture_output=True, text=True, timeout=5)
return result.returncode == 0
except Exception:
return False
def check_deno():
"""检查 Deno 是否可用"""
try:
result = subprocess.run(['deno', '--version'], capture_output=True, text=True, timeout=5, shell=True)
if result.returncode == 0:
return True
except Exception:
pass
# fallback: 检查默认安装路径
deno_path = os.path.join(os.path.expanduser('~'), '.deno', 'bin', 'deno.exe')
if os.path.isfile(deno_path):
try:
result = subprocess.run([deno_path, '--version'], capture_output=True, text=True, timeout=5)
return result.returncode == 0
except Exception:
pass
return False
def is_twitter_url(url):
"""检查是否是 Twitter/X 链接"""
domain = urlparse(url).netloc.lower()
return 'x.com' in domain or 'twitter.com' in domain
def is_bilibili_url(url):
"""检查是否是 Bilibili 链接"""
domain = urlparse(url).netloc.lower()
return 'bilibili.com' in domain or 'b23.tv' in domain
def is_instagram_url(url):
"""检查是否是 Instagram 链接"""
domain = urlparse(url).netloc.lower()
return 'instagram.com' in domain
def is_youtube_url(url):
"""检查是否是 YouTube 链接"""
domain = urlparse(url).netloc.lower()
return 'youtube.com' in domain or 'youtu.be' in domain
def _clean_title(title):
"""
清理视频标题,移除特殊字符,使文件名在文件系统中安全可用
"""
if not title:
return "video"
# 移除或替换特殊字符
# Windows 文件名不允许的字符
cleaned = re.sub(r'[<>:"/\\|?*]', '_', title)
# 控制字符
cleaned = re.sub(r'[\x00-\x1f\x7f]', '', cleaned)
# 移除 emoji 和其他 Unicode 特殊字符(保留中文、英文、数字、下划线)
# 匹配非中文、非英文、非数字、非下划线的字符
cleaned = re.sub(r'[^一-鿿㐀-䶿\w]', '_', cleaned)
# 将多个连续下划线合并为一个
cleaned = re.sub(r'_+', '_', cleaned)
# 移除首尾空格和点Windows不允许
cleaned = cleaned.strip('_. ')
# 限制长度(保留扩展名空间)
cleaned = cleaned[:150]
# 如果清理后为空,使用默认名
if not cleaned:
return "video"
return cleaned
def _find_files_by_title(output_dir, safe_title):
"""根据安全标题查找目录中的相关文件"""
return [
f for f in os.listdir(output_dir)
if f.startswith(safe_title) and os.path.isfile(os.path.join(output_dir, f))
]
def _merge_mp4_m4a(output_dir, safe_title):
"""
查找同名的 .mp4 和 .m4a 并用 FFmpeg 合并。
返回 (merged: bool, message: str, filepath: str)
"""
files = _find_files_by_title(output_dir, safe_title)
mp4_files = [f for f in files if f.endswith('.mp4')]
m4a_files = [f for f in files if f.endswith('.m4a')]
if not mp4_files or not m4a_files:
return False, "未检测到需要合并的 .mp4 + .m4a", ""
mp4_files.sort(key=len)
m4a_files.sort(key=len)
mp4_path = os.path.join(output_dir, mp4_files[0])
m4a_path = os.path.join(output_dir, m4a_files[0])
merged_name = safe_title + "_merged.mp4"
merged_path = os.path.join(output_dir, merged_name)
counter = 1
while os.path.exists(merged_path):
merged_name = f"{safe_title}_merged_{counter}.mp4"
merged_path = os.path.join(output_dir, merged_name)
counter += 1
cmd = ['ffmpeg', '-y', '-i', mp4_path, '-i', m4a_path, '-c', 'copy', merged_path]
try:
result = subprocess.run(cmd, capture_output=True, text=True, timeout=300)
if result.returncode == 0 and os.path.exists(merged_path):
try:
os.remove(mp4_path)
os.remove(m4a_path)
except Exception:
pass
return True, f"已合并为 {merged_name}", merged_path
else:
return False, f"FFmpeg 合并失败: {result.stderr[:200]}", ""
except Exception as e:
return False, f"合并异常: {str(e)}", ""
class DownloadProgressLogger:
"""捕获yt-dlp下载进度的回调类"""
def __init__(self, progress_callback=None):
self.progress_callback = progress_callback
def debug(self, msg):
pass
def warning(self, msg):
pass
def error(self, msg):
if self.progress_callback:
self.progress_callback({'type': 'error', 'message': msg})
def info(self, msg):
if self.progress_callback:
self.progress_callback({'type': 'info', 'message': msg})
def download_progress(self, d):
if d['status'] == 'downloading' and self.progress_callback:
percent = d.get('_percent_str', '0%').replace('%', '').strip()
try:
percent = float(percent)
except ValueError:
percent = 0
speed = d.get('_speed_str', 'N/A')
eta = d.get('_eta_str', 'N/A')
filename = d.get('filename', '').split('\\')[-1].split('/')[-1]
self.progress_callback({
'type': 'progress',
'percent': percent,
'speed': speed,
'eta': eta,
'filename': filename
})
elif d['status'] == 'finished' and self.progress_callback:
self.progress_callback({
'type': 'progress',
'percent': 100,
'speed': '',
'eta': '',
'filename': ''
})
def download_video(video_url, output_dir, platform, progress_callback=None, proxy_url=None):
"""
通用下载函数,返回 (success: bool, message: str, title: str, filepath: str)
支持进度回调
"""
if not YTDLP_AVAILABLE:
return False, "yt-dlp 未安装,视频下载功能不可用。", "", ""
os.makedirs(output_dir, exist_ok=True)
# 先提取视频信息以获取标题
temp_opts = {
'quiet': True,
'no_warnings': True,
'skip_download': True,
}
if platform == 'twitter':
cookie_file = TWITTER_COOKIE
if proxy_url:
temp_opts['proxy'] = proxy_url
elif platform == 'bilibili':
cookie_file = BILIBILI_COOKIE
elif platform == 'instagram':
cookie_file = INSTAGRAM_COOKIE
if proxy_url:
temp_opts['proxy'] = proxy_url
elif platform == 'youtube':
cookie_file = YOUTUBE_COOKIE
if proxy_url:
temp_opts['proxy'] = proxy_url
else:
return False, "不支持的平台", "", ""
if os.path.exists(cookie_file):
temp_opts['cookiefile'] = cookie_file
else:
return False, f"Cookie 文件 '{cookie_file}' 未找到", "", ""
try:
# 创建进度日志器
progress_logger = DownloadProgressLogger(progress_callback)
# 获取视频信息
with yt_dlp.YoutubeDL(temp_opts) as ydl:
info = ydl.extract_info(video_url, download=False)
if not info:
return False, "未能获取视频信息", "", ""
# 清理标题作为安全文件名
title = info.get('title', 'Unknown Title')
safe_title = _clean_title(title)
# 使用清理后的标题构建输出模板
ydl_opts = {
'outtmpl': os.path.join(output_dir, f'{safe_title}.%(ext)s'),
'ignoreerrors': True,
'progress_hooks': [progress_logger.download_progress] if progress_callback else [],
'logger': progress_logger if progress_callback else None,
'format': 'bestvideo+bestaudio/best',
}
if platform == 'twitter':
if proxy_url:
ydl_opts['proxy'] = proxy_url
ydl_opts['merge_output_format'] = 'mp4'
ydl_opts['postprocessors'] = [{
'key': 'FFmpegVideoConvertor',
'preferedformat': 'mp4',
}]
elif platform == 'bilibili':
ydl_opts['merge_output_format'] = 'mp4'
elif platform == 'instagram':
if proxy_url:
ydl_opts['proxy'] = proxy_url
ydl_opts['merge_output_format'] = 'mp4'
elif platform == 'youtube':
if proxy_url:
ydl_opts['proxy'] = proxy_url
ydl_opts['merge_output_format'] = 'mp4'
if os.path.exists(cookie_file):
ydl_opts['cookiefile'] = cookie_file
# 执行下载
with yt_dlp.YoutubeDL(ydl_opts) as ydl:
ydl.download([video_url])
# 查找下载的文件
files = _find_files_by_title(output_dir, safe_title)
if not files:
return False, "下载完成但未找到输出文件", title, ""
filepath = ""
candidate_files = [f for f in files if f.endswith('.mp4')]
if candidate_files:
merged_candidates = [f for f in candidate_files if '_merged' in f]
if merged_candidates:
filepath = os.path.join(output_dir, merged_candidates[0])
else:
candidate_files.sort(key=lambda f: os.path.getsize(os.path.join(output_dir, f)), reverse=True)
filepath = os.path.join(output_dir, candidate_files[0])
else:
files.sort(key=lambda f: os.path.getsize(os.path.join(output_dir, f)), reverse=True)
filepath = os.path.join(output_dir, files[0])
extra_msg = ""
if platform in ('bilibili', 'youtube'):
has_mp4 = any(f.endswith('.mp4') for f in files)
has_m4a = any(f.endswith('.m4a') for f in files)
if has_mp4 and has_m4a:
merged, merge_msg, merged_path = _merge_mp4_m4a(output_dir, safe_title)
extra_msg = f" ({merge_msg})" if merge_msg else ""
if merged and merged_path:
filepath = merged_path
return True, f"下载成功!保存到 {output_dir}{extra_msg}", title, filepath
except Exception as e:
return False, f"下载失败: {str(e)}", "", ""

View File

@@ -0,0 +1,34 @@
import chardet
# 自动选择解码格式
def detect_and_decode(data_bytes):
"""使用 chardet 检测编码并解码"""
# 使用 chardet 检测编码
detected_encoding_info = chardet.detect(data_bytes)
detected_encoding = detected_encoding_info.get('encoding')
confidence = detected_encoding_info.get('confidence', 0)
if detected_encoding:
try:
decoded_string = data_bytes.decode(detected_encoding)
print(f" - 使用检测到的编码 '{detected_encoding}' 成功解码,置信度:{confidence}")
return decoded_string
except UnicodeDecodeError as e:
print(f" - 使用检测到的编码 '{detected_encoding}' 解码失败: {e}")
# 方法二:如果检测失败或置信度低,尝试常见的编码
encodings_to_try = ['utf-8', 'gbk', 'gb2312', 'cp936']
print(f" - 检测失败/置信度低,尝试常见编码")
print(f" - 尝试常见编码列表: {encodings_to_try}")
for enc in encodings_to_try:
try:
decoded_string = data_bytes.decode(enc)
print(f" - 成功使用编码 '{enc}' 解码。")
return decoded_string
except UnicodeDecodeError as e:
print(f" - 尝试编码 '{enc}' 失败: {e}")
continue
# 如果所有方法都失败
print(" - 所有编码尝试均失败。")
return "所有编码尝试均失败。"

View File

@@ -0,0 +1,149 @@
# -*- coding: utf-8 -*-
"""
fen-ci 分词工具函数
"""
import os
import re
import json
from config import JIEBA_DICT_FILE, TOKEN_FILE_PATH
# 可选依赖
try:
import jieba
JIEBA_AVAILABLE = True
except ImportError:
JIEBA_AVAILABLE = False
class Tokenizer:
"""分词器:加载词典并提供分词能力"""
def __init__(self):
self.simple_words = []
self.compound_words = set()
self.compound_pattern = None
self._load_words()
def _load_words(self):
custom_words = set()
if os.path.exists(JIEBA_DICT_FILE):
try:
with open(JIEBA_DICT_FILE, 'r', encoding='utf-8') as f:
for line in f:
word = line.strip()
if word:
custom_words.add(word)
except Exception as e:
print(f"读取自定义词汇文件失败: {e}")
compound = {w for w in custom_words if any(c.isupper() for c in w)}
simple = custom_words - compound
self.simple_words = list(simple)
self.compound_words = compound
if compound:
self.compound_pattern = re.compile('|'.join(re.escape(w) for w in compound), re.IGNORECASE)
else:
self.compound_pattern = None
if JIEBA_AVAILABLE:
for w in self.simple_words:
jieba.add_word(w)
# print(f"加载了 {len(self.simple_words)} 个简单自定义词汇,{len(self.compound_words)} 个复合词。")
def tokenize(self, text):
if not text or not text.strip():
return []
tokens = []
chinese_pattern = re.compile(r'[\u4e00-\u9fff]+')
english_pattern = re.compile(r'[a-zA-Z]+(?:-[a-zA-Z]+)*')
number_pattern = re.compile(r'-?\d+\.?\d*')
pos = 0
text_length = len(text)
while pos < text_length:
char = text[pos]
if char.isspace():
pos += 1
continue
if self.compound_pattern:
compound_match = self.compound_pattern.match(text, pos)
if compound_match:
matched_word = compound_match.group()
tokens.append({'text': matched_word, 'type': 'compound_eng', 'length': len(matched_word)})
pos += len(matched_word)
continue
if chinese_pattern.match(char):
chinese_str = ''
while pos < text_length and chinese_pattern.match(text[pos]):
chinese_str += text[pos]
pos += 1
if JIEBA_AVAILABLE:
chinese_tokens = list(jieba.cut(chinese_str))
else:
chinese_tokens = list(chinese_str)
for token in chinese_tokens:
if token.strip():
tokens.append({'text': token, 'type': 'chinese', 'length': len(token)})
elif english_pattern.match(char):
match = english_pattern.match(text, pos)
if match:
word = match.group()
tokens.append({'text': word, 'type': 'english', 'length': len(word)})
pos += len(word)
elif number_pattern.match(char) or (char == '-' and pos + 1 < text_length and text[pos + 1].isdigit()):
match = number_pattern.match(text, pos)
if match:
number = match.group()
tokens.append({'text': number, 'type': 'number', 'length': len(number)})
pos += len(number)
else:
tokens.append({'text': char, 'type': 'punctuation', 'length': 1})
pos += 1
return tokens
@staticmethod
def get_stats(tokens):
stats = {'total': len(tokens), 'chinese': 0, 'english': 0, 'compound_eng': 0, 'number': 0, 'punctuation': 0}
for token in tokens:
t = token['type']
if t in stats:
stats[t] += 1
stats['total'] = sum(v for k, v in stats.items() if k != 'total')
return stats
class TokenAuth:
"""Token 认证管理"""
def __init__(self):
self.valid_tokens = {}
self._load_tokens()
def _load_tokens(self):
if not os.path.exists(TOKEN_FILE_PATH):
print(f"警告: 找不到Token文件 '{TOKEN_FILE_PATH}'")
return
try:
with open(TOKEN_FILE_PATH, 'r', encoding='utf-8') as f:
tokens_data = json.load(f)
token_map = {}
for item in tokens_data:
if 'user' in item and 'token' in item:
token_map[item['token']] = item['user']
self.valid_tokens = token_map
# print(f"成功加载了 {len(token_map)} 个有效Token。")
except Exception as e:
print(f"读取Token文件失败: {e}")
def get_user(self, token):
return self.valid_tokens.get(token)
# 全局单例
tokenizer = Tokenizer()
token_auth = TokenAuth()

View File

@@ -0,0 +1,72 @@
# -*- coding: utf-8 -*-
"""
pin-tu 图片浏览工具函数
"""
import os
import re
def atoi(text):
"""辅助函数:将文本中的数字部分转换为整数,用于排序"""
return int(text) if text.isdigit() else text
def natural_keys(text):
"""自然排序键生成器"""
return [atoi(c) for c in re.split(r'(\d+)', text)]
def find_media_in_folder(folder_path):
"""在指定文件夹中查找所有图片、视频和其他文件"""
allowed_image_exts = {'.jpg', '.jpeg', '.png', '.gif', '.bmp', '.tiff', '.webp'}
allowed_video_exts = {'.mp4', '.avi', '.mov', '.wmv', '.flv', '.webm', '.m4v', '.mkv', '.mpg', '.mpeg', '.3gp', '.3g2'}
try:
files = os.listdir(folder_path)
except PermissionError:
return [], [], []
image_files = []
video_files = []
other_files = []
for f in files:
if not os.path.isfile(os.path.join(folder_path, f)):
continue
ext = os.path.splitext(f)[1].lower()
if ext in allowed_image_exts:
image_files.append(f)
elif ext in allowed_video_exts:
video_files.append(f)
else:
other_files.append(f)
return sorted(image_files, key=natural_keys), sorted(video_files, key=natural_keys), sorted(other_files, key=natural_keys)
def get_subfolders(folder_path):
"""获取指定文件夹下的所有子文件夹"""
try:
items = os.listdir(folder_path)
except PermissionError:
return []
subfolders = [item for item in items if os.path.isdir(os.path.join(folder_path, item))]
return sorted(subfolders, key=natural_keys)
def build_breadcrumbs(subpath, base_url='/pin-tu/browse'):
"""构建面包屑导航数据
base_url 示例: /pin-tu/browse/RDovRGF0YQ
返回 URL 格式: {base_url}?p={累计子路径}
"""
path_parts = [p for p in subpath.split('/') if p]
breadcrumbs = [{'name': 'Home', 'url': base_url}]
cumulative = ''
for part in path_parts:
cumulative += ('/' if cumulative else '') + part
breadcrumbs.append({'name': part, 'url': f'{base_url}?p={cumulative}'})
return breadcrumbs
def is_safe_path(root, target):
"""安全检查:防止路径穿越"""
return os.path.normpath(target).startswith(os.path.normpath(root))

View File

@@ -0,0 +1,121 @@
# -*- coding: utf-8 -*-
"""
RVC 变声后台工作进程
通过 stdin/stdout 二进制协议与 Flask 通信
sys.stdout 重定向到 stderr防止 RVC 的 print() 污染通信通道
"""
import sys
import os
import json
import struct
import traceback
_comm_stdout = sys.stdout.buffer
sys.stdout = sys.stderr
rvc_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.abspath(__file__))
os.chdir(rvc_dir)
if rvc_dir not in sys.path:
sys.path.insert(0, rvc_dir)
# 必须在 Config() 之前,防止 argparse 解析失败
sys.argv = sys.argv[:1]
def send_msg(obj):
data = json.dumps(obj, ensure_ascii=False).encode('utf-8')
_comm_stdout.write(struct.pack('<I', len(data)))
_comm_stdout.write(data)
_comm_stdout.flush()
def recv_msg():
raw_len = sys.stdin.buffer.read(4)
if not raw_len:
return None
msg_len = struct.unpack('<I', raw_len)[0]
data = sys.stdin.buffer.read(msg_len)
return json.loads(data.decode('utf-8'))
def main():
try:
from dotenv import load_dotenv
load_dotenv(os.path.join(rvc_dir, '.env'))
from configs.config import Config
config = Config()
from infer.modules.vc.modules import VC
vc = VC(config)
send_msg({'type': 'ready', 'device': str(config.device), 'is_half': config.is_half})
except Exception:
send_msg({'type': 'error', 'error': traceback.format_exc()})
return
import numpy as np
import wave
while True:
req = recv_msg()
if req is None:
break
msg_type = req.get('type')
if msg_type == 'exit':
break
elif msg_type == 'load_model':
try:
vc.get_vc(req['model_name'])
send_msg({'type': 'done'})
except Exception:
send_msg({'type': 'error', 'error': traceback.format_exc()})
elif msg_type == 'convert':
try:
input_path = req['input_path']
output_path = req['output_path']
f0_up_key = req.get('f0_up_key', 0)
f0_method = req.get('f0_method', 'rmvpe')
file_index = req.get('file_index', '')
index_rate = req.get('index_rate', 0.75)
filter_radius = req.get('filter_radius', 3)
resample_sr = req.get('resample_sr', 0)
rms_mix_rate = req.get('rms_mix_rate', 0.25)
protect = req.get('protect', 0.33)
status, (sr, audio) = vc.vc_single(
sid=0,
input_audio_path=input_path,
f0_up_key=f0_up_key,
f0_file=None,
f0_method=f0_method,
file_index=file_index,
file_index2=None,
index_rate=index_rate,
filter_radius=filter_radius,
resample_sr=resample_sr,
rms_mix_rate=rms_mix_rate,
protect=protect,
)
if audio is None:
send_msg({'type': 'error', 'error': status or '变声失败'})
else:
if audio.dtype != np.int16:
audio = audio.astype(np.int16)
with wave.open(output_path, 'w') as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(sr)
wf.writeframes(audio.tobytes())
send_msg({'type': 'done', 'output_path': output_path, 'sample_rate': sr})
except Exception:
send_msg({'type': 'error', 'error': traceback.format_exc()})
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,108 @@
# -*- coding: utf-8 -*-
"""
GPT-SoVITS 后台工作进程
通过 stdin/stdout 二进制协议与 Flask 通信
sys.stdout 重定向到 stderr防止 GPT-SoVITS 的 print() 污染通信通道
"""
import sys
import os
import json
import struct
import traceback
_comm_stdout = sys.stdout.buffer
sys.stdout = sys.stderr
sovits_dir = sys.argv[1] if len(sys.argv) > 1 else os.path.dirname(os.path.abspath(__file__))
os.chdir(sovits_dir)
if sovits_dir not in sys.path:
sys.path.insert(0, sovits_dir)
gpt_sovits_dir = os.path.join(sovits_dir, 'GPT_SoVITS')
if gpt_sovits_dir not in sys.path:
sys.path.insert(0, gpt_sovits_dir)
def send_msg(obj):
data = json.dumps(obj, ensure_ascii=False).encode('utf-8')
_comm_stdout.write(struct.pack('<I', len(data)))
_comm_stdout.write(data)
_comm_stdout.flush()
def recv_msg():
raw_len = sys.stdin.buffer.read(4)
if not raw_len:
return None
msg_len = struct.unpack('<I', raw_len)[0]
data = sys.stdin.buffer.read(msg_len)
return json.loads(data.decode('utf-8'))
def main():
config_yaml = sys.argv[2] if len(sys.argv) > 2 else 'GPT_SoVITS/configs/tts_infer.yaml'
try:
from GPT_SoVITS.TTS_infer_pack.TTS import TTS, TTS_Config
yaml_path = os.path.join(sovits_dir, config_yaml)
tts_config = TTS_Config(yaml_path)
tts_engine = TTS(tts_config)
version = getattr(tts_config, 'version', 'unknown')
send_msg({'type': 'ready', 'version': version})
except Exception as e:
send_msg({'type': 'error', 'error': traceback.format_exc()})
return
import numpy as np
import wave
while True:
req = recv_msg()
if req is None:
break
msg_type = req.get('type')
if msg_type == 'exit':
break
elif msg_type == 'reload_gpt':
try:
tts_engine.init_t2s_weights(req['path'])
send_msg({'type': 'done'})
except Exception as e:
send_msg({'type': 'error', 'error': traceback.format_exc()})
elif msg_type == 'reload_sovits':
try:
tts_engine.init_vits_weights(req['path'])
send_msg({'type': 'done'})
except Exception as e:
send_msg({'type': 'error', 'error': traceback.format_exc()})
elif msg_type == 'synthesize':
try:
inputs = req['inputs']
output_path = req['output_path']
tts_generator = tts_engine.run(inputs)
sr, audio_data = next(tts_generator)
if audio_data.dtype != np.int16:
if audio_data.max() <= 1.0:
audio_data = (audio_data * 32767).astype(np.int16)
else:
audio_data = audio_data.astype(np.int16)
with wave.open(output_path, 'w') as wf:
wf.setnchannels(1)
wf.setsampwidth(2)
wf.setframerate(sr)
wf.writeframes(audio_data.tobytes())
send_msg({'type': 'done', 'output_path': output_path})
except Exception as e:
send_msg({'type': 'error', 'error': traceback.format_exc()})
if __name__ == '__main__':
main()

View File

@@ -0,0 +1,589 @@
# -*- coding: utf-8 -*-
"""
接口调用统计模块
使用 SQLite 存储调用记录,支持按日/月查询
"""
import sqlite3
import os
from datetime import datetime, timedelta
DB_PATH = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'stats.db')
try:
from config import BASE_DIR
DB_PATH = os.path.join(BASE_DIR, 'stats.db')
except ImportError:
pass
def _get_conn():
conn = sqlite3.connect(DB_PATH)
conn.row_factory = sqlite3.Row
return conn
def init_db():
conn = _get_conn()
conn.execute('''
CREATE TABLE IF NOT EXISTS api_calls (
id INTEGER PRIMARY KEY AUTOINCREMENT,
endpoint TEXT NOT NULL,
method TEXT NOT NULL,
status_code INTEGER,
duration_ms REAL,
called_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS search_history (
id INTEGER PRIMARY KEY AUTOINCREMENT,
keyword TEXT NOT NULL,
root_path TEXT NOT NULL,
result_count INTEGER DEFAULT 0,
searched_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS recent_paths (
id INTEGER PRIMARY KEY AUTOINCREMENT,
path TEXT NOT NULL UNIQUE,
last_used TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS content_tag_settings (
id INTEGER PRIMARY KEY CHECK (id = 1),
system_msg TEXT,
user_msg TEXT,
max_tags INTEGER DEFAULT 10,
min_length INTEGER DEFAULT 2,
max_length INTEGER DEFAULT 6
)
''')
conn.execute('INSERT OR IGNORE INTO content_tag_settings (id) VALUES (1)')
conn.commit()
conn.close()
def record_call(endpoint, method, status_code, duration_ms):
conn = _get_conn()
conn.execute(
'INSERT INTO api_calls (endpoint, method, status_code, duration_ms) VALUES (?, ?, ?, ?)',
(endpoint, method, status_code, round(duration_ms, 2))
)
conn.commit()
conn.close()
def get_total_count():
conn = _get_conn()
row = conn.execute('SELECT COUNT(*) as cnt FROM api_calls').fetchone()
conn.close()
return row['cnt']
def get_avg_duration():
conn = _get_conn()
row = conn.execute('SELECT AVG(duration_ms) as avg_dur FROM api_calls').fetchone()
conn.close()
return round(row['avg_dur'] or 0, 2)
def get_daily_counts(days=30):
since = (datetime.now() - timedelta(days=days)).strftime('%Y-%m-%d')
conn = _get_conn()
rows = conn.execute(
"SELECT DATE(called_at, '+8 hours') as day, COUNT(*) as cnt "
"FROM api_calls WHERE DATE(called_at, '+8 hours') >= ? "
"GROUP BY day ORDER BY day",
(since,)
).fetchall()
conn.close()
return [{'date': r['day'], 'count': r['cnt']} for r in rows]
def get_monthly_counts(year=None):
conn = _get_conn()
if year:
rows = conn.execute(
"SELECT strftime('%Y-%m', called_at, '+8 hours') as month, COUNT(*) as cnt "
"FROM api_calls WHERE strftime('%Y', called_at, '+8 hours') = ? "
"GROUP BY month ORDER BY month",
(str(year),)
).fetchall()
else:
rows = conn.execute(
"SELECT strftime('%Y-%m', called_at, '+8 hours') as month, COUNT(*) as cnt "
"FROM api_calls GROUP BY month ORDER BY month"
).fetchall()
conn.close()
return [{'month': r['month'], 'count': r['cnt']} for r in rows]
def get_available_years():
conn = _get_conn()
rows = conn.execute(
"SELECT DISTINCT strftime('%Y', called_at, '+8 hours') as year "
"FROM api_calls ORDER BY year DESC"
).fetchall()
conn.close()
return [r['year'] for r in rows]
def get_available_months(year):
conn = _get_conn()
rows = conn.execute(
"SELECT DISTINCT strftime('%m', called_at, '+8 hours') as month "
"FROM api_calls WHERE strftime('%Y', called_at, '+8 hours') = ? "
"ORDER BY month",
(str(year),)
).fetchall()
conn.close()
return [r['month'] for r in rows]
def get_daily_counts_by_month(year, month):
ym_prefix = f'{year}-{month:02d}'
conn = _get_conn()
rows = conn.execute(
"SELECT strftime('%d', called_at, '+8 hours') as day, COUNT(*) as cnt "
"FROM api_calls WHERE strftime('%Y-%m', called_at, '+8 hours') = ? "
"GROUP BY day ORDER BY day",
(ym_prefix,)
).fetchall()
conn.close()
day_map = {r['day']: r['cnt'] for r in rows}
# 获取该月天数
import calendar
_, days_in_month = calendar.monthrange(int(year), int(month))
return [{'date': f'{ym_prefix}-{d:02d}', 'day': str(d), 'count': day_map.get(f'{d:02d}', 0)} for d in range(1, days_in_month + 1)]
def get_hourly_counts(date_str):
conn = _get_conn()
rows = conn.execute(
"SELECT strftime('%H', called_at, '+8 hours') as hour, COUNT(*) as cnt "
"FROM api_calls WHERE DATE(called_at, '+8 hours') = ? "
"GROUP BY hour ORDER BY hour",
(date_str,)
).fetchall()
conn.close()
hour_map = {r['hour']: r['cnt'] for r in rows}
return [{'hour': f'{h:02d}', 'count': hour_map.get(f'{h:02d}', 0)} for h in range(24)]
def get_endpoint_stats(date=None):
conn = _get_conn()
if date:
rows = conn.execute(
"SELECT endpoint, COUNT(*) as cnt, AVG(duration_ms) as avg_dur "
"FROM api_calls WHERE DATE(called_at, '+8 hours') = ? "
"GROUP BY endpoint ORDER BY cnt DESC",
(date,)
).fetchall()
else:
rows = conn.execute(
"SELECT endpoint, COUNT(*) as cnt, AVG(duration_ms) as avg_dur "
"FROM api_calls GROUP BY endpoint ORDER BY cnt DESC"
).fetchall()
conn.close()
return [{'endpoint': r['endpoint'], 'count': r['cnt'], 'avg_duration': round(r['avg_dur'] or 0, 2)} for r in rows]
def get_today_count():
conn = _get_conn()
row = conn.execute(
"SELECT COUNT(*) as cnt FROM api_calls WHERE DATE(called_at, '+8 hours') = DATE('now', '+8 hours')"
).fetchone()
conn.close()
return row['cnt']
def get_month_count():
conn = _get_conn()
row = conn.execute(
"SELECT COUNT(*) as cnt FROM api_calls WHERE strftime('%Y-%m', called_at, '+8 hours') = strftime('%Y-%m', 'now', '+8 hours')"
).fetchone()
conn.close()
return row['cnt']
def get_yesterday_count():
conn = _get_conn()
row = conn.execute(
"SELECT COUNT(*) as cnt FROM api_calls WHERE DATE(called_at, '+8 hours') = DATE('now', '+8 hours', '-1 day')"
).fetchone()
conn.close()
return row['cnt']
def get_success_rate():
conn = _get_conn()
total = conn.execute("SELECT COUNT(*) as cnt FROM api_calls").fetchone()['cnt']
if total == 0:
conn.close()
return 100.0
success = conn.execute("SELECT COUNT(*) as cnt FROM api_calls WHERE status_code >= 200 AND status_code < 300").fetchone()['cnt']
conn.close()
return round(success / total * 100, 1)
def get_duration_distribution():
conn = _get_conn()
ranges = [
('<100ms', 0, 100),
('100-500ms', 100, 500),
('0.5-1s', 500, 1000),
('1-3s', 1000, 3000),
('3-10s', 3000, 10000),
('>10s', 10000, 999999999),
]
result = []
for label, lo, hi in ranges:
cnt = conn.execute(
"SELECT COUNT(*) as cnt FROM api_calls WHERE duration_ms >= ? AND duration_ms < ?", (lo, hi)
).fetchone()['cnt']
result.append({'label': label, 'count': cnt})
conn.close()
return result
def get_slowest_endpoints(limit=5):
conn = _get_conn()
rows = conn.execute(
"SELECT endpoint, AVG(duration_ms) as avg_dur, MAX(duration_ms) as max_dur, COUNT(*) as cnt "
"FROM api_calls GROUP BY endpoint ORDER BY avg_dur DESC LIMIT ?", (limit,)
).fetchall()
conn.close()
return [{
'endpoint': r['endpoint'],
'avg_duration': round(r['avg_dur'] or 0, 2),
'max_duration': round(r['max_dur'] or 0, 2),
'count': r['cnt']
} for r in rows]
def get_week_compare():
conn = _get_conn()
this_week = []
last_week = []
for i in range(7):
# 本周: 今天往前 i 天
row = conn.execute(
"SELECT COUNT(*) as cnt FROM api_calls WHERE DATE(called_at, '+8 hours') = DATE('now', '+8 hours', ?)",
(f'-{i} day',)
).fetchone()
this_week.append(row['cnt'])
# 上周: 本周对应天数再往前 7 天
row = conn.execute(
"SELECT COUNT(*) as cnt FROM api_calls WHERE DATE(called_at, '+8 hours') = DATE('now', '+8 hours', ?)",
(f'-{i + 7} day',)
).fetchone()
last_week.append(row['cnt'])
conn.close()
# 返回按周一到周日顺序(今天是 index 0需要反转
this_week.reverse()
last_week.reverse()
# 生成日期标签
from datetime import datetime, timedelta
today = datetime.now()
weekday = today.weekday() # 0=周一
labels = []
for i in range(7):
d = today - timedelta(days=weekday - i)
labels.append(d.strftime('%m-%d'))
return {'labels': labels, 'this_week': this_week, 'last_week': last_week}
def get_recent_calls(limit=20):
conn = _get_conn()
rows = conn.execute(
"SELECT endpoint, method, status_code, duration_ms, "
"strftime('%Y-%m-%d %H:%M:%S', called_at, '+8 hours') as called_at "
"FROM api_calls ORDER BY called_at DESC LIMIT ?", (limit,)
).fetchall()
conn.close()
return [dict(r) for r in rows]
def get_active_days():
conn = _get_conn()
row = conn.execute(
"SELECT COUNT(DISTINCT DATE(called_at, '+8 hours')) as cnt FROM api_calls"
).fetchone()
# 连续活跃天数
rows = conn.execute(
"SELECT DISTINCT DATE(called_at, '+8 hours') as day FROM api_calls ORDER BY day DESC"
).fetchall()
conn.close()
total_days = row['cnt']
streak = 0
if rows:
from datetime import datetime, timedelta
today = datetime.now().strftime('%Y-%m-%d')
expected = today
for r in rows:
if r['day'] == expected:
streak += 1
d = datetime.strptime(expected, '%Y-%m-%d') - timedelta(days=1)
expected = d.strftime('%Y-%m-%d')
else:
break
return {'total_days': total_days, 'streak': streak}
# ===== 搜索历史 =====
def add_search(keyword, root_path, result_count):
conn = _get_conn()
conn.execute(
'INSERT INTO search_history (keyword, root_path, result_count) VALUES (?, ?, ?)',
(keyword, root_path, result_count)
)
conn.commit()
conn.close()
def get_search_history(root_path, limit=20):
conn = _get_conn()
rows = conn.execute(
"SELECT id, keyword, result_count, strftime('%Y-%m-%d %H:%M', searched_at, '+8 hours') as searched_at "
"FROM search_history WHERE root_path = ? ORDER BY searched_at DESC LIMIT ?",
(root_path, limit)
).fetchall()
conn.close()
return [dict(r) for r in rows]
def delete_search_history(ids):
if not ids:
return
conn = _get_conn()
placeholders = ','.join('?' for _ in ids)
conn.execute(f'DELETE FROM search_history WHERE id IN ({placeholders})', ids)
conn.commit()
conn.close()
def clear_search_history(root_path):
conn = _get_conn()
conn.execute('DELETE FROM search_history WHERE root_path = ?', (root_path,))
conn.commit()
conn.close()
# ===== 最近访问路径 =====
def add_recent_path(path):
conn = _get_conn()
conn.execute(
'INSERT INTO recent_paths (path, last_used) VALUES (?, CURRENT_TIMESTAMP) '
'ON CONFLICT(path) DO UPDATE SET last_used = CURRENT_TIMESTAMP',
(path,)
)
conn.commit()
conn.close()
def get_recent_paths(limit=8):
conn = _get_conn()
rows = conn.execute(
"SELECT id, path, strftime('%Y-%m-%d %H:%M', last_used, '+8 hours') as last_used "
"FROM recent_paths ORDER BY last_used DESC LIMIT ?",
(limit,)
).fetchall()
conn.close()
return [dict(r) for r in rows]
def delete_recent_path(path):
conn = _get_conn()
conn.execute('DELETE FROM recent_paths WHERE path = ?', (path,))
conn.commit()
conn.close()
def clear_recent_paths():
conn = _get_conn()
conn.execute('DELETE FROM recent_paths')
conn.commit()
conn.close()
# ===== 通用 CRUD数据管理 =====
ALLOWED_TABLES = {
'api_calls': {
'columns': ['endpoint', 'method', 'status_code', 'duration_ms', 'called_at'],
'searchable': ['endpoint', 'method'],
},
'search_history': {
'columns': ['keyword', 'root_path', 'result_count', 'searched_at'],
'searchable': ['keyword', 'root_path'],
},
'recent_paths': {
'columns': ['path', 'last_used'],
'searchable': ['path'],
},
}
DISPLAY_COLUMNS = {
'api_calls': 'id, endpoint, method, status_code, duration_ms, strftime("%Y-%m-%d %H:%M:%S", called_at, "+8 hours") as called_at',
'search_history': 'id, keyword, root_path, result_count, strftime("%Y-%m-%d %H:%M", searched_at, "+8 hours") as searched_at',
'recent_paths': 'id, path, strftime("%Y-%m-%d %H:%M", last_used, "+8 hours") as last_used',
}
def get_table_info():
conn = _get_conn()
result = []
for table in ALLOWED_TABLES:
row = conn.execute(f'SELECT COUNT(*) as cnt FROM {table}').fetchone()
result.append({'table': table, 'count': row['cnt']})
conn.close()
return result
def get_table_rows(table, page=1, page_size=20, search=''):
if table not in ALLOWED_TABLES:
return []
cols = DISPLAY_COLUMNS[table]
offset = (page - 1) * page_size
conn = _get_conn()
if search:
searchable = ALLOWED_TABLES[table]['searchable']
conditions = ' OR '.join(f'{col} LIKE ?' for col in searchable)
params = [f'%{search}%'] * len(searchable) + [page_size, offset]
rows = conn.execute(
f'SELECT {cols} FROM {table} WHERE {conditions} ORDER BY id DESC LIMIT ? OFFSET ?',
params
).fetchall()
else:
rows = conn.execute(
f'SELECT {cols} FROM {table} ORDER BY id DESC LIMIT ? OFFSET ?',
(page_size, offset)
).fetchall()
conn.close()
return [dict(r) for r in rows]
def get_table_count(table, search=''):
if table not in ALLOWED_TABLES:
return 0
conn = _get_conn()
if search:
searchable = ALLOWED_TABLES[table]['searchable']
conditions = ' OR '.join(f'{col} LIKE ?' for col in searchable)
params = [f'%{search}%'] * len(searchable)
row = conn.execute(f'SELECT COUNT(*) as cnt FROM {table} WHERE {conditions}', params).fetchone()
else:
row = conn.execute(f'SELECT COUNT(*) as cnt FROM {table}').fetchone()
conn.close()
return row['cnt']
def insert_row(table, data):
if table not in ALLOWED_TABLES:
return None
allowed_cols = ALLOWED_TABLES[table]['columns']
cols = []
vals = []
for col in allowed_cols:
if col in data and data[col] != '':
cols.append(col)
vals.append(data[col])
if not cols:
return None
placeholders = ','.join('?' for _ in cols)
col_str = ','.join(cols)
conn = _get_conn()
cur = conn.execute(f'INSERT INTO {table} ({col_str}) VALUES ({placeholders})', vals)
conn.commit()
new_id = cur.lastrowid
conn.close()
return new_id
def update_row(table, row_id, data):
if table not in ALLOWED_TABLES:
return False
allowed_cols = ALLOWED_TABLES[table]['columns']
sets = []
vals = []
for col in allowed_cols:
if col in data:
sets.append(f'{col} = ?')
vals.append(data[col])
if not sets:
return False
vals.append(row_id)
conn = _get_conn()
conn.execute(f'UPDATE {table} SET {",".join(sets)} WHERE id = ?', vals)
conn.commit()
conn.close()
return True
def delete_rows(table, ids):
if table not in ALLOWED_TABLES or not ids:
return 0
conn = _get_conn()
placeholders = ','.join('?' for _ in ids)
cur = conn.execute(f'DELETE FROM {table} WHERE id IN ({placeholders})', ids)
conn.commit()
deleted = cur.rowcount
conn.close()
return deleted
def clear_table(table):
if table not in ALLOWED_TABLES:
return 0
conn = _get_conn()
cur = conn.execute(f'DELETE FROM {table}')
conn.commit()
deleted = cur.rowcount
conn.close()
return deleted
# ===== 文章标签设置 =====
DEFAULT_CONTENT_TAG_SETTINGS = {
'system_msg': "你是一个专门生成文章标签的助手,请你根据我给你的文章的内容总结并生成一系列的标签,格式可以参考[关键词1, 关键词2, 关键词3].你只需要给我生成这种形式的标签即可,其他分析内容无需输出.",
'user_msg': "请严格按照以下要求,从提供的文章内容中提取关键词。\n\n文章内容:\n{content}\n\n要求:\n- 提取最多 {max_tags} 个最能概括文章主旨和核心概念的关键词。\n- 关键词必须来源于文章内容,准确反映文章主题。\n- 每个关键词的长度必须在 {min_length}{max_length} 个字符之间。\n- 输出格式为关键词1, 关键词2, 关键词3, ...\n- 只输出关键词列表,不要有任何其他解释或前缀。",
'max_tags': 10,
'min_length': 2,
'max_length': 6,
}
def get_content_tag_settings():
conn = _get_conn()
row = conn.execute('SELECT * FROM content_tag_settings WHERE id = 1').fetchone()
conn.close()
if not row:
return DEFAULT_CONTENT_TAG_SETTINGS
return {
'system_msg': row['system_msg'] or DEFAULT_CONTENT_TAG_SETTINGS['system_msg'],
'user_msg': row['user_msg'] or DEFAULT_CONTENT_TAG_SETTINGS['user_msg'],
'max_tags': row['max_tags'] if row['max_tags'] is not None else DEFAULT_CONTENT_TAG_SETTINGS['max_tags'],
'min_length': row['min_length'] if row['min_length'] is not None else DEFAULT_CONTENT_TAG_SETTINGS['min_length'],
'max_length': row['max_length'] if row['max_length'] is not None else DEFAULT_CONTENT_TAG_SETTINGS['max_length'],
}
def update_content_tag_settings(data):
allowed = ['system_msg', 'user_msg', 'max_tags', 'min_length', 'max_length']
sets = []
vals = []
for col in allowed:
if col in data:
sets.append(f'{col} = ?')
vals.append(data[col])
if not sets:
return False
conn = _get_conn()
conn.execute(f'UPDATE content_tag_settings SET {", ".join(sets)} WHERE id = 1', vals)
conn.commit()
conn.close()
return True