100 lines
2.6 KiB
Plaintext
100 lines
2.6 KiB
Plaintext
{
|
|
"cells": [
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 6,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"import zipfile\n",
|
|
"from typing import *\n",
|
|
"import os\n",
|
|
"import json\n",
|
|
"import yaml\n",
|
|
"\n",
|
|
"log_home = '../logs'"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 3,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"def get_log_file_line(file: str, level='DEBUG') -> Generator:\n",
|
|
" if file.endswith('.log'):\n",
|
|
" for line in open(file, 'r', encoding='utf-8'):\n",
|
|
" if level in line:\n",
|
|
" yield line\n",
|
|
" elif file.endswith('.zip'):\n",
|
|
" with zipfile.ZipFile(file, 'r') as zip:\n",
|
|
" for zip_file in zip.namelist():\n",
|
|
" file_bytes = zip.read(zip_file)\n",
|
|
" for line in file_bytes.decode('utf-8').split('\\n'):\n",
|
|
" if level in line:\n",
|
|
" yield line\n",
|
|
" "
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 8,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"rag_log_files = [os.path.join(log_home, file) for file in os.listdir(log_home) if file.startswith('rag')]\n",
|
|
"\n",
|
|
"interesting_data = []\n",
|
|
"\n",
|
|
"for rag_file in rag_log_files:\n",
|
|
" for line in get_log_file_line(rag_file, level='DEBUG'):\n",
|
|
" try:\n",
|
|
" data = line.split('|')[-1].strip()\n",
|
|
" data = eval(data)\n",
|
|
" if data['intent']['name'] != 'others':\n",
|
|
" interesting_data.append(data)\n",
|
|
" except Exception:\n",
|
|
" pass"
|
|
]
|
|
},
|
|
{
|
|
"cell_type": "code",
|
|
"execution_count": 10,
|
|
"metadata": {},
|
|
"outputs": [],
|
|
"source": [
|
|
"story = {'stories': []}\n",
|
|
"for d in interesting_data:\n",
|
|
" story['stories'].append({\n",
|
|
" 'message': d['query'],\n",
|
|
" 'intent': d['intent']['name']\n",
|
|
" })\n",
|
|
"\n",
|
|
"with open('../config/qq.story.yml', 'w', encoding='utf-8') as fp:\n",
|
|
" yaml.dump(story, fp, allow_unicode=True)"
|
|
]
|
|
}
|
|
],
|
|
"metadata": {
|
|
"kernelspec": {
|
|
"display_name": "base",
|
|
"language": "python",
|
|
"name": "python3"
|
|
},
|
|
"language_info": {
|
|
"codemirror_mode": {
|
|
"name": "ipython",
|
|
"version": 3
|
|
},
|
|
"file_extension": ".py",
|
|
"mimetype": "text/x-python",
|
|
"name": "python",
|
|
"nbconvert_exporter": "python",
|
|
"pygments_lexer": "ipython3",
|
|
"version": "3.11.5"
|
|
}
|
|
},
|
|
"nbformat": 4,
|
|
"nbformat_minor": 2
|
|
}
|