{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import zipfile\n",
    "from typing import *\n",
    "import os\n",
    "import json\n",
    "import yaml\n",
    "\n",
    "log_home = '../logs'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_log_file_line(file: str, level='DEBUG') -> Generator:\n",
    "    if file.endswith('.log'):\n",
    "        for line in open(file, 'r', encoding='utf-8'):\n",
    "            if level in line:\n",
    "                yield line\n",
    "    elif file.endswith('.zip'):\n",
    "        with zipfile.ZipFile(file, 'r') as zip:\n",
    "            for zip_file in zip.namelist():\n",
    "                file_bytes = zip.read(zip_file)\n",
    "                for line in file_bytes.decode('utf-8').split('\\n'):\n",
    "                    if level in line:\n",
    "                        yield line\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "rag_log_files = [os.path.join(log_home, file) for file in os.listdir(log_home) if file.startswith('rag')]\n",
    "\n",
    "interesting_data = []\n",
    "\n",
    "for rag_file in rag_log_files:\n",
    "    for line in get_log_file_line(rag_file, level='DEBUG'):\n",
    "        try:\n",
    "            data = line.split('|')[-1].strip()\n",
    "            data = eval(data)\n",
    "            if data['intent']['name'] != 'others':\n",
    "                interesting_data.append(data)\n",
    "        except Exception:\n",
    "            pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "story = {'stories': []}\n",
    "for d in interesting_data:\n",
    "    story['stories'].append({\n",
    "        'message': d['query'],\n",
    "        'intent': d['intent']['name']\n",
    "    })\n",
    "\n",
    "with open('../config/qq.story.yml', 'w', encoding='utf-8') as fp:\n",
    "    yaml.dump(story, fp, allow_unicode=True)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}