openmcp 完成闭环

renderer/public/iconfont.css

@@ -1,8 +1,8 @@
 @font-face {
   font-family: "iconfont"; /* Project id 4870215 */
-  src: url('iconfont.woff2?t=1745735110196') format('woff2'),
+  src: url('iconfont.woff2?t=1745774700883') format('woff2'),
-       url('iconfont.woff?t=1745735110196') format('woff'),
+       url('iconfont.woff?t=1745774700883') format('woff'),
-       url('iconfont.ttf?t=1745735110196') format('truetype');
+       url('iconfont.ttf?t=1745774700883') format('truetype');
 }
 
 .iconfont {
@@ -13,6 +13,18 @@
   -moz-osx-font-smoothing: grayscale;
 }
 
+.icon-video:before {
+  content: "\e865";
+}
+
+.icon-image:before {
+  content: "\ebc7";
+}
+
+.icon-audio:before {
+  content: "\e768";
+}
+
 .icon-error:before {
   content: "\e6c6";
 }

renderer/src/App.vue

@@ -29,8 +29,8 @@ bridge.addCommandListener('hello', data => {
 
 
 function initDebug() {
-	connectionArgs.commandString = 'node /Users/bytedance/projects/mcp/servers/src/puppeteer/dist/index.js';
+	// connectionArgs.commandString = 'node /Users/bytedance/projects/mcp/servers/src/puppeteer/dist/index.js';
-	// connectionArgs.commandString = 'node C:/Users/K/code/servers/src/puppeteer/dist/index.js';
+	connectionArgs.commandString = 'node C:/Users/K/code/servers/src/puppeteer/dist/index.js';
 	// connectionArgs.commandString = 'uv run mcp run bing-picture.py';
 	connectionArgs.cwd = '../servers';
 	connectionMethods.current = 'STDIO';

renderer/src/api/message-bridge.ts

@@ -123,6 +123,7 @@ class MessageBridge {
 		if (!this.handlers.has(command)) {
 			this.handlers.set(command, new Set<CommandHandler>());
 		}
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
 		const commandHandlers = this.handlers.get(command)!;
 
 		const wrapperCommandHandler = option.once ? (data: any) => {

renderer/src/components/main-panel/chat/message/toolcall-result-item.vue

@@ -5,12 +5,18 @@
         </div>
 
         <div v-else-if="props.item.type === 'image'" class="tool-image">
-            #{{ props.item.data }}
+            <div class="media-item">
+                <img :src="thumbnail" alt="screenshot"/>
+                <span class="float-container">
+                    <span class="iconfont icon-image"></span>
+                </span>
+            </div>
+
             <span v-if="!finishProcess">
                 <el-progress
+                    class="progress"
                     :percentage="progress"
-                    :stroke-width="2"
+                    :stroke-width="3"
-                    :show-text="false"
                 >
                     <template #default="{ percentage }">
                         <span class="percentage-label">{{ progressText }}</span>
@@ -27,12 +33,11 @@
 <script setup lang="ts">
 import { useMessageBridge } from '@/api/message-bridge';
 import { ToolCallContent } from '@/hook/type';
+import { getBlobUrlByFilename } from '@/hook/util';
 import { defineComponent, PropType, defineProps, ref, defineEmits } from 'vue';
-import { tabs } from '../../panel';
-import { IRenderMessage } from '../chat';
 
 defineComponent({ name: 'toolcall-result-item' });
-const emit = defineEmits(['update:item']);
+const emit = defineEmits(['update:item', 'update:ocr-done']);
 
 const props = defineProps({
     item: {
@@ -46,7 +51,7 @@ const { ocr = false, workerId = '' } = metaInfo;
 
 // 确认当前已经完成，如果没有完成，说明
 const progress = ref(0);
-const progressText = ref('');
+const progressText = ref('OCR');
 const finishProcess = ref(true);
 
 if (ocr) {
@@ -57,7 +62,7 @@ if (ocr) {
         const { id, progress: p = 1.0, status = 'finish' } = data;
         if (id === workerId) {
             progressText.value = status;
-            progress.value = Math.min(Math.max(p * 100 ,0), 100);
+            progress.value = Math.min(Math.ceil(Math.max(p * 100 ,0)), 100);
         }
     }, { once: false });
 
@@ -70,9 +75,26 @@ if (ocr) {
             emit('update:item', { ...rest });
         }
 
+        emit('update:ocr-done');
+
         cancel();
     }, { once: true });
 }
+
+
+const thumbnail = ref('');
+
+if (props.item.data) {
+    console.log(props.item.data);
+    getBlobUrlByFilename(props.item.data).then(url => {
+        console.log(url);
+        
+        if (url) {
+            thumbnail.value = url;
+        }
+    });
+}
+
 </script>
 
 <style>
@@ -80,10 +102,63 @@ if (ocr) {
     position: relative;
 }
 
-.el-progress {
+.tool-image .progress {
-    position: absolute;
+    margin-top: 10px;
-    bottom: 0;
-    left: 0;
-    right: 0;
 }
+
+.percentage-label {
+    margin-right: 10px;
+}
+
+.tool-image .media-item {
+    position: relative;
+    width: 100px;
+    height: 100px;
+    background-color: var(--sidebar);
+    border-radius: .5em;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+}
+
+.tool-image .media-item .iconfont {
+    font-size: 40px;
+}
+
+.tool-image .media-item {
+    object-fit: cover;
+    overflow: hidden;
+}
+
+.tool-image .media-item > img {
+    position: absolute;
+    top: 50%;
+    height: 100%;
+    width: 100%;
+    object-fit: cover;
+    transform: translateY(-50%);
+}
+
+.tool-image .media-item .float-container {
+    position: absolute;
+    left: 0;
+    top: 0;
+    width: 100px;
+    height: 100px;
+    background-color: rgba(0, 0, 0, 0.5);
+    opacity: 1;
+    display: flex;
+    justify-content: center;
+    align-items: center;
+    transition: var(--animation-3s);
+}
+
+.tool-image .media-item .float-container .iconfont {
+    color: var(--background);
+}
+
+.tool-image .media-item:hover .float-container {
+    opacity: 1;
+}
+
 </style>

renderer/src/components/main-panel/chat/message/toolcall.vue

@@ -68,6 +68,7 @@
                                     <ToolcallResultItem
                                         :item="item"
                                         @update:item="value => updateToolCallResultItem(value, index)"
+                                        @update:ocr-done="value => collposePanel()"
                                     />
                                 </div>
                             </span>
@@ -113,19 +114,38 @@ const props = defineProps({
     }
 });
 
+const hasOcr = computed(() => {
+    for (const item of props.message.toolResult || []) {
+        const metaInfo = item._meta || {};
+        const { ocr = false } = metaInfo;
+        if (ocr) {
+            return true;
+        }
+    }
+    return false;
+});
+
 const activeNames = ref<string[]>(props.message.toolResult ? [''] : ['tool']);
 
 watch(
     () => props.message.toolResult,
-    (value, oldValue) => {
+    (value, _) => {
+        if (hasOcr.value) {
+            return;
+        }
+
         if (value) {
-            setTimeout(() => {
+            collposePanel();
-                activeNames.value = [''];
-            }, 1000);
         }
     }
 );
 
+function collposePanel() {
+    setTimeout(() => {
+        activeNames.value = [''];
+    }, 1000);
+}
+
 /**
  * @description 将工具调用结果转换成 html
  * @param toolResult 

renderer/src/components/main-panel/chat/task-loop.ts

@@ -68,7 +68,10 @@ export class TaskLoop {
                 console.log(toolResponse);
                 
                 return {
-                    content: toolResponse,
+                    content: [{
+                        type: 'error',
+                        text: toolResponse
+                    }],
                     state: MessageState.ToolCall
                 }
             } else if (!toolResponse.isError) {
@@ -314,7 +317,7 @@ export class TaskLoop {
                 if (toolCallResult.state === MessageState.ParseJsonError) {
                     // 如果是因为解析 JSON 错误，则重新开始
                     tabStorage.messages.pop();
-                    redLog('解析 JSON 错误 ' + this.streamingToolCalls.value[0].function.arguments);
+                    redLog('解析 JSON 错误 ' + this.streamingToolCalls.value[0]?.function?.arguments);
                     continue;
                 }
 

renderer/src/components/main-panel/tool/tools.ts

@@ -36,10 +36,7 @@ export function callTool(toolName: string, toolArgs: Record<string, any>) {
             command: 'tools/call',
             data: {
                 toolName,
-                toolArgs: JSON.parse(JSON.stringify(toolArgs, (key, value) => {
+                toolArgs: JSON.parse(JSON.stringify(toolArgs))
-                    // 确保所有值都保持原始字符串形式
-                    return typeof value === 'number' ? String(value) : value;
-                }))
             }
         });
     });

renderer/src/hook/util.ts

@@ -1,3 +1,5 @@
+import { useMessageBridge } from "@/api/message-bridge";
+
 export function getCurrentTime() {
     // 创建一个Date对象
     const date = new Date();
@@ -20,3 +22,70 @@ export function getCurrentTime() {
     const timeStr = year + "年" + month + "月" + day + "日" + " " + hour + ":" + minute;
     return timeStr;
 }
+
+
+export function getBase64StringByFilename(filename: string) {
+    const bridge = useMessageBridge();
+
+    return new Promise<string>(resolve => {
+        bridge.addCommandListener('ocr/get-ocr-image', data => {
+            const { code, msg = {} } = data;
+            resolve(msg.base64String);
+        }, { once: true});
+
+        bridge.postMessage({
+            command: 'ocr/get-ocr-image',
+            data: {
+                filename
+            }
+        });
+    });
+}
+
+const blobUrlCache = new Map<string, string>();
+
+export async function getBlobUrlByFilename(filename: string) {
+    // 检查缓存中是否存在该文件
+    if (blobUrlCache.has(filename)) {
+        return blobUrlCache.get(filename);
+    }
+
+    const base64String = await getBase64StringByFilename(filename);        
+
+    if (!base64String) {
+        return '';
+    }
+
+    // 根据文件后缀获取 mimeType
+    const extension = filename.split('.').pop()?.toLowerCase();
+    let mimeType = 'image/png'; // 默认值
+    switch (extension) {
+        case 'jpg':
+        case 'jpeg':
+            mimeType = 'image/jpeg';
+            break;
+        case 'gif':
+            mimeType = 'image/gif';
+            break;
+        case 'webp':
+            mimeType = 'image/webp';
+            break;
+        case 'bmp':
+            mimeType = 'image/bmp';
+            break;
+        case 'svg':
+            mimeType = 'image/svg+xml';
+            break;
+    }
+    const byteCharacters = atob(base64String);
+    const byteNumbers = new Array(byteCharacters.length);
+    for (let i = 0; i < byteCharacters.length; i++) {
+        byteNumbers[i] = byteCharacters.charCodeAt(i);
+    }
+    const byteArray = new Uint8Array(byteNumbers);
+    const blob = new Blob([byteArray], { type: mimeType });
+    const blobUrl = URL.createObjectURL(blob);
+    // 将结果存入缓存
+    blobUrlCache.set(filename, blobUrl);
+    return blobUrl;
+}

service/src/common/router.ts

@@ -4,6 +4,7 @@ import { LlmController } from "../llm/llm.controller";
 import { ClientController } from "../mcp/client.controller";
 import { ConnectController } from "../mcp/connect.controller";
 import { client } from "../mcp/connect.service";
+import { OcrController } from "../mcp/ocr.controller";
 import { PanelController } from "../panel/panel.controller";
 import { SettingController } from "../setting/setting.controller";
 
@@ -12,7 +13,8 @@ export const ModuleControllers = [
     ClientController,
     LlmController,
     PanelController,
-    SettingController
+    SettingController,
+    OcrController
 ];
 
 export async function routeMessage(command: string, data: any, webview: PostMessageble) {

service/src/mcp/client.controller.ts

@@ -132,11 +132,11 @@ export class ClientController {
             arguments: option.toolArgs
         });
 
-        console.log(JSON.stringify(toolResult, null, 2));
+        // console.log(JSON.stringify(toolResult, null, 2));
         
         postProcessMcpToolcallResponse(toolResult, webview);
 
-        console.log(JSON.stringify(toolResult, null, 2));
+        // console.log(JSON.stringify(toolResult, null, 2));
         
 
         return {

service/src/mcp/ocr.controller.ts

@@ -0,0 +1,18 @@
+import { Controller, RequestClientType } from "../common";
+import { PostMessageble } from "../hook/adapter";
+import { diskStorage } from "../hook/db";
+
+export class OcrController {
+    @Controller('ocr/get-ocr-image')
+    async getOcrImage(client: RequestClientType, data: any, webview: PostMessageble) {
+        const { filename } = data;
+        const buffer = diskStorage.getSync(filename);
+        const base64String = buffer ? buffer.toString('base64'): undefined;
+        return {
+            code: 200,
+            msg: {
+                base64String
+            }
+        }
+    }
+}

service/src/mcp/ocr.service.ts

@@ -77,6 +77,8 @@ export function createOcrWorker(filename: string, webview: PostMessageble): OcrW
     };
 
     const imagePath = diskStorage.getStoragePath(filename);
+    console.log(imagePath);
+    
     const fut = tesseractOCR(imagePath, logger);
 
     fut.then((text) => {