| # File created with: OpenVINO Model Server 2025.4.0.8db21b81 | |
| input_stream: "HTTP_REQUEST_PAYLOAD:input" | |
| output_stream: "HTTP_RESPONSE_PAYLOAD:output" | |
| node: { | |
| name: "LLMExecutor" | |
| calculator: "HttpLLMCalculator" | |
| input_stream: "LOOPBACK:loopback" | |
| input_stream: "HTTP_REQUEST_PAYLOAD:input" | |
| input_side_packet: "LLM_NODE_RESOURCES:llm" | |
| output_stream: "LOOPBACK:loopback" | |
| output_stream: "HTTP_RESPONSE_PAYLOAD:output" | |
| input_stream_info: { | |
| tag_index: 'LOOPBACK:0', | |
| back_edge: true | |
| } | |
| node_options: { | |
| [type.googleapis.com / mediapipe.LLMCalculatorOptions]: { | |
| max_num_seqs:256, | |
| device: "NPU", | |
| models_path: "./", | |
| plugin_config: '{"NPUW_LLM_ENABLE_PREFIX_CACHING":true}', | |
| enable_prefix_caching: true, | |
| cache_size: 2, | |
| } | |
| } | |
| input_stream_handler { | |
| input_stream_handler: "SyncSetInputStreamHandler", | |
| options { | |
| [mediapipe.SyncSetInputStreamHandlerOptions.ext] { | |
| sync_set { | |
| tag_index: "LOOPBACK:0" | |
| } | |
| } | |
| } | |
| } | |
| } | |