mortadhabbb commited on
Commit
e3fae6c
·
1 Parent(s): 758929a

Update chatbot microservice orchestration

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .dockerignore +0 -2
  2. .gitignore +5 -1
  3. ARCHITECTURE_GUIDE.md +0 -2
  4. CHATBOT_20_ALGORITHM_VALIDATION.md +3 -0
  5. CHATBOT_MICROSERVICE_REFACTOR_AUDIT.md +6 -0
  6. CODE_EVALUATION_72_100.md +0 -2
  7. DELIVERABLES_CHECKLIST.md +0 -2
  8. DOCUMENTATION_INDEX.md +0 -2
  9. Dockerfile +1 -3
  10. EXECUTIVE_SUMMARY.md +0 -2
  11. HUGGING_FACE_SPACE_TRAINING.md +1 -4
  12. MODEL_OPS_AND_APP_SPLIT.md +4 -6
  13. PRODUCTION_SETUP_GUIDE.md +0 -2
  14. README.md +2 -2
  15. README_v2.md +0 -2
  16. REBUILD_SUMMARY.md +0 -2
  17. START_HERE.md +0 -2
  18. VALIDATION_REPORT_100_100.md +0 -2
  19. apps/__init__.py +1 -0
  20. apps/chat/__init__.py +1 -0
  21. apps/chat/admin.py +0 -2
  22. apps/chat/migrations/0001_initial.py +0 -2
  23. apps/chat/migrations/0002_predefinedresponse_runtime_fields.py +0 -2
  24. apps/chat/migrations/__init__.py +1 -0
  25. apps/chat/models.py +0 -2
  26. apps/chat/services/{N8N_INTEGRATION_README.md → INTERNAL_ORCHESTRATION_README.md} +32 -34
  27. apps/chat/services/RunData.py +0 -2
  28. apps/chat/services/RunMarkdowns.py +0 -2
  29. apps/chat/services/RunModel.py +0 -2
  30. apps/chat/services/__init__.py +1 -0
  31. apps/chat/services/ai_model.py +0 -2
  32. apps/chat/services/api_auth.py +0 -2
  33. apps/chat/services/backend_orchestration.py +111 -0
  34. apps/chat/services/bot.py +1 -3
  35. apps/chat/services/data_registry.py +0 -2
  36. apps/chat/services/helpers/ClassifierHelper.py +0 -2
  37. apps/chat/services/helpers/GetBestAnser.py +0 -2
  38. apps/chat/services/helpers/GetDataFromGoogle.py +0 -2
  39. apps/chat/services/helpers/GetPredefinedResponse.py +0 -2
  40. apps/chat/services/helpers/MarkdownDecisionEngine.py +0 -2
  41. apps/chat/services/helpers/TranslationHelper.py +0 -2
  42. apps/chat/services/helpers/__init__.py +1 -0
  43. apps/chat/services/helpers/intent_rules.py +0 -2
  44. apps/chat/services/helpers/predefined.py +0 -2
  45. apps/chat/services/helpers/qa_loader.py +0 -2
  46. apps/chat/services/intent.py +0 -2
  47. apps/chat/services/language_processor.py +0 -2
  48. apps/chat/services/markdowns_registry.py +0 -2
  49. apps/chat/services/ml_index.py +0 -2
  50. apps/chat/services/model_registry.py +0 -2
.dockerignore CHANGED
@@ -14,5 +14,3 @@ db.sqlite3
14
  artifacts/service_intent_model/checkpoint-*/
15
  artifacts/service_intent_model_remote_cpu*/
16
  artifacts/hf_base_models/
17
-
18
-
 
14
  artifacts/service_intent_model/checkpoint-*/
15
  artifacts/service_intent_model_remote_cpu*/
16
  artifacts/hf_base_models/
 
 
.gitignore CHANGED
@@ -10,4 +10,8 @@ env_py10/
10
  staticfiles/
11
  *.json
12
 
13
-
 
 
 
 
 
10
  staticfiles/
11
  *.json
12
 
13
+ all_chatbot_codes.txt
14
+ all_backend_codes.txt
15
+ chatbot.zip
16
+ src.zip
17
+ .pytest_cache/
ARCHITECTURE_GUIDE.md CHANGED
@@ -660,5 +660,3 @@ Required in production:
660
  ---
661
 
662
  **Questions?** Check the logs! 🔍
663
-
664
-
 
660
  ---
661
 
662
  **Questions?** Check the logs! 🔍
 
 
CHATBOT_20_ALGORITHM_VALIDATION.md ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ # Chatbot Microservice 20-Algorithm Validation
2
+
3
+ The microservice now performs decision routing, service detection/extraction, knowledge QA routing, and service-result finalization. n8n execution remains backend-only.
CHATBOT_MICROSERVICE_REFACTOR_AUDIT.md ADDED
@@ -0,0 +1,6 @@
 
 
 
 
 
 
 
1
+ # Chatbot Microservice Refactor Audit
2
+
3
+ - `/api/chat` returns `KNOWLEDGE_QA`, `SERVICE_REQUEST`, or `CLARIFICATION_REQUIRED`.
4
+ - n8n is no longer executed by the chatbot microservice.
5
+ - `/api/chat/finalize-service-response` formats backend execution results.
6
+ - `/api/knowledge` endpoints manage chatbot-owned `data.csv`.
CODE_EVALUATION_72_100.md CHANGED
@@ -651,5 +651,3 @@ This document includes:
651
  **Evaluation Completed:** May 8, 2026
652
  **Evaluator:** Code Quality Analysis System
653
  **Project:** Django ML Chatbot
654
-
655
-
 
651
  **Evaluation Completed:** May 8, 2026
652
  **Evaluator:** Code Quality Analysis System
653
  **Project:** Django ML Chatbot
 
 
DELIVERABLES_CHECKLIST.md CHANGED
@@ -395,5 +395,3 @@ gunicorn chatbot.wsgi --workers=4
395
 
396
  **Date:** May 8, 2026
397
  **🚀 READY TO DEPLOY!**
398
-
399
-
 
395
 
396
  **Date:** May 8, 2026
397
  **🚀 READY TO DEPLOY!**
 
 
DOCUMENTATION_INDEX.md CHANGED
@@ -358,5 +358,3 @@ All documented in [ARCHITECTURE_GUIDE.md](./ARCHITECTURE_GUIDE.md)
358
  **Start with [README_v2.md](./README_v2.md) 👉**
359
 
360
  🚀 **Ready to deploy!**
361
-
362
-
 
358
  **Start with [README_v2.md](./README_v2.md) 👉**
359
 
360
  🚀 **Ready to deploy!**
 
 
Dockerfile CHANGED
@@ -6,7 +6,7 @@ ENV DJANGO_DEBUG=0
6
  ENV DJANGO_ALLOWED_HOSTS=.hf.space,localhost,127.0.0.1
7
  ENV DJANGO_CACHE_BACKEND=locmem
8
  ENV SERVICE_INTENT_MODEL_DIR=/app/artifacts/service_intent_model
9
- ENV N8N_BASE_URL=https://mortadhabbb-n8n-service-orchestration.hf.space
10
 
11
  WORKDIR /app
12
 
@@ -29,5 +29,3 @@ RUN chmod +x /app/docker-entrypoint.sh
29
  EXPOSE 7860
30
 
31
  CMD ["/app/docker-entrypoint.sh"]
32
-
33
-
 
6
  ENV DJANGO_ALLOWED_HOSTS=.hf.space,localhost,127.0.0.1
7
  ENV DJANGO_CACHE_BACKEND=locmem
8
  ENV SERVICE_INTENT_MODEL_DIR=/app/artifacts/service_intent_model
9
+ ENV internal orchestration_BASE_URL=https://mortadhabbb-internal-service-orchestration.hf.space
10
 
11
  WORKDIR /app
12
 
 
29
  EXPOSE 7860
30
 
31
  CMD ["/app/docker-entrypoint.sh"]
 
 
EXECUTIVE_SUMMARY.md CHANGED
@@ -374,5 +374,3 @@ The chatbot is **fully rebuilt, thoroughly tested, extensively documented, and p
374
  **Date:** May 8, 2026
375
 
376
  🚀 **Ready to deploy!**
377
-
378
-
 
374
  **Date:** May 8, 2026
375
 
376
  🚀 **Ready to deploy!**
 
 
HUGGING_FACE_SPACE_TRAINING.md CHANGED
@@ -1,6 +1,6 @@
1
  # Hugging Face Space Training
2
 
3
- This project keeps n8n as an external orchestration microservice. Model training
4
  is handled by a separate Hugging Face Docker Space:
5
 
6
  ```text
@@ -82,6 +82,3 @@ python manage.py submit_remote_service_intent_training --model-id google/flan-t5
82
  ```
83
 
84
  Use `google/flan-t5-base` when the Space has enough CPU/GPU memory and time.
85
-
86
-
87
-
 
1
  # Hugging Face Space Training
2
 
3
+ This project keeps internal orchestration as an external orchestration microservice. Model training
4
  is handled by a separate Hugging Face Docker Space:
5
 
6
  ```text
 
82
  ```
83
 
84
  Use `google/flan-t5-base` when the Space has enough CPU/GPU memory and time.
 
 
 
MODEL_OPS_AND_APP_SPLIT.md CHANGED
@@ -1,6 +1,6 @@
1
  # Chatbot App Split and Service-Intent Model Ops
2
 
3
- The chatbot project now keeps responsibilities separated without moving n8n into Django.
4
 
5
  ## Apps
6
 
@@ -8,7 +8,7 @@ The chatbot project now keeps responsibilities separated without moving n8n into
8
  - Chat API/UI.
9
  - Knowledge-base retrieval.
10
  - Runtime service detection.
11
- - n8n microservice execution client.
12
 
13
  - `apps.service_intents`
14
  - Dataset ownership commands.
@@ -55,8 +55,6 @@ Dry-run without downloading/training:
55
  python manage.py prepare_service_intent_model --dry-run --skip-install
56
  ```
57
 
58
- ## n8n
59
-
60
- n8n remains an external microservice. Django still talks to it through the existing service executor.
61
-
62
 
 
 
1
  # Chatbot App Split and Service-Intent Model Ops
2
 
3
+ The chatbot project now keeps responsibilities separated without moving internal orchestration into Django.
4
 
5
  ## Apps
6
 
 
8
  - Chat API/UI.
9
  - Knowledge-base retrieval.
10
  - Runtime service detection.
11
+ - internal orchestration microservice execution client.
12
 
13
  - `apps.service_intents`
14
  - Dataset ownership commands.
 
55
  python manage.py prepare_service_intent_model --dry-run --skip-install
56
  ```
57
 
58
+ ## internal orchestration
 
 
 
59
 
60
+ internal orchestration remains an external microservice. Django still talks to it through the existing service executor.
PRODUCTION_SETUP_GUIDE.md CHANGED
@@ -550,5 +550,3 @@ For issues:
550
  5. Enable debug: `DJANGO_DEBUG=1`
551
 
552
  **Happy deploying!** 🚀
553
-
554
-
 
550
  5. Enable debug: `DJANGO_DEBUG=1`
551
 
552
  **Happy deploying!** 🚀
 
 
README.md CHANGED
@@ -25,7 +25,7 @@ Basic intent detection
25
 
26
  Service detection before knowledge-base search
27
 
28
- If service detected: execute backend/n8n-compatible service
29
  If no service detected: search CSV knowledge base
30
  ```
31
 
@@ -62,7 +62,7 @@ DJANGO_DEBUG=0
62
  DJANGO_ALLOWED_HOSTS=.hf.space,localhost,127.0.0.1
63
  SERVICE_INTENT_MODEL_DIR=/app/artifacts/service_intent_model
64
  SERVICE_INTENT_TRAINING_SPACE_URL=https://mortadhabbb-train-model-chatbot.hf.space
65
- N8N_BASE_URL=https://mortadhabbb-n8n-service-orchestration.hf.space
66
  ```
67
 
68
  Optional secrets:
 
25
 
26
  Service detection before knowledge-base search
27
 
28
+ If service detected: execute backend/internal orchestration-compatible service
29
  If no service detected: search CSV knowledge base
30
  ```
31
 
 
62
  DJANGO_ALLOWED_HOSTS=.hf.space,localhost,127.0.0.1
63
  SERVICE_INTENT_MODEL_DIR=/app/artifacts/service_intent_model
64
  SERVICE_INTENT_TRAINING_SPACE_URL=https://mortadhabbb-train-model-chatbot.hf.space
65
+ internal orchestration_BASE_URL=https://mortadhabbb-internal-service-orchestration.hf.space
66
  ```
67
 
68
  Optional secrets:
README_v2.md CHANGED
@@ -455,5 +455,3 @@ Start with:
455
  ---
456
 
457
  **Score: 100/100 ✅ | Production Ready 🚀 | Well Tested ✓ | Fully Documented 📚**
458
-
459
-
 
455
  ---
456
 
457
  **Score: 100/100 ✅ | Production Ready 🚀 | Well Tested ✓ | Fully Documented 📚**
 
 
REBUILD_SUMMARY.md CHANGED
@@ -440,5 +440,3 @@ The chatbot has been completely rebuilt from a v1.0 codebase (72/100) to a produ
440
  ---
441
 
442
  **Ready to deploy?** Start with `PRODUCTION_SETUP_GUIDE.md` ✨
443
-
444
-
 
440
  ---
441
 
442
  **Ready to deploy?** Start with `PRODUCTION_SETUP_GUIDE.md` ✨
 
 
START_HERE.md CHANGED
@@ -468,5 +468,3 @@ QUALITY ASSURANCE
468
  **Result:** Enterprise-grade production-ready chatbot
469
 
470
  👉 **[START WITH DOCUMENTATION_INDEX.md](./DOCUMENTATION_INDEX.md)** 👈
471
-
472
-
 
468
  **Result:** Enterprise-grade production-ready chatbot
469
 
470
  👉 **[START WITH DOCUMENTATION_INDEX.md](./DOCUMENTATION_INDEX.md)** 👈
 
 
VALIDATION_REPORT_100_100.md CHANGED
@@ -529,5 +529,3 @@ The chatbot has been successfully rebuilt from v1.0 (72/100) to v2.0 (100/100) w
529
  **Documentation:** Complete
530
 
531
  **Date Completed:** May 8, 2026
532
-
533
-
 
529
  **Documentation:** Complete
530
 
531
  **Date Completed:** May 8, 2026
 
 
apps/__init__.py CHANGED
@@ -0,0 +1 @@
 
 
1
+
apps/chat/__init__.py CHANGED
@@ -0,0 +1 @@
 
 
1
+
apps/chat/admin.py CHANGED
@@ -2,5 +2,3 @@ from django.contrib import admin
2
  from .models import PredefinedResponse
3
 
4
  admin.site.register(PredefinedResponse)
5
-
6
-
 
2
  from .models import PredefinedResponse
3
 
4
  admin.site.register(PredefinedResponse)
 
 
apps/chat/migrations/0001_initial.py CHANGED
@@ -14,5 +14,3 @@ class Migration(migrations.Migration):
14
  ],
15
  ),
16
  ]
17
-
18
-
 
14
  ],
15
  ),
16
  ]
 
 
apps/chat/migrations/0002_predefinedresponse_runtime_fields.py CHANGED
@@ -121,5 +121,3 @@ class Migration(migrations.Migration):
121
  },
122
  ),
123
  ]
124
-
125
-
 
121
  },
122
  ),
123
  ]
 
 
apps/chat/migrations/__init__.py CHANGED
@@ -0,0 +1 @@
 
 
1
+
apps/chat/models.py CHANGED
@@ -154,5 +154,3 @@ class ApiKey(models.Model):
154
  """Update last_used_at timestamp."""
155
  self.last_used_at = timezone.now()
156
  self.save(update_fields=['last_used_at'])
157
-
158
-
 
154
  """Update last_used_at timestamp."""
155
  self.last_used_at = timezone.now()
156
  self.save(update_fields=['last_used_at'])
 
 
apps/chat/services/{N8N_INTEGRATION_README.md → INTERNAL_ORCHESTRATION_README.md} RENAMED
@@ -1,18 +1,18 @@
1
- # Chatbot N8N Service Integration Guide
2
 
3
  ## Overview
4
 
5
- The chatbot now includes automatic service detection for n8n-orchestrated services. When a user message matches service keywords, the chatbot detects the requested service(s) and routes the task to the n8n project for execution.
6
 
7
  ## Current Runtime Flow
8
 
9
  The current implementation uses a safe two-stage service flow:
10
 
11
- 1. `service_intents.csv` plus deterministic service rules decide if the user is asking for a backend/n8n service and which service should run.
12
  2. The trained service-intent model at `artifacts/service_intent_model` extracts structured JSON parameters from the prompt.
13
  3. The chatbot merges model parameters with safe rule/context parameters. Session values like `user_id`, `academy_id`, `sport_id`, and role-specific IDs are protected from model overwrite.
14
- 4. n8n receives the selected service plus `parameters`, `extracted_parameters`, `model_extraction`, and `service_intent`.
15
- 5. n8n executes the backend workflow and returns the service response to the page.
16
 
17
  Runtime model extraction is controlled by:
18
 
@@ -30,7 +30,7 @@ pip install -r requirements-ml.txt
30
 
31
  If `transformers` or `torch` are missing, the chatbot does not crash. It keeps routing services with `service_intents.csv` and deterministic extractors, and marks model extraction as unavailable in the payload.
32
 
33
- Example n8n payload shape for `give me best 10 football players`:
34
 
35
  ```json
36
  {
@@ -79,7 +79,7 @@ User Message
79
  [Chatbot] Service Detection (keyword matching)
80
 
81
  IF service detected:
82
- ├→ [N8N Webhook] Execute Service
83
  └→ Return formatted response
84
 
85
  IF no service detected:
@@ -137,7 +137,7 @@ The chatbot can detect and route the following services:
137
 
138
  ### 1. Service Detection (service_detection.py)
139
  The `ServiceDetector` class:
140
- - Loads services from `n8n-service-orchestration/service-registry/services.json`
141
  - Uses fuzzy string matching (rapidfuzz) to detect services
142
  - Matches user keywords against service registry keywords
143
  - Returns detected service IDs with confidence scores (threshold: 70%)
@@ -151,10 +151,10 @@ The `Chatbot` class now includes a new response tier:
151
 
152
  When a service is detected, the chatbot now returns both the service IDs and structured `service_requests` with extracted parameters such as `query_scope`, `start_date`, `end_date`, `target_date`, `subject_scope`, and `activity_type`.
153
 
154
- ### 3. Service Execution (n8n_service_executor.py)
155
- The `N8NServiceExecutor` class:
156
  - Receives detected services with parameters
157
- - Routes them to n8n webhooks
158
  - Handles execution, timeouts, and errors
159
  - Returns formatted responses
160
 
@@ -162,12 +162,12 @@ The `N8NServiceExecutor` class:
162
 
163
  ### Django Settings (chatbot/settings.py)
164
 
165
- Add n8n configuration:
166
 
167
  ```python
168
- # N8N Service Integration
169
- N8N_BASE_URL = os.getenv('N8N_BASE_URL', 'http://localhost:5678')
170
- N8N_TIMEOUT = int(os.getenv('N8N_TIMEOUT', '10')) # seconds
171
 
172
  # Service Detection Threshold
173
  SERVICE_DETECTION_THRESHOLD = 70.0 # 0-100, percentage confidence
@@ -176,8 +176,8 @@ SERVICE_DETECTION_THRESHOLD = 70.0 # 0-100, percentage confidence
176
  ### Environment Variables (.env)
177
 
178
  ```
179
- N8N_BASE_URL=http://localhost:5678
180
- N8N_TIMEOUT=10
181
  ```
182
 
183
  ## Usage Examples
@@ -187,7 +187,7 @@ N8N_TIMEOUT=10
187
  User: "Show me my player statistics"
188
  → Service detected: get-player-stats
189
  → Confidence: 95%
190
- N8N executes: GET /webhook/get-player-stats
191
  → Response: Player performance card with stats
192
  ```
193
 
@@ -202,7 +202,7 @@ User: "What activities does my child have next week?"
202
  - start_date / end_date: normalized week range
203
  - subject_scope: child
204
  - activity_type: all
205
- N8N executes the activities workflow with the requester identity and date window
206
  ```
207
 
208
  ### Example 2: Multiple Services
@@ -238,7 +238,7 @@ When a service is detected, the chatbot returns:
238
  "response": "I can help you with: Get Player Statistics. Let me route your request to the appropriate service...",
239
  "score": 0.85,
240
  "category": "service_detection",
241
- "source": "n8n_services",
242
  "matched_question": null,
243
  "services": ["get-player-stats"],
244
  "service_details": [
@@ -253,7 +253,7 @@ When a service is detected, the chatbot returns:
253
  }
254
  ```
255
 
256
- The webhook payload sent to N8N also includes top-level identity fields for compatibility with workflows that read them directly:
257
 
258
  ```json
259
  {
@@ -275,7 +275,7 @@ The webhook payload sent to N8N also includes top-level identity fields for comp
275
  The system handles various error scenarios:
276
 
277
  1. **Service not found**: Logs warning, continues to ML search
278
- 2. **N8N timeout**: Returns error message, suggests retrying
279
  3. **Connection error**: Returns service unavailable message
280
  4. **Invalid parameters**: Returns error with hint for valid parameters
281
 
@@ -283,7 +283,7 @@ The system handles various error scenarios:
283
 
284
  To add new services:
285
 
286
- 1. **Update services.json** in `n8n-service-orchestration/service-registry/`:
287
  ```json
288
  {
289
  "id": "new-service-id",
@@ -297,9 +297,9 @@ To add new services:
297
  }
298
  ```
299
 
300
- 2. **Create n8n workflow** at `/workflows/new-service-id.json`
301
 
302
- 3. **Deploy to n8n** and create webhook
303
 
304
  4. **Restart chatbot** to reload service registry
305
 
@@ -319,7 +319,7 @@ grep "Service.*executed" logs/chatbot.log
319
  ## Performance Considerations
320
 
321
  - Service detection: ~5-10ms (fuzzy matching)
322
- - N8N execution: Configurable timeout (default: 10s)
323
  - Service registry: Loaded once at startup, cached in memory
324
 
325
  ## Troubleshooting
@@ -329,10 +329,10 @@ grep "Service.*executed" logs/chatbot.log
329
  - Verify fuzzy match threshold (default 70%)
330
  - Check logs: `grep "SERVICE" logs/chatbot.log`
331
 
332
- ### N8N webhooks not responding
333
- - Verify N8N is running: `curl http://localhost:5678`
334
- - Check N8N_BASE_URL in settings
335
- - Verify webhook paths in services.json match n8n workflows
336
 
337
  ### False positives (wrong service detected)
338
  - Adjust keywords in services.json
@@ -343,8 +343,6 @@ grep "Service.*executed" logs/chatbot.log
343
 
344
  - **Service Detection**: `apps/chat/services/service_detection.py`
345
  - **Chatbot Core**: `apps/chat/services/bot.py`
346
- - **N8N Executor**: `apps/chat/services/n8n_service_executor.py`
347
- - **Service Registry**: `../../n8n-service-orchestration/service-registry/services.json`
348
  - **Test Suite**: `apps/chat/tests/test_service_detection.py`
349
-
350
-
 
1
+ # Chatbot internal orchestration Service Integration Guide
2
 
3
  ## Overview
4
 
5
+ The chatbot now includes automatic service detection for internal orchestration-orchestrated services. When a user message matches service keywords, the chatbot detects the requested service(s) and routes the task to the internal orchestration project for execution.
6
 
7
  ## Current Runtime Flow
8
 
9
  The current implementation uses a safe two-stage service flow:
10
 
11
+ 1. `service_intents.csv` plus deterministic service rules decide if the user is asking for a backend/internal orchestration service and which service should run.
12
  2. The trained service-intent model at `artifacts/service_intent_model` extracts structured JSON parameters from the prompt.
13
  3. The chatbot merges model parameters with safe rule/context parameters. Session values like `user_id`, `academy_id`, `sport_id`, and role-specific IDs are protected from model overwrite.
14
+ 4. internal orchestration receives the selected service plus `parameters`, `extracted_parameters`, `model_extraction`, and `service_intent`.
15
+ 5. internal orchestration executes the backend workflow and returns the service response to the page.
16
 
17
  Runtime model extraction is controlled by:
18
 
 
30
 
31
  If `transformers` or `torch` are missing, the chatbot does not crash. It keeps routing services with `service_intents.csv` and deterministic extractors, and marks model extraction as unavailable in the payload.
32
 
33
+ Example internal orchestration payload shape for `give me best 10 football players`:
34
 
35
  ```json
36
  {
 
79
  [Chatbot] Service Detection (keyword matching)
80
 
81
  IF service detected:
82
+ ├→ [internal orchestration Webhook] Execute Service
83
  └→ Return formatted response
84
 
85
  IF no service detected:
 
137
 
138
  ### 1. Service Detection (service_detection.py)
139
  The `ServiceDetector` class:
140
+ - Loads services from `internal-service-orchestration/service-registry/services.json`
141
  - Uses fuzzy string matching (rapidfuzz) to detect services
142
  - Matches user keywords against service registry keywords
143
  - Returns detected service IDs with confidence scores (threshold: 70%)
 
151
 
152
  When a service is detected, the chatbot now returns both the service IDs and structured `service_requests` with extracted parameters such as `query_scope`, `start_date`, `end_date`, `target_date`, `subject_scope`, and `activity_type`.
153
 
154
+ ### 3. Service Execution (internal orchestration_service_executor.py)
155
+ The `InternalServiceExecutor` class:
156
  - Receives detected services with parameters
157
+ - Routes them to internal orchestration webhooks
158
  - Handles execution, timeouts, and errors
159
  - Returns formatted responses
160
 
 
162
 
163
  ### Django Settings (chatbot/settings.py)
164
 
165
+ Add internal orchestration configuration:
166
 
167
  ```python
168
+ # internal orchestration Service Integration
169
+ internal orchestration_BASE_URL = os.getenv('internal orchestration_BASE_URL', 'http://localhost:5678')
170
+ internal orchestration_TIMEOUT = int(os.getenv('internal orchestration_TIMEOUT', '10')) # seconds
171
 
172
  # Service Detection Threshold
173
  SERVICE_DETECTION_THRESHOLD = 70.0 # 0-100, percentage confidence
 
176
  ### Environment Variables (.env)
177
 
178
  ```
179
+ internal orchestration_BASE_URL=http://localhost:5678
180
+ internal orchestration_TIMEOUT=10
181
  ```
182
 
183
  ## Usage Examples
 
187
  User: "Show me my player statistics"
188
  → Service detected: get-player-stats
189
  → Confidence: 95%
190
+ internal orchestration executes: GET /webhook/get-player-stats
191
  → Response: Player performance card with stats
192
  ```
193
 
 
202
  - start_date / end_date: normalized week range
203
  - subject_scope: child
204
  - activity_type: all
205
+ internal orchestration executes the activities workflow with the requester identity and date window
206
  ```
207
 
208
  ### Example 2: Multiple Services
 
238
  "response": "I can help you with: Get Player Statistics. Let me route your request to the appropriate service...",
239
  "score": 0.85,
240
  "category": "service_detection",
241
+ "source": "internal orchestration_services",
242
  "matched_question": null,
243
  "services": ["get-player-stats"],
244
  "service_details": [
 
253
  }
254
  ```
255
 
256
+ The webhook payload sent to internal orchestration also includes top-level identity fields for compatibility with workflows that read them directly:
257
 
258
  ```json
259
  {
 
275
  The system handles various error scenarios:
276
 
277
  1. **Service not found**: Logs warning, continues to ML search
278
+ 2. **internal orchestration timeout**: Returns error message, suggests retrying
279
  3. **Connection error**: Returns service unavailable message
280
  4. **Invalid parameters**: Returns error with hint for valid parameters
281
 
 
283
 
284
  To add new services:
285
 
286
+ 1. **Update services.json** in `internal-service-orchestration/service-registry/`:
287
  ```json
288
  {
289
  "id": "new-service-id",
 
297
  }
298
  ```
299
 
300
+ 2. **Create internal orchestration workflow** at `/workflows/new-service-id.json`
301
 
302
+ 3. **Deploy to internal orchestration** and create webhook
303
 
304
  4. **Restart chatbot** to reload service registry
305
 
 
319
  ## Performance Considerations
320
 
321
  - Service detection: ~5-10ms (fuzzy matching)
322
+ - internal orchestration execution: Configurable timeout (default: 10s)
323
  - Service registry: Loaded once at startup, cached in memory
324
 
325
  ## Troubleshooting
 
329
  - Verify fuzzy match threshold (default 70%)
330
  - Check logs: `grep "SERVICE" logs/chatbot.log`
331
 
332
+ ### internal orchestration webhooks not responding
333
+ - Verify internal orchestration is running: `curl http://localhost:5678`
334
+ - Check internal orchestration_BASE_URL in settings
335
+ - Verify webhook paths in services.json match internal orchestration workflows
336
 
337
  ### False positives (wrong service detected)
338
  - Adjust keywords in services.json
 
343
 
344
  - **Service Detection**: `apps/chat/services/service_detection.py`
345
  - **Chatbot Core**: `apps/chat/services/bot.py`
346
+ - **internal orchestration Executor**: `apps/chat/services/internal orchestration_service_executor.py`
347
+ - **Service Registry**: `../../internal-service-orchestration/service-registry/services.json`
348
  - **Test Suite**: `apps/chat/tests/test_service_detection.py`
 
 
apps/chat/services/RunData.py CHANGED
@@ -31,5 +31,3 @@ class RunData:
31
  exact = {q.lower().strip().rstrip('?!'): a for q, a in zip(qa_q, qa_a)}
32
 
33
  DataRegistry.register_components(qa_q=qa_q, qa_a=qa_a, exact=exact)
34
-
35
-
 
31
  exact = {q.lower().strip().rstrip('?!'): a for q, a in zip(qa_q, qa_a)}
32
 
33
  DataRegistry.register_components(qa_q=qa_q, qa_a=qa_a, exact=exact)
 
 
apps/chat/services/RunMarkdowns.py CHANGED
@@ -2,5 +2,3 @@ class RunMarkdowns:
2
  def __init__(self):
3
  # Placeholder (your previous markdown registry was heavy).
4
  pass
5
-
6
-
 
2
  def __init__(self):
3
  # Placeholder (your previous markdown registry was heavy).
4
  pass
 
 
apps/chat/services/RunModel.py CHANGED
@@ -36,5 +36,3 @@ class RunModel:
36
  logger.warning("Unable to load language identification model: %s", exc)
37
  lid=None
38
  ModelRegistry.register_components(lid_176_ftz=lid, tokenizer=None, falcon_model=None)
39
-
40
-
 
36
  logger.warning("Unable to load language identification model: %s", exc)
37
  lid=None
38
  ModelRegistry.register_components(lid_176_ftz=lid, tokenizer=None, falcon_model=None)
 
 
apps/chat/services/__init__.py CHANGED
@@ -0,0 +1 @@
 
 
1
+
apps/chat/services/ai_model.py CHANGED
@@ -43,5 +43,3 @@ class Chatbot:
43
  return self.trans_helper.translate(self.markdown.format_text(ans), target_lang=user_lang, source_lang='en')
44
 
45
  return self.trans_helper.translate('No confident answer found.', target_lang=user_lang, source_lang='en')
46
-
47
-
 
43
  return self.trans_helper.translate(self.markdown.format_text(ans), target_lang=user_lang, source_lang='en')
44
 
45
  return self.trans_helper.translate('No confident answer found.', target_lang=user_lang, source_lang='en')
 
 
apps/chat/services/api_auth.py CHANGED
@@ -42,5 +42,3 @@ def require_api_key(request) -> None:
42
  raise ApiAuthError('Invalid API key')
43
 
44
  logger.debug("API authentication successful")
45
-
46
-
 
42
  raise ApiAuthError('Invalid API key')
43
 
44
  logger.debug("API authentication successful")
 
 
apps/chat/services/backend_orchestration.py ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Backend-owned orchestration contracts for secure chatbot decisions.
2
+ The chatbot detects QA/service intent and extracts fields; it never executes n8n directly.
3
+ """
4
+ from __future__ import annotations
5
+
6
+ import csv
7
+ import json
8
+ import re
9
+ from dataclasses import dataclass
10
+ from pathlib import Path
11
+ from typing import Any, Dict, List, Optional
12
+
13
+ from django.conf import settings
14
+
15
+
16
+ SERVICE_KEYWORDS = {
17
+ "create_training_session": ["training", "session", "practice", "séance", "entrainement", "entraînement"],
18
+ "send_notification": ["notify", "notification", "send message", "parents", "email", "sms"],
19
+ "generate_invoice": ["invoice", "payment", "bill", "facture", "paiement"],
20
+ "register_player": ["register player", "add player", "new player", "inscrire", "joueur"],
21
+ "create_scouting_report": ["scouting", "report", "rapport", "player evaluation"],
22
+ "rank_academies": ["rank", "ranking", "classement", "academy ranking"],
23
+ }
24
+ REQUIRED_FIELDS = {
25
+ "create_training_session": ["team", "date", "time"],
26
+ "send_notification": ["audience", "message"],
27
+ "generate_invoice": ["player"],
28
+ "register_player": ["player"],
29
+ "create_scouting_report": ["player"],
30
+ "rank_academies": [],
31
+ }
32
+
33
+
34
+ def decide_chatbot_response(message: str, context: Dict[str, Any], bot_payload: Dict[str, Any]) -> Dict[str, Any]:
35
+ msg = (message or "").strip()
36
+ allowed = set(context.get("allowed_service_ids") or [])
37
+ service_id, confidence = detect_service(msg)
38
+ if service_id and (not allowed or service_id in allowed):
39
+ extracted = extract_fields(service_id, msg)
40
+ missing = [f for f in REQUIRED_FIELDS.get(service_id, []) if not extracted.get(f)]
41
+ if missing:
42
+ return {
43
+ "type": "CLARIFICATION_REQUIRED",
44
+ "service_id": service_id,
45
+ "confidence": confidence,
46
+ "missing_fields": missing,
47
+ "question": "Please provide " + ", ".join(missing) + " so I can complete the service request.",
48
+ "safety": {"safe": True, "risk_level": "LOW"},
49
+ }
50
+ return {
51
+ "type": "SERVICE_REQUEST",
52
+ "service_id": service_id,
53
+ "confidence": confidence,
54
+ "missing_fields": [],
55
+ "extracted_data": extracted,
56
+ "user_message": "I detected a service request and extracted the required data. The backend must authorize and execute it.",
57
+ "safety": {"safe": True, "risk_level": "LOW"},
58
+ }
59
+ return {
60
+ "type": "KNOWLEDGE_QA",
61
+ "answer": bot_payload.get("response") or bot_payload.get("answer") or "I could not find a confident answer.",
62
+ "response": bot_payload.get("response") or bot_payload.get("answer") or "I could not find a confident answer.",
63
+ "confidence": float(bot_payload.get("score") or bot_payload.get("confidence") or 0.0),
64
+ "source": bot_payload.get("source") or "data.csv",
65
+ "matched_question": bot_payload.get("matched_question"),
66
+ "tenant_scope": context.get("tenant_scope") or "GLOBAL",
67
+ "safety": {"safe": True, "risk_level": "LOW"},
68
+ }
69
+
70
+
71
+ def detect_service(message: str) -> tuple[Optional[str], float]:
72
+ m = message.lower()
73
+ best = (None, 0.0)
74
+ for sid, words in SERVICE_KEYWORDS.items():
75
+ score = sum(1 for w in words if w in m) / max(1, len(words))
76
+ if score > best[1]:
77
+ best = (sid, score)
78
+ if best[1] >= 0.18:
79
+ return best[0], min(0.95, 0.65 + best[1])
80
+ return None, 0.0
81
+
82
+
83
+ def extract_fields(service_id: str, message: str) -> Dict[str, Any]:
84
+ data: Dict[str, Any] = {}
85
+ team = re.search(r"\b(U\s?\d{2}|u\s?\d{2}|under\s?\d{2})\b", message, re.I)
86
+ if team: data["team"] = team.group(1).replace(" ", "").upper()
87
+ time = re.search(r"\b(\d{1,2})(?::(\d{2}))?\s*(am|pm)?\b", message, re.I)
88
+ if time: data["time"] = time.group(0)
89
+ date_words = re.search(r"\b(today|tomorrow|next\s+\w+|\d{4}-\d{2}-\d{2}|\d{1,2}/\d{1,2}/\d{2,4})\b", message, re.I)
90
+ if date_words: data["date"] = date_words.group(0)
91
+ quoted = re.search(r"['\"]([^'\"]{2,80})['\"]", message)
92
+ if quoted: data["message"] = quoted.group(1)
93
+ player = re.search(r"(?:player|joueur)\s+([A-Z][\w\-]+(?:\s+[A-Z][\w\-]+)?)", message)
94
+ if player: data["player"] = player.group(1)
95
+ if "parent" in message.lower(): data["audience"] = "parents"
96
+ if service_id == "create_training_session" and "topic" not in data:
97
+ data["topic"] = message[:160]
98
+ return data
99
+
100
+
101
+ def finalize_service_response(payload: Dict[str, Any]) -> Dict[str, Any]:
102
+ service_id = payload.get("service_id") or "service"
103
+ result = payload.get("execution_result") or {}
104
+ status = str(result.get("status") or "failed").lower()
105
+ if status == "success":
106
+ message = result.get("safe_summary") or f"Done — the {service_id.replace('_', ' ')} service was executed successfully."
107
+ elif status == "requires_confirmation":
108
+ message = result.get("safe_summary") or "I need your confirmation before completing this action."
109
+ else:
110
+ message = result.get("safe_summary") or result.get("error") or f"I could not complete the {service_id.replace('_', ' ')} service."
111
+ return {"type": "SERVICE_FINAL_RESPONSE", "message": str(message), "response": str(message), "service_id": service_id, "status": status}
apps/chat/services/bot.py CHANGED
@@ -47,7 +47,7 @@ class Chatbot:
47
  Uses multi-tier strategy:
48
  1. Detect intent (greeting, goodbye, etc.)
49
  2. Check predefined responses
50
- 3. Detect service requests (n8n services) - PRIMARY BEFORE KB
51
  4. Query ML index (exact/fuzzy/TF-IDF)
52
  5. Return fallback message
53
 
@@ -271,5 +271,3 @@ class Chatbot:
271
  if any(connector in text for connector in connectors):
272
  return True
273
  return text.count('?') > 1
274
-
275
-
 
47
  Uses multi-tier strategy:
48
  1. Detect intent (greeting, goodbye, etc.)
49
  2. Check predefined responses
50
+ 3. Detect service requests (internal orchestration services) - PRIMARY BEFORE KB
51
  4. Query ML index (exact/fuzzy/TF-IDF)
52
  5. Return fallback message
53
 
 
271
  if any(connector in text for connector in connectors):
272
  return True
273
  return text.count('?') > 1
 
 
apps/chat/services/data_registry.py CHANGED
@@ -8,5 +8,3 @@ class DataRegistry:
8
  @classmethod
9
  def get(cls, key, default=None):
10
  return cls._registry.get(key, default)
11
-
12
-
 
8
  @classmethod
9
  def get(cls, key, default=None):
10
  return cls._registry.get(key, default)
 
 
apps/chat/services/helpers/ClassifierHelper.py CHANGED
@@ -8,5 +8,3 @@ class ClassifierHelper:
8
  if pat.search(txt):
9
  return intent, []
10
  return 'unknown', []
11
-
12
-
 
8
  if pat.search(txt):
9
  return intent, []
10
  return 'unknown', []
 
 
apps/chat/services/helpers/GetBestAnser.py CHANGED
@@ -20,5 +20,3 @@ class QARetriever:
20
  best_key, score, _ = m
21
  return [(self.exact[best_key], float(score)/100.0)]
22
  return []
23
-
24
-
 
20
  best_key, score, _ = m
21
  return [(self.exact[best_key], float(score)/100.0)]
22
  return []
 
 
apps/chat/services/helpers/GetDataFromGoogle.py CHANGED
@@ -4,5 +4,3 @@ class SiteIndexer:
4
  def index_with_progress(self):
5
  yield "FOUND:0"
6
  yield "RESULT:NO_RESULT"
7
-
8
-
 
4
  def index_with_progress(self):
5
  yield "FOUND:0"
6
  yield "RESULT:NO_RESULT"
 
 
apps/chat/services/helpers/GetPredefinedResponse.py CHANGED
@@ -8,5 +8,3 @@ class ResponseHelper:
8
  return PredefinedResponse.objects.get(intent=intent).response_text
9
  except PredefinedResponse.DoesNotExist:
10
  return None
11
-
12
-
 
8
  return PredefinedResponse.objects.get(intent=intent).response_text
9
  except PredefinedResponse.DoesNotExist:
10
  return None
 
 
apps/chat/services/helpers/MarkdownDecisionEngine.py CHANGED
@@ -1,5 +1,3 @@
1
  class MarkdownDecisionEngine:
2
  def format_text(self, text: str) -> str:
3
  return text
4
-
5
-
 
1
  class MarkdownDecisionEngine:
2
  def format_text(self, text: str) -> str:
3
  return text
 
 
apps/chat/services/helpers/TranslationHelper.py CHANGED
@@ -6,5 +6,3 @@ class TranslationHelper:
6
 
7
  def translate(self, text: str, target_lang: str, source_lang: str='auto') -> str:
8
  return get_language_processor().translate(text, target_lang=target_lang, source_lang=source_lang)
9
-
10
-
 
6
 
7
  def translate(self, text: str, target_lang: str, source_lang: str='auto') -> str:
8
  return get_language_processor().translate(text, target_lang=target_lang, source_lang=source_lang)
 
 
apps/chat/services/helpers/__init__.py CHANGED
@@ -0,0 +1 @@
 
 
1
+
apps/chat/services/helpers/intent_rules.py CHANGED
@@ -7,5 +7,3 @@ RULES = [
7
  ('ask_contact', re.compile(r"\b(contact|email|phone|tel|number)\b", re.I)),
8
  ('ask_location', re.compile(r"\b(where|location|address|map)\b", re.I)),
9
  ]
10
-
11
-
 
7
  ('ask_contact', re.compile(r"\b(contact|email|phone|tel|number)\b", re.I)),
8
  ('ask_location', re.compile(r"\b(where|location|address|map)\b", re.I)),
9
  ]
 
 
apps/chat/services/helpers/predefined.py CHANGED
@@ -7,5 +7,3 @@ def get_predefined(intent: str):
7
  return PredefinedResponse.objects.get(intent=intent).response_text
8
  except PredefinedResponse.DoesNotExist:
9
  return None
10
-
11
-
 
7
  return PredefinedResponse.objects.get(intent=intent).response_text
8
  except PredefinedResponse.DoesNotExist:
9
  return None
 
 
apps/chat/services/helpers/qa_loader.py CHANGED
@@ -19,5 +19,3 @@ class QALoader:
19
  answers = df['Answer'].tolist()
20
  exact = {q.lower().strip().rstrip('?!'): a for q, a in zip(questions, answers)}
21
  return questions, answers, exact
22
-
23
-
 
19
  answers = df['Answer'].tolist()
20
  exact = {q.lower().strip().rstrip('?!'): a for q, a in zip(questions, answers)}
21
  return questions, answers, exact
 
 
apps/chat/services/intent.py CHANGED
@@ -56,5 +56,3 @@ def get_intent_description(intent: str) -> str:
56
  'unknown': 'No specific intent detected',
57
  }
58
  return descriptions.get(intent, 'Unknown intent')
59
-
60
-
 
56
  'unknown': 'No specific intent detected',
57
  }
58
  return descriptions.get(intent, 'Unknown intent')
 
 
apps/chat/services/language_processor.py CHANGED
@@ -239,5 +239,3 @@ class LanguageProcessor:
239
  @lru_cache(maxsize=1)
240
  def get_language_processor() -> LanguageProcessor:
241
  return LanguageProcessor()
242
-
243
-
 
239
  @lru_cache(maxsize=1)
240
  def get_language_processor() -> LanguageProcessor:
241
  return LanguageProcessor()
 
 
apps/chat/services/markdowns_registry.py CHANGED
@@ -55,5 +55,3 @@ class MarkdownsRegistry:
55
  def keyphrase_model(self):
56
  return self.get('keyphrase_model')
57
 
58
-
59
-
 
55
  def keyphrase_model(self):
56
  return self.get('keyphrase_model')
57
 
 
 
apps/chat/services/ml_index.py CHANGED
@@ -326,5 +326,3 @@ class MLIndex:
326
  except (TypeError, ValueError):
327
  return self.df[self.df['academy_id'].isna()]
328
  return self.df[self.df['academy_id'].isna() | (self.df['academy_id'] == academy_id)]
329
-
330
-
 
326
  except (TypeError, ValueError):
327
  return self.df[self.df['academy_id'].isna()]
328
  return self.df[self.df['academy_id'].isna() | (self.df['academy_id'] == academy_id)]
 
 
apps/chat/services/model_registry.py CHANGED
@@ -20,5 +20,3 @@ class ModelRegistry:
20
  @classmethod
21
  def get_lid_176_ftz(cls):
22
  return cls.get('lid_176_ftz')
23
-
24
-
 
20
  @classmethod
21
  def get_lid_176_ftz(cls):
22
  return cls.get('lid_176_ftz')