diff --git a/flexus_client_kit/ckit_cloudtool.py b/flexus_client_kit/ckit_cloudtool.py index fc04184f..e8df4182 100644 --- a/flexus_client_kit/ckit_cloudtool.py +++ b/flexus_client_kit/ckit_cloudtool.py @@ -30,13 +30,14 @@ CLOUDTOOLS_VECDB = {"flexus_vector_search", "flexus_read_original"} CLOUDTOOLS_PYTHON = {"python_execute"} CLOUDTOOLS_WEB = {"web"} -CLOUDTOOLS_NOT_KANBAN = CLOUDTOOLS_VECDB | CLOUDTOOLS_PYTHON | CLOUDTOOLS_WEB +CLOUDTOOLS_AGENTS = {"flexus_hand_over_task"} +CLOUDTOOLS_NOT_KANBAN = CLOUDTOOLS_VECDB | CLOUDTOOLS_PYTHON | CLOUDTOOLS_WEB | CLOUDTOOLS_AGENTS CLOUDTOOLS_MCP = {"mcp_*"} CLOUDTOOLS_QUITE_A_LOT = KANBAN_ADVANCED | CLOUDTOOLS_NOT_KANBAN | CLOUDTOOLS_MCP -CLOUDTOOLS_ALL_KNOWN = KANBAN_ALL | CLOUDTOOLS_NOT_KANBAN # unsable in a bot +CLOUDTOOLS_ALL_KNOWN = KANBAN_ALL | CLOUDTOOLS_NOT_KANBAN # unusable in a bot def gql_error_4xx_to_model_reraise_5xx(e: gql.transport.exceptions.TransportQueryError, label: str) -> str: diff --git a/flexus_simple_bots/integration_tester/__init__.py b/flexus_simple_bots/integration_tester/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/flexus_simple_bots/integration_tester/autonomous__failed.yaml b/flexus_simple_bots/integration_tester/autonomous__failed.yaml new file mode 100644 index 00000000..c25c4015 --- /dev/null +++ b/flexus_simple_bots/integration_tester/autonomous__failed.yaml @@ -0,0 +1,57 @@ +messages: +- role: user + content: Work on the assigned task. +- role: assistant + tool_calls: + - function: + arguments: '{"op":"assign_to_this_chat","args":{"batch":["int_test_002"]}}' + name: flexus_kanban_advanced + id: call_assign_002 + type: function +- role: tool + call_id: call_assign_002 + content: |- + Assigned task int_test_002 to this chat and also moved it to 'inprogress' column + + { + "id": "int_test_002", + "title": "Test integrations batch 1/1", + "details": { + "description": "Integrations: newsapi\nTool mapping: newsapi->newsapi" + } + } +- role: assistant + tool_calls: + - function: + arguments: '{"op":"help","args":{}}' + name: newsapi + id: call_help_002 + type: function +- role: tool + call_id: call_help_002 + content: '[HELP OUTPUT - NOT A TEST] newsapi help' +- role: assistant + tool_calls: + - function: + arguments: '{"op":"status","args":{}}' + name: newsapi + id: call_status_002 + type: function +- role: tool + call_id: call_status_002 + content: 'Error [AUTH_ERROR]: API key invalid or unauthorized' +- role: assistant + tool_calls: + - function: + arguments: '{"op":"resolve","args":{"task_id":"int_test_002","resolution_code":"FAILED","resolution_summary":"| Integration | Status | Details |\n|-------------|--------|---------|\n| newsapi | FAILED | op=status: Error [AUTH_ERROR]: API key invalid or unauthorized |"}}' + name: flexus_kanban_advanced + id: call_resolve_002 + type: function +- role: tool + call_id: call_resolve_002 + content: Task int_test_002 resolved as FAILED +- role: assistant + content: |- + | Integration | Status | Details | + |-------------|--------|---------| + | newsapi | FAILED | op=status: Error [AUTH_ERROR]: API key invalid or unauthorized | diff --git a/flexus_simple_bots/integration_tester/autonomous__untested.yaml b/flexus_simple_bots/integration_tester/autonomous__untested.yaml new file mode 100644 index 00000000..d026d4be --- /dev/null +++ b/flexus_simple_bots/integration_tester/autonomous__untested.yaml @@ -0,0 +1,57 @@ +messages: +- role: user + content: Work on the assigned task. +- role: assistant + tool_calls: + - function: + arguments: '{"op":"assign_to_this_chat","args":{"batch":["int_test_001"]}}' + name: flexus_kanban_advanced + id: call_assign_001 + type: function +- role: tool + call_id: call_assign_001 + content: |- + Assigned task int_test_001 to this chat and also moved it to 'inprogress' column + + { + "id": "int_test_001", + "title": "Test integrations batch 1/1", + "details": { + "description": "Integrations: newsapi\nTool mapping: newsapi->newsapi" + } + } +- role: assistant + tool_calls: + - function: + arguments: '{"op":"help","args":{}}' + name: newsapi + id: call_help_001 + type: function +- role: tool + call_id: call_help_001 + content: '[HELP OUTPUT - NOT A TEST] newsapi help' +- role: assistant + tool_calls: + - function: + arguments: '{"op":"list_methods","args":{}}' + name: newsapi + id: call_methods_001 + type: function +- role: tool + call_id: call_methods_001 + content: 'api_key_hint=***dc9, method_ids=[newsapi.everything.v1, newsapi.top_headlines.v1, newsapi.sources.v1]' +- role: assistant + tool_calls: + - function: + arguments: '{"op":"resolve","args":{"task_id":"int_test_001","resolution_code":"FAILED","resolution_summary":"| Integration | Status | Details |\n|-------------|--------|---------|\n| newsapi | UNTESTED | help and list_methods only; no real API test; api_key_hint=***dc9 |"}}' + name: flexus_kanban_advanced + id: call_resolve_001 + type: function +- role: tool + call_id: call_resolve_001 + content: Task int_test_001 resolved as FAILED +- role: assistant + content: |- + | Integration | Status | Details | + |-------------|--------|---------| + | newsapi | UNTESTED | help and list_methods only; no real API test; api_key_hint=***dc9 | diff --git a/flexus_simple_bots/integration_tester/integration_tester-1024x1536.webp b/flexus_simple_bots/integration_tester/integration_tester-1024x1536.webp new file mode 100644 index 00000000..d7f73452 Binary files /dev/null and b/flexus_simple_bots/integration_tester/integration_tester-1024x1536.webp differ diff --git a/flexus_simple_bots/integration_tester/integration_tester-256x256.webp b/flexus_simple_bots/integration_tester/integration_tester-256x256.webp new file mode 100644 index 00000000..3295df1d Binary files /dev/null and b/flexus_simple_bots/integration_tester/integration_tester-256x256.webp differ diff --git a/flexus_simple_bots/integration_tester/integration_tester__s1.yaml b/flexus_simple_bots/integration_tester/integration_tester__s1.yaml new file mode 100644 index 00000000..4b74e92c --- /dev/null +++ b/flexus_simple_bots/integration_tester/integration_tester__s1.yaml @@ -0,0 +1,48 @@ +messages: +- role: user + content: test all integrations +- role: assistant + tool_calls: + - function: + arguments: '{"requested":"all","batch_size":5,"configured_only":true}' + name: integration_plan_batches + id: call_plan_001 + type: function +- role: tool + call_id: call_plan_001 + content: |- + { + "ok": true, + "requested": ["all"], + "supported": ["newsapi", "resend"], + "configured": ["newsapi", "resend"], + "configured_only": true, + "selected": ["newsapi", "resend"], + "unsupported": [], + "batch_size": 5, + "batches": [["newsapi", "resend"]], + "task_specs": [ + { + "title": "Test integrations batch 1/1", + "description": "Integrations: newsapi,resend\nTool mapping: newsapi->newsapi, resend->email_setup_domain", + "integrations": ["newsapi", "resend"] + } + ] + } +- role: assistant + tool_calls: + - function: + arguments: '{"description":"Integrations: newsapi,resend\nTool mapping: newsapi->newsapi, resend->email_setup_domain","fexp_name":"autonomous","title":"Test integrations batch 1/1","to_bot":"Integration Tester"}' + name: flexus_hand_over_task + id: call_handover_001 + type: function +- role: tool + call_id: call_handover_001 + content: |- + ✓ Task handed over to 'Integration Tester' + Task ID: int_test_001 +- role: assistant + content: |- + Created 1 batch task for 2 integrations (newsapi, resend). + + Task handed over to autonomous worker for execution. diff --git a/flexus_simple_bots/integration_tester/integration_tester_bot.py b/flexus_simple_bots/integration_tester/integration_tester_bot.py new file mode 100644 index 00000000..d3bf7a44 --- /dev/null +++ b/flexus_simple_bots/integration_tester/integration_tester_bot.py @@ -0,0 +1,148 @@ +import asyncio +import json +import sys +from pathlib import Path + +_repo_root = Path(__file__).parents[2] +if str(_repo_root) not in sys.path: + sys.path.insert(0, str(_repo_root)) + +from flexus_client_kit import ckit_bot_exec, ckit_client, ckit_shutdown, ckit_integrations_db +from flexus_client_kit import ckit_bot_version +from flexus_simple_bots.integration_tester import integration_tester_shared as shared +from flexus_simple_bots.integration_tester import integration_tester_install + +BOT_NAME = ckit_bot_version.bot_name_from_file(__file__) +BOT_VERSION = (Path(__file__).parents[1] / "VERSION").read_text().strip() +SETUP_SCHEMA = json.loads((Path(__file__).parent / "setup_schema.json").read_text()) + + +async def integration_tester_main_loop( + fclient: ckit_client.FlexusClient, + rcx: ckit_bot_exec.RobotContext, +) -> None: + setup = ckit_bot_exec.official_setup_mixing_procedure(SETUP_SCHEMA, rcx.persona.persona_setup) + shared.load_env_config(setup) + + integr_records = shared.INTEGRATION_TESTER_INTEGRATIONS + setup_allow = shared._setup_allowlist_names(setup) + if setup_allow: + allow = set(setup_allow) + integr_records = [r for r in integr_records if r.integr_name in allow] + + await ckit_integrations_db.main_loop_integrations_init(integr_records, rcx, setup) + supported_integrations = sorted({r.integr_name for r in integr_records}) + + for name, reg in shared.INTEGRATION_REGISTRY.items(): + if name not in supported_integrations: + continue + obj = reg["integration_cls"](*reg["integration_args"](fclient, rcx, setup)) + rcx.on_tool_call(reg["tool"].name)(shared.IntegrationHandler(reg, obj)) + + @rcx.on_tool_call(shared.PLAN_BATCHES_TOOL.name) + async def toolcall_plan_batches(toolcall, model_produced_args): + args = model_produced_args or {} + req = shared._requested_names(str(args.get("requested", "all"))) + bs = args.get("batch_size", 5) + configured_only = bool(args.get("configured_only", True)) + try: + bs = int(bs) + except (TypeError, ValueError): + bs = 5 + + configured = {x["name"] for x in shared.get_configured_integrations()} + selected = [] + unsupported = [] + + if "all" in req: + pool = [x for x in supported_integrations if (x in configured or not configured_only)] + selected = pool + else: + for x in req: + if x not in supported_integrations: + unsupported.append(x) + continue + if configured_only and x not in configured: + continue + if x not in selected: + selected.append(x) + + batches = shared._chunk_names(selected, bs) + task_specs = [] + total = len(batches) + for i, b in enumerate(batches, start=1): + tool_map = ", ".join(f"{name}->{shared.INTEGRATION_REGISTRY[name]['tool'].name}" for name in b) + task_specs.append({ + "title": f"Test integrations batch {i}/{total}", + "description": f"Integrations: {','.join(b)}\nTool mapping: {tool_map}", + "integrations": b, + }) + + return json.dumps({ + "ok": True, + "requested": req, + "supported": supported_integrations, + "configured": sorted(configured), + "configured_only": configured_only, + "selected": selected, + "unsupported": unsupported, + "batch_size": bs, + "batches": batches, + "task_specs": task_specs, + }, indent=2) + + configured = shared.get_configured_integrations() + shared.logger.info(f"Integration Tester started. Configured integrations: {[i['name'] for i in configured]}") + + @rcx.on_updated_task + async def on_task_update(action, old_task, new_task): + task = new_task or old_task + if not task: + shared.logger.info(f"TASK UPDATE: {action} with no task payload") + return + col = task.calc_bucket() + title = task.ktask_title + tid = task.ktask_id + if col == "inprogress": + shared.logger.info(f"TASK ASSIGNED: {title} (id={tid}) - will test now") + elif col == "done": + shared.logger.info(f"TASK COMPLETED: {title} (id={tid})") + else: + shared.logger.info(f"TASK UPDATE: {title} moved to {col} (id={tid})") + + while not ckit_shutdown.shutdown_event.is_set(): + await rcx.unpark_collected_events(sleep_if_no_work=10.0) + + shared.logger.info(f"{rcx.persona.persona_id} exit") + + +def main(): + scenario_fn = ckit_bot_exec.parse_bot_args() + fclient = ckit_client.FlexusClient( + ckit_client.bot_service_name(BOT_NAME, BOT_VERSION), + endpoint="/v1/jailed-bot", + ) + + from dotenv import load_dotenv + load_dotenv() + + async def _install_compat(client: ckit_client.FlexusClient) -> int: + await integration_tester_install.install( + client, + bot_name=BOT_NAME, + bot_version=BOT_VERSION, + tools=shared.TOOLS, + ) + return 0 + + asyncio.run(ckit_bot_exec.run_bots_in_this_group( + fclient, + bot_main_loop=integration_tester_main_loop, + inprocess_tools=shared.TOOLS, + scenario_fn=scenario_fn, + install_func=_install_compat, + )) + + +if __name__ == "__main__": + main() diff --git a/flexus_simple_bots/integration_tester/integration_tester_install.py b/flexus_simple_bots/integration_tester/integration_tester_install.py new file mode 100644 index 00000000..5a83fcf2 --- /dev/null +++ b/flexus_simple_bots/integration_tester/integration_tester_install.py @@ -0,0 +1,208 @@ +import asyncio +import json +import logging +import os +from typing import List + +from flexus_client_kit import ckit_client, ckit_bot_install, ckit_cloudtool, ckit_skills +from flexus_simple_bots import prompts_common +from flexus_simple_bots.integration_tester import integration_tester_shared as shared + +logger = logging.getLogger("integration_tester") + +INTEGRATION_TESTER_SKILLS = ckit_skills.static_skills_find(shared.INTEGRATION_TESTER_ROOTDIR, shared_skills_allowlist="", integration_skills_allowlist="") + + +def _build_experts(tools): + builtin_skills = ckit_skills.read_name_description(shared.INTEGRATION_TESTER_ROOTDIR, INTEGRATION_TESTER_SKILLS) + tool_names = {reg["tool"].name for reg in shared.INTEGRATION_REGISTRY.values()} + tool_names.add(shared.PLAN_BATCHES_TOOL.name) + allow_tools = ",".join(tool_names | ckit_cloudtool.KANBAN_ADVANCED | {"flexus_hand_over_task"}) + + default_prompt = """You are Integration Tester. Your job is to queue autonomous smoke tests for supported API-key integrations and then report the finished results clearly. + +Rules: +- Supported requests are: "all" or a comma-separated list of supported integration names. +- First call integration_plan_batches(requested="...", batch_size=5, configured_only=true). +- Use every returned task_spec to create a task with flexus_hand_over_task(to_bot="Integration Tester", title=..., description=..., fexp_name="autonomous"). +- Do not run integration tools in this interactive chat. This chat only plans work and reports completed task results. +- If nothing supported/configured was selected, explain that briefly and stop. +- Mention unsupported requested names if any. + +After queueing tasks, reply in this format: +Queued {{N}} batch covering {{X}} integrations: {{name1}} and {{name2}}. + +Detailed per-integration results will appear here after the autonomous worker finishes. + +When a completed-task message arrives: +- read resolution_summary +- present it as a markdown table if it is a table, otherwise give a short plain summary +- do not dump raw task metadata +""" + + autonomous_prompt = """You are Integration Tester autonomous worker. You own one kanban task and must finish it without asking the user anything. + +Task handling: +- Read the assigned task. +- Parse integration names from the description line: "Integrations: name1,name2,...". +- Read the optional mapping line: "Tool mapping: integration1->tool_name1, integration2->tool_name2". +- If a mapping line is present, use that tool name literally for the matching integration. +- Example: integration `resend` may map to tool `email_setup_domain`. +- If parsing fails, resolve the task as FAILED with summary "Batch parse error". +- You must finish this batch in the current task/thread. +- Do not hand over, delegate, split, or create sibling tasks for individual integrations. +- A delegated or promised future test does not count as a test result for this task. + +For each integration: +1. Call op="help" once to discover safe operations. +2. If available, you may call list_methods to inspect method names. +3. Then run at least one more real non-discovery read-only call. +3. Never use create, update, delete, send, or other state-changing operations. +4. Prefer status, list, get, search, or a simple call with harmless arguments. + +Operation classes: +- discovery/local ops: help, list_methods, and any status op that only reports local readiness, configured credentials, or known method counts +- provider-check ops: call, list, get, search, or a status op only when it clearly performs a real provider/API check instead of local metadata reporting +- prefer provider-check ops over discovery/local ops +- when using op="call", method_id must be copied literally from list_methods or help output +- never invent or guess method_id values +- op names such as help, status, list, or search are not valid method_id values unless they appear literally in the method list + +What counts as a real test: +- help is not a test +- list_methods is discovery, not a test +- a local/status readiness check is not a test if it only reports local metadata such as has_api_key, ready, configured, or method_count +- tool output starting with [HELP OUTPUT - NOT A TEST] is not a test +- you must not stop after help or list_methods +- the first call that can count as a test must be a provider-check op +- every integration listed in the task must get its own real non-discovery call +- the real non-discovery call must happen in this task/thread, not in another task +- the integration is UNTESTED if you only called help/list_methods or never made a non-discovery call +- if you skipped a listed integration, it is UNTESTED +- if you delegated a listed integration instead of testing it here, it is UNTESTED +- do not invent reasons such as "no safe method available" unless the tool itself explicitly told you that +- the integration is FAILED if a non-help call returns an error, including 401/403/auth problems +- the integration is PASSED if a provider-check op succeeds and returns concrete provider data or provider-backed metadata +- if op="call" fails with METHOD_UNKNOWN because you guessed method_id, read list_methods and retry once with a literal listed method_id before deciding the final status + +Report requirements: +- Build a markdown table with exactly these columns: Integration | Status | Details +- Status must be one of: PASSED, FAILED, UNTESTED +- Details must include the real provider-check operation you used, api_key_hint if present, and one concrete result such as count, returned object type, key fields, or error text +- Keep details concise and factual + +Resolve the task with flexus_kanban_advanced using: +- resolution_code=PASSED only if every listed integration is PASSED +- if any integration is FAILED or UNTESTED, resolution_code must be FAILED +- resolution_summary= + +Do not wait for user input. Do not leave the task unresolved. +""" + + return [ + ("default", ckit_bot_install.FMarketplaceExpertInput( + fexp_system_prompt=default_prompt, + fexp_python_kernel="", + fexp_allow_tools=allow_tools, + fexp_nature="NATURE_INTERACTIVE", + fexp_builtin_skills=builtin_skills, + fexp_description="Test API key integrations", + )), + ("autonomous", ckit_bot_install.FMarketplaceExpertInput( + fexp_system_prompt=autonomous_prompt, + fexp_python_kernel="", + fexp_allow_tools=allow_tools, + fexp_nature="NATURE_AUTONOMOUS", + fexp_inactivity_timeout=600, + fexp_builtin_skills=builtin_skills, + fexp_description="Autonomous integration testing", + )), + ] + + +INTEGRATION_TESTER_DESC = """ +**Job description** + +Integration Tester validates that Flexus API key-based integrations are properly configured and functional. +It only tests integrations that are explicitly allowed for this bot and have API keys provided through ENV_CONFIG. + +**How it works:** +1. User starts a test session via "Test Integrations" button +2. Bot checks which supported integrations are configured +3. User selects what to test (all or specific supported integrations) +4. Bot creates deterministic kanban batch tasks in inbox +5. Autonomous worker discovers safe operations, runs at least one real read-only API call per integration, and resolves the task with a table of results + +**What it tests:** +- Any integration included in this bot's supported allowlist +- Real read-only operations only +- No create/update/delete/send actions + +**Results:** +- PASSED: A real non-help read-only call succeeded +- FAILED: A real non-help call failed +- UNTESTED: Only discovery calls were made, so the integration was not actually tested +""" + + +def _ensure_marketplace_images() -> None: + pic_big_path = shared.INTEGRATION_TESTER_ROOTDIR / "integration_tester-1024x1536.webp" + pic_small_path = shared.INTEGRATION_TESTER_ROOTDIR / "integration_tester-256x256.webp" + fallback_big_path = shared.INTEGRATION_TESTER_ROOTDIR.parent / "bob" / "bob-1024x1536.webp" + fallback_small_path = shared.INTEGRATION_TESTER_ROOTDIR.parent / "bob" / "bob-256x256.webp" + + if not pic_big_path.exists() and fallback_big_path.exists(): + pic_big_path.write_bytes(fallback_big_path.read_bytes()) + if not pic_small_path.exists() and fallback_small_path.exists(): + pic_small_path.write_bytes(fallback_small_path.read_bytes()) + + +async def install( + client: ckit_client.FlexusClient, + bot_name: str, + bot_version: str, + tools: List[ckit_cloudtool.CloudTool], +): + setup_schema_path = shared.INTEGRATION_TESTER_ROOTDIR / "setup_schema.json" + integration_tester_setup_default = json.loads(setup_schema_path.read_text()) + + _ensure_marketplace_images() + + experts = _build_experts(tools) + + await ckit_bot_install.marketplace_upsert_dev_bot( + client, + ws_id=client.ws_id, + bot_dir=shared.INTEGRATION_TESTER_ROOTDIR, + marketable_title1="Integration Tester", + marketable_title2="Test API key integrations", + marketable_author="Flexus", + marketable_accent_color="#4CAF50", + marketable_occupation="QA Engineer", + marketable_description=INTEGRATION_TESTER_DESC, + marketable_typical_group="Development", + marketable_schedule=[ + prompts_common.SCHED_TASK_SORT_10M | {"sched_when": "EVERY:1m", "sched_fexp_name": "default"}, + prompts_common.SCHED_TODO_5M | {"sched_when": "EVERY:1m", "sched_fexp_name": "autonomous"}, + ], + marketable_setup_default=integration_tester_setup_default, + marketable_featured_actions=[ + {"feat_question": "Test all integrations", "feat_expert": "default"}, + {"feat_question": "Test newsapi", "feat_expert": "default"}, + {"feat_question": "Test resend", "feat_expert": "default"}, + ], + marketable_intro_message="Hi! I'm Integration Tester. I create deterministic kanban batch tasks and resolve them autonomously.", + marketable_preferred_model_expensive="gpt-5.4-mini", + marketable_preferred_model_cheap="gpt-5.4-mini", + marketable_experts=[(name, exp.filter_tools(tools)) for name, exp in experts], + marketable_tags=["testing", "integrations", "qa"], + marketable_forms=ckit_bot_install.load_form_bundles(__file__), + ) + + +if __name__ == "__main__": + from dotenv import load_dotenv + load_dotenv() + + client = ckit_client.FlexusClient("integration_tester_install") + asyncio.run(install(client, bot_name="integration_tester", bot_version="dev", tools=shared.TOOLS)) diff --git a/flexus_simple_bots/integration_tester/integration_tester_shared.py b/flexus_simple_bots/integration_tester/integration_tester_shared.py new file mode 100644 index 00000000..8f00358c --- /dev/null +++ b/flexus_simple_bots/integration_tester/integration_tester_shared.py @@ -0,0 +1,225 @@ +import json +import os +import logging +from pathlib import Path +from typing import Any, Dict, List + +from flexus_client_kit import ckit_cloudtool, ckit_integrations_db +from flexus_client_kit.integrations import fi_newsapi, fi_resend + +logger = logging.getLogger("integration_tester") + +INTEGRATION_TESTER_ROOTDIR = Path(__file__).parent + +PLAN_BATCHES_TOOL = ckit_cloudtool.CloudTool( + strict=True, + name="integration_plan_batches", + description="Plan deterministic integration test batches and return task specs for kanban fan-out.", + parameters={ + "type": "object", + "additionalProperties": False, + "properties": { + "requested": {"type": "string", "description": "Requested integrations, e.g. 'all' or 'newsapi,resend'."}, + "batch_size": {"type": "integer", "description": "Max integrations per task batch."}, + "configured_only": {"type": "boolean", "description": "If true, include only integrations with configured keys."}, + }, + "required": ["requested", "batch_size", "configured_only"], + }, +) + +NEWSAPI_TOOL = ckit_cloudtool.CloudTool( + strict=False, + name=fi_newsapi.PROVIDER_NAME, + description=f"{fi_newsapi.PROVIDER_NAME}: data provider. op=help|status|list_methods|call", + parameters={ + "type": "object", + "additionalProperties": False, + "properties": { + "op": {"type": "string", "enum": ["help", "status", "list_methods", "call"]}, + "args": { + "type": "object", + "additionalProperties": False, + "properties": { + "method_id": {"type": "string"}, + "include_raw": {"type": "boolean"}, + "q": {"type": "string"}, + "query": {"type": "string"}, + "sources": {"type": "string"}, + "domains": {"type": "string"}, + "excludeDomains": {"type": "string"}, + "from": {"type": "string"}, + "to": {"type": "string"}, + "language": {"type": "string"}, + "sortBy": {"type": "string"}, + "pageSize": {"type": "integer"}, + "page": {"type": "integer"}, + "country": {"type": "string"}, + "category": {"type": "string"}, + "time_window": {"type": "string"}, + "start_date": {"type": "string"}, + "end_date": {"type": "string"}, + }, + }, + }, + "required": ["op", "args"], + }, +) + +INTEGRATION_REGISTRY: Dict[str, Dict[str, Any]] = { + "newsapi": { + "env_var": "NEWSAPI_API_KEY", + "alt_env_vars": ["NEWSAPI_KEY"], + "tool": NEWSAPI_TOOL, + "integration_cls": fi_newsapi.IntegrationNewsapi, + "integration_args": lambda fclient, rcx, setup: (rcx,), + "handler_method": "called_by_model", + }, + "resend": { + "env_var": "RESEND_API_KEY", + "tool": fi_resend.RESEND_SETUP_TOOL, + "integration_cls": fi_resend.IntegrationResend, + "integration_args": lambda fclient, rcx, setup: (fclient, rcx, {}), + "handler_method": "setup_called_by_model", + }, +} + +INTEGRATION_TESTER_INTEGRATIONS: list[ckit_integrations_db.IntegrationRecord] = ckit_integrations_db.static_integrations_load( + INTEGRATION_TESTER_ROOTDIR, + allowlist=["newsapi", "resend"], + builtin_skills=[], +) + +TOOLS = [PLAN_BATCHES_TOOL] + [reg["tool"] for reg in INTEGRATION_REGISTRY.values()] + + +def _chunk_names(xs: List[str], n: int) -> List[List[str]]: + if n < 1: + n = 1 + return [xs[i:i + n] for i in range(0, len(xs), n)] + + +def _requested_names(raw: str) -> List[str]: + s = (raw or "").strip().lower() + if not s or s == "all": + return ["all"] + names = [] + for x in s.replace(";", ",").split(","): + x = x.strip() + if x: + names.append(x) + return names or ["all"] + + +def _setup_allowlist_names(setup: Dict[str, Any]) -> List[str]: + raw = str(setup.get("INTEGRATION_TESTER_ALLOWLIST", "") or "").strip().lower() + if not raw: + return [] + names: List[str] = [] + for x in raw.replace(";", ",").split(","): + x = x.strip() + if x: + names.append(x) + return names + + +def load_env_config(setup: Dict[str, Any]) -> None: + env_config = setup.get("ENV_CONFIG", "") + if not env_config: + logger.info("No ENV_CONFIG found in persona_setup") + return + count = 0 + for line in env_config.strip().split('\n'): + if '=' in line and not line.startswith('#'): + key, value = line.split('=', 1) + os.environ[key.strip()] = value.strip() + count += 1 + logger.info(f"Loaded {count} environment variables from ENV_CONFIG") + + +def get_configured_integrations() -> List[Dict[str, Any]]: + result = [] + for name, reg in INTEGRATION_REGISTRY.items(): + key = os.environ.get(reg["env_var"]) + if not key and "alt_env_vars" in reg: + for alt in reg["alt_env_vars"]: + key = os.environ.get(alt) + if key: + break + if key: + result.append({ + "name": name, + "env_var": reg["env_var"], + "key_hint": key[-4:] if len(key) > 4 else "***", + }) + return result + + +def _resolve_api_key(env_var: str, alt_env_vars: list[str]) -> str | None: + key = os.environ.get(env_var) + if not key: + for alt in alt_env_vars: + key = os.environ.get(alt) + if key: + break + return key + + +def classify_error(e: Exception) -> tuple[str, str]: + msg = str(e).lower() + if any(k in msg for k in ("401", "403", "unauthorized", "invalid api", "forbidden", "invalid_api")): + return "AUTH_ERROR", "API key invalid or unauthorized" + if any(k in msg for k in ("timeout", "connection", "dns", "network", "connect", "refused")): + return "NETWORK_ERROR", "Network/connectivity issue" + if any(k in msg for k in ("rate", "429", "quota", "limit")): + return "RATE_LIMIT", "API rate limit or quota exceeded" + return "UNKNOWN_ERROR", str(e)[:200] + + +class IntegrationHandler: + def __init__(self, reg: Dict[str, Any], obj: Any): + self.env_var = reg["env_var"] + self.alt_env_vars = reg.get("alt_env_vars", []) + self.handler_method = reg["handler_method"] + self.tool_name = reg["tool"].name + self.obj = obj + + @staticmethod + def _format_result(raw: str) -> str: + try: + data = json.loads(raw) + if isinstance(data, dict): + skip = {"ok", "provider", "description", "help_text"} + parts = [] + for k, v in data.items(): + if k in skip: + continue + if isinstance(v, list) and v and all(isinstance(x, str) for x in v): + parts.append(f"{k}=[{', '.join(v)}]") + elif not isinstance(v, (dict, list)): + parts.append(f"{k}={v}") + if parts: + return ", ".join(parts) + except json.JSONDecodeError: + pass + return raw + + async def __call__(self, toolcall: ckit_cloudtool.FCloudtoolCall, model_produced_args: Dict[str, Any]) -> str: + key = _resolve_api_key(self.env_var, self.alt_env_vars) + logger.info(f"Testing {self.tool_name} - API key present: {bool(key)}") + if not key: + logger.warning(f"{self.tool_name} test FAILED - no API key configured") + return f"Error [AUTH_ERROR]: {self.env_var} not configured. Resolve the kanban task as FAILED with status: 'FAILED - No API key configured for {self.tool_name}'" + try: + result = await getattr(self.obj, self.handler_method)(toolcall, model_produced_args) + op = str(model_produced_args.get("op", "")).strip() if model_produced_args else "" + if op == "help": + result = "[HELP OUTPUT - NOT A TEST] " + result + formatted = self._format_result(result) + key_hint = key[-3:] if len(key) > 3 else "***" + out = f"api_key_hint=***{key_hint}, {formatted}" + logger.info(f"{self.tool_name} test result: {out[:120]}..." if len(out) > 120 else f"{self.tool_name} test result: {out}") + return out + except Exception as e: + category, detail = classify_error(e) + logger.error(f"toolcall_{self.tool_name}: {category}: {detail}", exc_info=True) + return f"Error [{category}]: {detail}" diff --git a/flexus_simple_bots/integration_tester/setup_schema.json b/flexus_simple_bots/integration_tester/setup_schema.json new file mode 100644 index 00000000..6536bbca --- /dev/null +++ b/flexus_simple_bots/integration_tester/setup_schema.json @@ -0,0 +1,20 @@ +[ + { + "bs_name": "ENV_CONFIG", + "bs_type": "string_multiline", + "bs_default": "", + "bs_group": "integrations", + "bs_importance": 1, + "bs_placeholder": "NEWSAPI_API_KEY=...\nRESEND_API_KEY=...\n...", + "bs_description": "Paste your .env file content here with all integration keys. Each line should be KEY=VALUE format." + }, + { + "bs_name": "INTEGRATION_TESTER_ALLOWLIST", + "bs_type": "string_short", + "bs_default": "", + "bs_group": "integrations", + "bs_importance": 1, + "bs_placeholder": "newsapi,resend", + "bs_description": "Optional CSV list of integration names to include. Empty means all discovered integrations with supported smoke tests." + } +]