diff --git a/agent.ts b/agent.ts index 335c201..12cde89 100644 --- a/agent.ts +++ b/agent.ts @@ -1,9 +1,11 @@ import { OpenAI } from 'openai'; +import * as readline from 'readline'; import { z } from 'zod'; import { Agent, AgentInputItem, + hostedMcpTool, Runner, tool, withTrace, @@ -18,20 +20,64 @@ const getRetentionOffers = tool({ customer_id: z.string(), account_type: z.string(), current_plan: z.string(), - tenure_months: z.integer(), + tenure_months: z.number().int(), recent_complaints: z.boolean(), }), execute: async (input: { customer_id: string; account_type: string; current_plan: string; - tenure_months: integer; + tenure_months: number; recent_complaints: boolean; }) => { - // TODO: Unimplemented + // Mock implementation - returns sample retention offers + return { + offers: [ + { + type: "discount", + description: "20% off for 12 months", + monthly_savings: 15, + }, + { + type: "upgrade", + description: "Free upgrade to premium plan for 3 months", + value: 45, + }, + ], + }; }, }); +const mcp = hostedMcpTool({ + serverLabel: "dropbox", + connectorId: "connector_dropbox", + serverDescription: "Return Policy Knowledge", + allowedTools: [ + "fetch", + "fetch_file", + "get_profile", + "list_recent_files", + "search", + "search_files", + ], + requireApproval: "never", +}); + +const mcp1 = hostedMcpTool({ + serverLabel: "dropbox", + connectorId: "connector_dropbox", + serverDescription: "Knowledge Base", + allowedTools: [ + "fetch", + "fetch_file", + "get_profile", + "list_recent_files", + "search", + "search_files", + ], + requireApproval: "never", +}); + // Shared client for guardrails and file search const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY }); @@ -50,11 +96,11 @@ const jailbreakGuardrailConfig = { const context = { guardrailLlm: client }; // Guardrails utils -function guardrailsHasTripwire(results) { - return (results ?? []).some((r) => r?.tripwireTriggered === true); +function guardrailsHasTripwire(results: any) { + return (results ?? []).some((r: any) => r?.tripwireTriggered === true); } -function getGuardrailSafeText(results, fallbackText) { +function getGuardrailSafeText(results: any, fallbackText: string) { // Prefer checked_text as the generic safe/processed text for (const r of results ?? []) { if (r?.info && "checked_text" in r.info) { @@ -63,14 +109,14 @@ function getGuardrailSafeText(results, fallbackText) { } // Fall back to PII-specific anonymized_text if present const pii = (results ?? []).find( - (r) => r?.info && "anonymized_text" in r.info + (r: any) => r?.info && "anonymized_text" in r.info ); return pii?.info?.anonymized_text ?? fallbackText; } -function buildGuardrailFailOutput(results) { - const get = (name) => - (results ?? []).find((r) => { +function buildGuardrailFailOutput(results: any) { + const get = (name: string) => + (results ?? []).find((r: any) => { const info = r?.info ?? {}; const n = info?.guardrail_name ?? info?.guardrailName; return n === name; @@ -81,7 +127,7 @@ function buildGuardrailFailOutput(results) { hal = get("Hallucination Detection"), piiCounts = Object.entries(pii?.info?.detected_entities ?? {}) .filter(([, v]) => Array.isArray(v)) - .map(([k, v]) => k + ":" + v.length), + .map(([k, v]) => k + ":" + (v as any[]).length), thr = jb?.info?.threshold, conf = jb?.info?.confidence; @@ -130,6 +176,7 @@ function buildGuardrailFailOutput(results) { }, }; } + const ClassificationAgentSchema = z.object({ classification: z.enum([ "return_item", @@ -139,7 +186,7 @@ const ClassificationAgentSchema = z.object({ }); const classificationAgent = new Agent({ name: "Classification agent", - instructions: `Classify the user’s intent into one of the following categories: \"return_item\", \"cancel_subscription\", or \"get_information\". + instructions: `Classify the user's intent into one of the following categories: \"return_item\", \"cancel_subscription\", or \"get_information\". 1. Any device-related return requests should route to return_item. 2. Any retention or cancellation risk, including any request for discounts should route to cancel_subscription. @@ -156,9 +203,10 @@ const classificationAgent = new Agent({ const returnAgent = new Agent({ name: "Return agent", - instructions: `Offer a replacement device with free shipping. -`, - model: "gpt-4.1-mini", + instructions: + "Always check return policy related content in dropbox first, only if the request qualify against the policies, then you can issue return", + model: "gpt-4.1", + tools: [mcp], modelSettings: { temperature: 1, topP: 1, @@ -184,56 +232,15 @@ const retentionAgent = new Agent({ const informationAgent = new Agent({ name: "Information agent", - instructions: `You are an information agent for answering informational queries. Your aim is to provide clear, concise responses to user questions. Use the policy below to assemble your answer. - -Company Name: HorizonTel Communications Industry: Telecommunications Region: North America -📋 Policy Summary: Mobile Service Plan Adjustments -Policy ID: MOB-PLN-2025-03 Effective Date: March 1, 2025 Applies To: All residential and small business mobile customers -Purpose: To ensure customers have transparent and flexible options when modifying or upgrading their existing mobile service plans. -🔄 Plan Changes & Upgrades -Eligibility: Customers must have an active account in good standing (no outstanding balance > $50). -Upgrade Rules: -Device upgrades are permitted once every 12 months if the customer is on an eligible plan. -Early upgrades incur a $99 early-change fee unless the new plan’s monthly cost is higher by at least $15. -Downgrades: Customers can switch to a lower-tier plan at any time; changes take effect at the next billing cycle. -CS Rep Tip: When customers request plan changes, confirm their next billing cycle and remind them that prorated charges may apply. Always check for active device installment agreements before confirming a downgrade. -💰 Billing & Credits -Billing Cycle: Monthly, aligned with the activation date. -Credit Adjustments: -Overcharges under $10 are automatically credited to the next bill. -For amounts >$10, open a “Billing Adjustment – Tier 2” ticket for supervisor review. -Refund Policy: -Refunds are issued to the original payment method within 7–10 business days. -For prepaid accounts, credits are applied to the balance—no cash refunds. -CS Rep Tip: If a customer reports a billing discrepancy within 30 days, you can issue an immediate one-time goodwill credit (up to $25) without manager approval. -🛜 Network & Outage Handling -Planned Maintenance: Customers receive SMS alerts for outages >1 hour. -Unplanned Outages: -Check the internal “Network Status Dashboard” before escalating. -If multiple customers in a region report the same issue, tag the ticket as “Regional Event – Network Ops.” -Compensation: Customers experiencing service interruption exceeding 24 consecutive hours are eligible for a 1-day service credit upon request. -📞 Retention & Cancellations -Notice Period: 30 days for postpaid accounts; immediate for prepaid. -Retention Offers: -Agents may offer up to 20% off the next 3 billing cycles if the customer cites “cost concerns.” -Retention codes must be logged under “RET-SAVE20.” -Cancellation Fee: -Applies only to term contracts (usually $199 flat rate). -Fee waived for verified relocation to non-serviceable area. -CS Rep Tip: Before processing a cancellation, review alternative retention offers—customers frequently stay when offered a temporary discount or bonus data package. -🧾 Documentation Checklist for CS Reps -Verify customer ID and account number. -Check account standing (billing, contracts, upgrades). -Record all interactions in the CRM ticket. -Confirm next billing cycle date for any changes. -Apply standard note template: -“Customer requested [plan/billing/support] change. Informed of applicable fees, next cycle adjustment, and confirmation reference #[ticket].” -⚠️ Compliance & Privacy -All interactions must comply with CCPA and FCC privacy standards. -Do not record or store personal payment information outside the secure billing system. -Use the “Secure Verification Flow” for identity confirmation before discussing account details. -🧠 Example`, - model: "gpt-4.1-mini", + instructions: `You are an information agent for answering informational queries. Your aim is to provide clear, concise responses to user questions. + +Always check dropbox to give the accurate answers + +Check all files first that you can see, use any files available to your access. + +You can use all files but really only reference customer QA related content, nothing else`, + model: "gpt-4.1", + tools: [mcp1], modelSettings: { temperature: 1, topP: 1, @@ -242,35 +249,111 @@ Use the “Secure Verification Flow” for identity confirmation before discussi }, }); -const approvalRequest = (message: string) => { - // TODO: Implement - return true; +const approvalRequest = async (message: string): Promise => { + // Auto-approve for automated testing (red team, CI/CD) + // Set PROMPTFOO_AUTO_APPROVE=false to enable interactive mode + const autoApprove = process.env.PROMPTFOO_AUTO_APPROVE !== "false"; + + if (autoApprove) { + console.log(`${message} (auto-approved)`); + return true; + } + + // Interactive mode for manual testing + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + return new Promise((resolve) => { + rl.question(`${message} (yes/no): `, (answer) => { + rl.close(); + resolve(answer.toLowerCase() === "yes" || answer.toLowerCase() === "y"); + }); + }); }; -type WorkflowInput = { input_as_text: string }; +type WorkflowInput = { + input_as_text?: string; + messages?: Array<{ role: string; content: string }>; +}; -// Main code entrypoint -export const runWorkflow = async (workflow: WorkflowInput) => { - return await withTrace("customer-service-demo", async () => { - const state = {}; - const conversationHistory: AgentInputItem[] = [ - { - role: "user", +// Helper to convert OpenAI message format to Agents SDK format +function convertToAgentInputItems( + messages: Array<{ role: string; content: string }> +): AgentInputItem[] { + return messages.map((msg) => { + if (msg.role === "assistant") { + // Assistant messages use output_text type + return { + role: "assistant", + content: [ + { + type: "output_text", + text: msg.content, + }, + ], + }; + } else { + // User messages use input_text type + return { + role: msg.role, content: [ { type: "input_text", - text: workflow.input_as_text, + text: msg.content, }, ], - }, - ]; + }; + } + }) as AgentInputItem[]; +} + +// Main code entrypoint +export const runWorkflow = async (workflow: WorkflowInput) => { + return await withTrace("customer-service-demo", async () => { + const state = {}; + + // Support both message array (for multi-turn) and plain text (for single-turn) + let conversationHistory: AgentInputItem[]; + if (workflow.messages && workflow.messages.length > 0) { + // Use provided message array - enables multi-turn conversations + conversationHistory = convertToAgentInputItems(workflow.messages); + } else if (workflow.input_as_text) { + // Fallback to plain text - single turn + conversationHistory = [ + { + role: "user", + content: [ + { + type: "input_text", + text: workflow.input_as_text, + }, + ], + }, + ]; + } else { + throw new Error("Must provide either 'messages' or 'input_as_text'"); + } const runner = new Runner({ traceMetadata: { __trace_source__: "agent-builder", workflow_id: "wf_68ffb83dbfc88190a38103c2bb9f421003f913035dbdb131", }, }); - const guardrailsInputtext = workflow.input_as_text; + + // Extract text for guardrails check - use last user message + const lastUserMsg = conversationHistory + .filter((m) => "role" in m && m.role === "user") + .pop(); + const guardrailsInputtext = + lastUserMsg && + "content" in lastUserMsg && + Array.isArray(lastUserMsg.content) && + lastUserMsg.content[0] && + "text" in lastUserMsg.content[0] + ? lastUserMsg.content[0].text + : ""; const guardrailsResult = await runGuardrails( guardrailsInputtext, jailbreakGuardrailConfig, @@ -285,6 +368,7 @@ export const runWorkflow = async (workflow: WorkflowInput) => { const guardrailsOutput = guardrailsHastripwire ? buildGuardrailFailOutput(guardrailsResult ?? []) : { safe_text: guardrailsAnonymizedtext ?? guardrailsInputtext }; + if (guardrailsHastripwire) { return guardrailsOutput; } else { @@ -304,6 +388,11 @@ export const runWorkflow = async (workflow: WorkflowInput) => { output_text: JSON.stringify(classificationAgentResultTemp.finalOutput), output_parsed: classificationAgentResultTemp.finalOutput, }; + + console.log( + `\n🔍 Classification: ${classificationAgentResult.output_parsed.classification}\n` + ); + if ( classificationAgentResult.output_parsed.classification == "return_item" ) { @@ -321,9 +410,13 @@ export const runWorkflow = async (workflow: WorkflowInput) => { const returnAgentResult = { output_text: returnAgentResultTemp.finalOutput ?? "", }; + + console.log(`\n💬 Return Agent: ${returnAgentResult.output_text}\n`); + const approvalMessage = "Does this work for you?"; + const approved = await approvalRequest(approvalMessage); - if (approvalRequest(approvalMessage)) { + if (approved) { const endResult = { message: "Your return is on the way.", }; @@ -352,6 +445,14 @@ export const runWorkflow = async (workflow: WorkflowInput) => { const retentionAgentResult = { output_text: retentionAgentResultTemp.finalOutput ?? "", }; + + console.log( + `\n💬 Retention Agent: ${retentionAgentResult.output_text}\n` + ); + + return { + message: retentionAgentResult.output_text, + }; } else if ( classificationAgentResult.output_parsed.classification == "get_information" @@ -370,8 +471,18 @@ export const runWorkflow = async (workflow: WorkflowInput) => { const informationAgentResult = { output_text: informationAgentResultTemp.finalOutput ?? "", }; + + console.log( + `\n💬 Information Agent: ${informationAgentResult.output_text}\n` + ); + + return { + message: informationAgentResult.output_text, + }; } else { - return classificationAgentResult; + return { + message: classificationAgentResult.output_text, + }; } } });