Files
app/src/__tests__/ai-tools.test.ts
BOHA 78f39e9f82 fix(odin): dedupe tool-trace chips — one per tool, warning only if all attempts failed
A failed call followed by a successful retry showed two identical chips
(one orange). Retries are loop mechanics: the trace now carries one entry
per tool with ok = delivered-at-least-once, for live chips and persisted
history alike (also keeps meta.tools under its 30-entry cap).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-06-11 00:20:43 +02:00

632 lines
21 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import { describe, it, expect, vi, beforeAll, afterAll } from "vitest";
import prisma from "../config/database";
import {
toolDefinitionsFor,
executeTool,
ctxCan,
TOOL_LABELS,
type AiAuthCtx,
} from "../services/ai-tools";
// The agentic loop talks to a true external (Anthropic API) — mock the SDK
// module; everything below it (tools, services, DB) runs for real.
const { createMock } = vi.hoisted(() => ({ createMock: vi.fn() }));
vi.mock("@anthropic-ai/sdk", () => ({
default: class MockAnthropic {
messages = { create: createMock };
},
}));
// Imported AFTER the mock so ai.service's `new Anthropic()` gets the mock.
import { agenticChat } from "../services/ai.service";
const ADMIN: AiAuthCtx = {
userId: 999999993,
roleName: "admin",
permissions: [],
};
const VIEWER_INVOICES: AiAuthCtx = {
userId: 999999994,
roleName: "viewer",
permissions: ["invoices.view"],
};
const NOBODY: AiAuthCtx = {
userId: 999999995,
roleName: "viewer",
permissions: [],
};
// People-centric fixtures (far-future dates per fixture hygiene; unique
// names/spz so re-runs can't collide with real rows). 07:0015:30 with a
// 30min break = 8.00 worked hours; trip 2 has a stored distance (10) that
// deliberately differs from end-start (8) to pin the stats-coalesce rule.
const FIX = {
username: "aitest.dominik",
email: "aitest.dominik@test.local",
first: "Dominik",
last: "AiToolsTest",
spz: "AITEST999",
note: "AiTools fixture",
role: "aitest-role",
username2: "aitest.other",
email2: "aitest.other@test.local",
};
let fixUserId = 0;
let fixUser2Id = 0;
let fixRoleId = 0;
let fixVehicleId = 0;
beforeAll(async () => {
// Defensive pre-clean of a previously failed run (Restrict FK on
// work_plan_entries.created_by means entries go before the user).
await prisma.work_plan_entries.deleteMany({ where: { note: FIX.note } });
await prisma.users.deleteMany({
where: { username: { in: [FIX.username, FIX.username2] } },
});
await prisma.roles.deleteMany({ where: { name: FIX.role } });
await prisma.vehicles.deleteMany({ where: { spz: FIX.spz } });
await prisma.ai_usage.deleteMany({
where: { user_id: { in: [ADMIN.userId, NOBODY.userId] }, kind: "agent" },
});
// find_employee scopes its population to role-permission holders (route
// parity), so the fixture user needs a role carrying attendance.record +
// trips.record (both permissions exist from migrations/seed).
const role = await prisma.roles.create({
data: { name: FIX.role, display_name: "AiTools test role" },
});
fixRoleId = role.id;
const perms = await prisma.permissions.findMany({
where: { name: { in: ["attendance.record", "trips.record"] } },
select: { id: true },
});
if (perms.length !== 2) {
throw new Error("Test setup: attendance.record/trips.record missing");
}
await prisma.role_permissions.createMany({
data: perms.map((p) => ({ role_id: fixRoleId, permission_id: p.id })),
});
const u = await prisma.users.create({
data: {
username: FIX.username,
email: FIX.email,
password_hash: "x",
first_name: FIX.first,
last_name: FIX.last,
is_active: true,
role_id: fixRoleId,
},
});
fixUserId = u.id;
// Role-less second user: invisible to scoped find_employee populations,
// off the plan grid, and the foreign-trip fixture for scoping tests.
const u2 = await prisma.users.create({
data: {
username: FIX.username2,
email: FIX.email2,
password_hash: "x",
first_name: "Otto",
last_name: "AiToolsTest",
is_active: true,
},
});
fixUser2Id = u2.id;
await prisma.attendance.create({
data: {
user_id: fixUserId,
shift_date: new Date(Date.UTC(2098, 5, 15)),
arrival_time: new Date(2098, 5, 15, 7, 0, 0),
departure_time: new Date(2098, 5, 15, 15, 30, 0),
break_start: new Date(2098, 5, 15, 11, 0, 0),
break_end: new Date(2098, 5, 15, 11, 30, 0),
leave_type: "work",
},
});
const v = await prisma.vehicles.create({
data: { spz: FIX.spz, name: "AiTest Van" },
});
fixVehicleId = v.id;
await prisma.trips.createMany({
data: [
{
vehicle_id: fixVehicleId,
user_id: fixUserId,
trip_date: new Date(Date.UTC(2098, 5, 15)),
start_km: 1000,
end_km: 1042,
route_from: "Brno",
route_to: "Praha",
is_business: true,
},
{
vehicle_id: fixVehicleId,
user_id: fixUserId,
trip_date: new Date(Date.UTC(2098, 5, 16)),
start_km: 1042,
end_km: 1050,
distance: 10,
route_from: "Praha",
route_to: "Brno",
is_business: false,
},
{
// Second user's trip in the SAME window — must be excluded by the
// non-manager self-scoping (an unscoped query would return 3 rows).
vehicle_id: fixVehicleId,
user_id: fixUser2Id,
trip_date: new Date(Date.UTC(2098, 5, 15)),
start_km: 500,
end_km: 600,
route_from: "Ostrava",
route_to: "Brno",
is_business: true,
},
],
});
await prisma.work_plan_entries.create({
data: {
user_id: fixUserId,
created_by: fixUserId,
date_from: new Date(Date.UTC(2098, 5, 15)),
date_to: new Date(Date.UTC(2098, 5, 16)),
category: "aitest-cat",
note: FIX.note,
},
});
});
afterAll(async () => {
// FK-safe order: plan entries carry a Restrict FK on created_by.
const fixUserIds = [fixUserId, fixUser2Id];
await prisma.work_plan_entries.deleteMany({
where: { user_id: { in: fixUserIds } },
});
await prisma.trips.deleteMany({ where: { user_id: { in: fixUserIds } } });
await prisma.attendance.deleteMany({
where: { user_id: { in: fixUserIds } },
});
await prisma.users.deleteMany({ where: { id: { in: fixUserIds } } });
await prisma.roles.deleteMany({ where: { id: fixRoleId } });
await prisma.vehicles.deleteMany({ where: { id: fixVehicleId } });
await prisma.ai_usage.deleteMany({
where: { user_id: { in: [ADMIN.userId, NOBODY.userId] }, kind: "agent" },
});
});
describe("ai-tools — permission delegation", () => {
it("ctxCan: admin bypasses, others need the exact permission", () => {
expect(ctxCan(ADMIN, "invoices.view")).toBe(true);
expect(ctxCan(VIEWER_INVOICES, "invoices.view")).toBe(true);
expect(ctxCan(VIEWER_INVOICES, "orders.view")).toBe(false);
expect(ctxCan(NOBODY, "invoices.view")).toBe(false);
});
it("toolDefinitionsFor filters the tool list by the user's permissions", () => {
const adminTools = toolDefinitionsFor(ADMIN).map((t) => t.name);
// All registered tools (the labels map is the authoritative name list).
expect(adminTools.sort()).toEqual(Object.keys(TOOL_LABELS).sort());
const viewerTools = toolDefinitionsFor(VIEWER_INVOICES).map((t) => t.name);
expect(viewerTools).toContain("list_invoices");
expect(viewerTools).toContain("get_invoice_stats");
expect(viewerTools).not.toContain("list_orders");
expect(viewerTools).not.toContain("get_stock_overview");
expect(toolDefinitionsFor(NOBODY)).toHaveLength(0);
});
it("executeTool DENIES a tool the user lacks the permission for", async () => {
const res = await executeTool("list_orders", {}, VIEWER_INVOICES);
expect(res.ok).toBe(false);
expect(JSON.stringify(res.result)).toContain("oprávnění");
});
it("executeTool rejects an unknown (hallucinated) tool name", async () => {
const res = await executeTool("drop_all_tables", {}, ADMIN);
expect(res.ok).toBe(false);
expect(JSON.stringify(res.result)).toContain("Neznámý nástroj");
});
it("get_attendance_summary: another user's data needs attendance.manage", async () => {
const self: AiAuthCtx = {
userId: 999999996,
roleName: "viewer",
permissions: ["attendance.record"],
};
// Own data — allowed (empty month is fine, shape matters).
const own = await executeTool("get_attendance_summary", {}, self);
expect(own.ok).toBe(true);
expect(own.result).toMatchObject({ user_id: self.userId });
// Someone else's data without attendance.manage — denied in-result, and
// executeTool flags handler-level { error } results as not-ok so denial
// chips render consistently.
const other = await executeTool(
"get_attendance_summary",
{ user_id: 1 },
self,
);
expect(other.ok).toBe(false);
expect(JSON.stringify(other.result)).toContain("oprávnění");
});
});
describe("ai-tools — employees, attendance detail, trips, work plan", () => {
const RECORDER: AiAuthCtx = {
userId: 999999997,
roleName: "viewer",
permissions: ["attendance.record"],
};
it("find_employee resolves a name (and full name) to user_id", async () => {
const res = await executeTool(
"find_employee",
{ search: FIX.last },
RECORDER,
);
expect(res.ok).toBe(true);
const r = res.result as {
employees: {
user_id: number;
name: string;
username?: string;
active: boolean;
}[];
};
const hit = r.employees.find((e) => e.user_id === fixUserId);
// Attendance callers see usernames (GET /plan/users parity).
expect(hit).toMatchObject({
name: `${FIX.first} ${FIX.last}`,
username: FIX.username,
active: true,
});
// The role-less second user is OUTSIDE the attendance.record population
// (route parity with listPlanUsers) — must not be returned.
expect(r.employees.some((e) => e.user_id === fixUser2Id)).toBe(false);
const full = await executeTool(
"find_employee",
{ search: `${FIX.first} ${FIX.last}` },
RECORDER,
);
expect(
(full.result as { employees: { user_id: number }[] }).employees.some(
(e) => e.user_id === fixUserId,
),
).toBe(true);
});
it("find_employee: trips callers get no usernames; trips.history alone is denied", async () => {
// GET /trips/users parity: drivers' picker has no usernames.
const trips = await executeTool(
"find_employee",
{ search: FIX.last },
{ userId: 999999998, roleName: "viewer", permissions: ["trips.record"] },
);
expect(trips.ok).toBe(true);
const t = trips.result as {
employees: { user_id: number; username?: string }[];
};
const hit = t.employees.find((e) => e.user_id === fixUserId);
expect(hit).toBeDefined();
expect(hit).not.toHaveProperty("username");
// trips.history grants no picker route → no find_employee either.
const hist = await executeTool(
"find_employee",
{ search: FIX.last },
{ userId: 999999998, roleName: "viewer", permissions: ["trips.history"] },
);
expect(hist.ok).toBe(false);
});
it("find_employee is denied without any people-related permission", async () => {
const res = await executeTool("find_employee", { search: "x" }, NOBODY);
expect(res.ok).toBe(false);
expect(toolDefinitionsFor(NOBODY).map((t) => t.name)).not.toContain(
"find_employee",
);
});
it("find_employee: only users.view sees the full directory (role-less users)", async () => {
// ADMIN bypasses ctxCan → full-directory branch finds the role-less user.
const admin = await executeTool("find_employee", { search: "Otto" }, ADMIN);
expect(
(admin.result as { employees: { user_id: number }[] }).employees.some(
(e) => e.user_id === fixUser2Id,
),
).toBe(true);
});
it("get_attendance_summary returns day detail with worked hours for a date range", async () => {
const res = await executeTool(
"get_attendance_summary",
{ user_id: fixUserId, date_from: "2098-06-15", date_to: "2098-06-15" },
ADMIN,
);
expect(res.ok).toBe(true);
const r = res.result as {
days: {
date: string;
arrival: string | null;
departure: string | null;
worked_hours: number | null;
}[];
worked_hours_total: number;
days_by_type: Record<string, number>;
};
expect(r.days).toHaveLength(1);
expect(r.days[0]).toMatchObject({
date: "2098-06-15",
arrival: "07:00",
departure: "15:30",
worked_hours: 8, // 8.5 h minus 30 min break
});
expect(r.worked_hours_total).toBe(8);
expect(r.days_by_type).toEqual({ work: 1 });
});
it("list_trips: manager filter + km totals with the stats coalesce; non-managers scoped to self", async () => {
const mgr = await executeTool(
"list_trips",
{ user_id: fixUserId, date_from: "2098-06-15", date_to: "2098-06-16" },
ADMIN,
);
expect(mgr.ok).toBe(true);
const m = mgr.result as {
total_matching: number;
total_km: number;
business_km: number;
private_km: number;
trips: { km: number; date: string }[];
};
expect(m.total_matching).toBe(2);
expect(m.business_km).toBe(42);
expect(m.private_km).toBe(10); // stored distance wins over end-start (8)
expect(m.total_km).toBe(52);
expect(m.trips[0].date).toBe("2098-06-16"); // newest first
// Non-manager asking for someone else's trips → denied.
const denied = await executeTool(
"list_trips",
{ user_id: fixUserId },
{ userId: 999999998, roleName: "viewer", permissions: ["trips.record"] },
);
expect(denied.ok).toBe(false);
expect(JSON.stringify(denied.result)).toContain("oprávnění");
// Non-manager without user_id is hard-scoped to their own trips: the
// second user's trip sits in the same window, so an unscoped query
// would return 3 — the scoping must yield exactly the caller's 2.
const own = await executeTool(
"list_trips",
{ date_from: "2098-06-15", date_to: "2098-06-16" },
{ userId: fixUserId, roleName: "viewer", permissions: ["trips.history"] },
);
expect(own.ok).toBe(true);
const o = own.result as { total_matching: number; total_km: number };
expect(o.total_matching).toBe(2);
expect(o.total_km).toBe(52); // foreign 100km trip excluded from totals too
});
it("get_work_plan resolves the range-entry into per-day rows", async () => {
const res = await executeTool(
"get_work_plan",
{ user_id: fixUserId, date_from: "2098-06-15", date_to: "2098-06-17" },
RECORDER,
);
expect(res.ok).toBe(true);
const r = res.result as {
plan: { date: string; user: string; plan: string; note: string }[];
records: number;
};
// The entry spans 15.16. → exactly two resolved days, none on the 17th.
expect(r.records).toBe(2);
expect(r.plan[0]).toMatchObject({
date: "2098-06-15",
user: `${FIX.first} ${FIX.last}`,
plan: "aitest-cat", // no plan_categories row → raw key fallback
note: FIX.note,
});
expect(r.plan[1].date).toBe("2098-06-16");
});
it("get_work_plan: off-grid users need attendance.manage (grid-route parity)", async () => {
// The role-less second user is not a plan-grid user → a bare
// attendance.record caller is denied...
const denied = await executeTool(
"get_work_plan",
{ user_id: fixUser2Id, date_from: "2098-06-15", date_to: "2098-06-16" },
RECORDER,
);
expect(denied.ok).toBe(false);
expect(JSON.stringify(denied.result)).toContain("oprávnění");
// ...while a manager/admin may read them (empty plan, no error).
const admin = await executeTool(
"get_work_plan",
{ user_id: fixUser2Id, date_from: "2098-06-15", date_to: "2098-06-16" },
ADMIN,
);
expect(admin.ok).toBe(true);
expect((admin.result as { records: number }).records).toBe(0);
});
});
describe("ai-tools — real reads", () => {
it("list_invoices returns the compact shape from the real DB", async () => {
const res = await executeTool("list_invoices", {}, ADMIN);
expect(res.ok).toBe(true);
const r = res.result as {
total_matching: number;
shown: number;
invoices: unknown[];
};
expect(typeof r.total_matching).toBe("number");
expect(Array.isArray(r.invoices)).toBe(true);
expect(r.shown).toBe(r.invoices.length);
expect(r.shown).toBeLessThanOrEqual(20);
});
it("list_projects + get_stock_overview return their shapes", async () => {
const projects = await executeTool("list_projects", {}, ADMIN);
expect(projects.ok).toBe(true);
expect(
Array.isArray((projects.result as { projects: unknown[] }).projects),
).toBe(true);
const stock = await executeTool("get_stock_overview", {}, ADMIN);
expect(stock.ok).toBe(true);
expect(
typeof (stock.result as { below_minimum_count: number })
.below_minimum_count,
).toBe("number");
});
});
describe("agenticChat loop (SDK mocked)", () => {
it("executes a requested tool, feeds the result back, returns final text + trace", async () => {
createMock.mockReset();
createMock
.mockResolvedValueOnce({
stop_reason: "tool_use",
content: [
{ type: "text", text: "Podívám se." },
{
type: "tool_use",
id: "toolu_1",
name: "list_projects",
input: {},
},
],
usage: { input_tokens: 100, output_tokens: 20 },
})
.mockResolvedValueOnce({
stop_reason: "end_turn",
content: [{ type: "text", text: "Máte 2 projekty." }],
usage: { input_tokens: 200, output_tokens: 30 },
});
const res = await agenticChat(
[{ role: "user", content: "Projekty?" }],
ADMIN,
);
expect(res.reply).toBe("Máte 2 projekty.");
expect(res.toolTrace).toEqual([
{ name: "list_projects", label: "Projekty", ok: true },
]);
expect(createMock).toHaveBeenCalledTimes(2);
// Second call must carry the tool_result back to the model.
const secondCall = createMock.mock.calls[1][0];
const lastMsg = secondCall.messages[secondCall.messages.length - 1];
expect(lastMsg.role).toBe("user");
expect(lastMsg.content[0]).toMatchObject({
type: "tool_result",
tool_use_id: "toolu_1",
});
// Usage was recorded per round-trip.
const usageRows = await prisma.ai_usage.findMany({
where: { user_id: ADMIN.userId, kind: "agent" },
});
expect(usageRows.length).toBe(2);
});
it("dedupes the tool trace: failed attempt + successful retry = one ok chip", async () => {
const self: AiAuthCtx = {
userId: 999999996,
roleName: "viewer",
permissions: ["attendance.record"],
};
createMock.mockReset();
createMock
.mockResolvedValueOnce({
stop_reason: "tool_use",
content: [
{
type: "tool_use",
id: "toolu_a",
name: "get_attendance_summary",
input: { user_id: 1 }, // denied: someone else without manage
},
],
usage: { input_tokens: 10, output_tokens: 5 },
})
.mockResolvedValueOnce({
stop_reason: "tool_use",
content: [
{
type: "tool_use",
id: "toolu_b",
name: "get_attendance_summary",
input: {}, // retry: own data, succeeds
},
],
usage: { input_tokens: 10, output_tokens: 5 },
})
.mockResolvedValueOnce({
stop_reason: "end_turn",
content: [{ type: "text", text: "Hotovo." }],
usage: { input_tokens: 10, output_tokens: 5 },
});
const res = await agenticChat(
[{ role: "user", content: "Moje docházka?" }],
self,
);
expect(res.toolTrace).toEqual([
{ name: "get_attendance_summary", label: "Docházka", ok: true },
]);
await prisma.ai_usage.deleteMany({
where: { user_id: self.userId, kind: "agent" },
});
});
it("injects the caller's identity into the system prompt", async () => {
createMock.mockReset();
createMock.mockResolvedValueOnce({
stop_reason: "end_turn",
content: [{ type: "text", text: "Ahoj." }],
usage: { input_tokens: 10, output_tokens: 5 },
});
await agenticChat([{ role: "user", content: "Kdo jsem?" }], {
userId: fixUserId,
roleName: "viewer",
permissions: ["attendance.record"],
});
const system: string = createMock.mock.calls[0][0].system;
expect(system).toContain(`${FIX.first} ${FIX.last}`);
expect(system).toContain(`user_id ${fixUserId}`);
// Cleanup the usage row this extra fixture-user turn recorded.
await prisma.ai_usage.deleteMany({
where: { user_id: fixUserId, kind: "agent" },
});
});
it("a user without permissions gets NO tools passed to the model", async () => {
createMock.mockReset();
createMock.mockResolvedValueOnce({
stop_reason: "end_turn",
content: [{ type: "text", text: "Ahoj." }],
usage: { input_tokens: 10, output_tokens: 5 },
});
await agenticChat([{ role: "user", content: "Ahoj" }], NOBODY);
expect(createMock.mock.calls[0][0].tools).toBeUndefined();
});
it("stops at the iteration cap with an explanatory note", async () => {
createMock.mockReset();
createMock.mockResolvedValue({
stop_reason: "tool_use",
content: [
{ type: "tool_use", id: "toolu_x", name: "list_projects", input: {} },
],
usage: { input_tokens: 10, output_tokens: 5 },
});
const res = await agenticChat([{ role: "user", content: "smyčka" }], ADMIN);
expect(createMock.mock.calls.length).toBe(6); // MAX_AGENT_ITERATIONS
expect(res.reply).toContain("příliš složitý");
});
});