langfuse

1
/** @jest-environment node */
2

3
import { prisma } from "@/src/server/db";
4
import { makeAPICall, pruneDatabase } from "@/src/__tests__/test-utils";
5
import { v4 as uuidv4 } from "uuid";
6

7
describe("/api/public/scores API Endpoint", () => {
8
  beforeEach(async () => await pruneDatabase());
9
  afterEach(async () => await pruneDatabase());
10

11
  it("should create score for a trace", async () => {
12
    await pruneDatabase();
13

14
    const traceId = uuidv4();
15

16
    await makeAPICall("POST", "/api/public/traces", {
17
      id: traceId,
18
      name: "trace-name",
19
      userId: "user-1",
20
      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
21
      metadata: { key: "value" },
22
      release: "1.0.0",
23
      version: "2.0.0",
24
    });
25

26
    const dbTrace = await prisma.trace.findMany({
27
      where: {
28
        id: traceId,
29
      },
30
    });
31

32
    expect(dbTrace.length).toBeGreaterThan(0);
33
    expect(dbTrace[0]?.id).toBe(traceId);
34

35
    const scoreId = uuidv4();
36
    const createScore = await makeAPICall("POST", "/api/public/scores", {
37
      id: scoreId,
38
      name: "score-name",
39
      value: 100.5,
40
      traceId: traceId,
41
      comment: "comment",
42
    });
43

44
    expect(createScore.status).toBe(200);
45
    const dbScore = await prisma.score.findUnique({
46
      where: {
47
        id: scoreId,
48
      },
49
    });
50

51
    expect(dbScore?.id).toBe(scoreId);
52
    expect(dbScore?.traceId).toBe(traceId);
53
    expect(dbScore?.name).toBe("score-name");
54
    expect(dbScore?.value).toBe(100.5);
55
    expect(dbScore?.observationId).toBeNull();
56
    expect(dbScore?.comment).toBe("comment");
57
  });
58

59
  it("should create score for a trace with int", async () => {
60
    await pruneDatabase();
61

62
    const traceId = uuidv4();
63

64
    await makeAPICall("POST", "/api/public/traces", {
65
      id: traceId,
66
      name: "trace-name",
67
      userId: "user-1",
68
      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
69
      metadata: { key: "value" },
70
      release: "1.0.0",
71
      version: "2.0.0",
72
    });
73

74
    const dbTrace = await prisma.trace.findMany({
75
      where: {
76
        id: traceId,
77
      },
78
    });
79

80
    expect(dbTrace.length).toBeGreaterThan(0);
81
    expect(dbTrace[0]?.id).toBe(traceId);
82

83
    const scoreId = uuidv4();
84
    const createScore = await makeAPICall("POST", "/api/public/scores", {
85
      id: scoreId,
86
      name: "score-name",
87
      value: 100,
88
      traceId: traceId,
89
    });
90

91
    expect(createScore.status).toBe(200);
92
    const dbScore = await prisma.score.findUnique({
93
      where: {
94
        id: scoreId,
95
      },
96
    });
97

98
    expect(dbScore?.id).toBe(scoreId);
99
    expect(dbScore?.traceId).toBe(traceId);
100
    expect(dbScore?.name).toBe("score-name");
101
    expect(dbScore?.value).toBe(100);
102
    expect(dbScore?.observationId).toBeNull();
103
  });
104

105
  it("should create score for a generation", async () => {
106
    await pruneDatabase();
107

108
    const generationId = uuidv4();
109

110
    await makeAPICall("POST", "/api/public/generations", {
111
      id: generationId,
112
      name: "generation-name",
113
      startTime: "2021-01-01T00:00:00.000Z",
114
      endTime: "2021-01-01T00:00:00.000Z",
115
      model: "model-name",
116
      modelParameters: { key: "value" },
117
      prompt: { key: "value" },
118
      metadata: { key: "value" },
119
      version: "2.0.0",
120
    });
121

122
    const dbGeneration = await prisma.observation.findMany({
123
      where: {
124
        id: generationId,
125
      },
126
    });
127

128
    expect(dbGeneration.length).toBeGreaterThan(0);
129
    expect(dbGeneration[0]?.id).toBe(generationId);
130

131
    const scoreId = uuidv4();
132
    const createScore = await makeAPICall("POST", "/api/public/scores", {
133
      id: scoreId,
134
      name: "score-name",
135
      value: 100,
136
      traceId: dbGeneration[0]!.traceId!,
137
      observationId: dbGeneration[0]!.id,
138
    });
139

140
    expect(createScore.status).toBe(200);
141
    const dbScore = await prisma.score.findUnique({
142
      where: {
143
        id: scoreId,
144
      },
145
    });
146

147
    expect(dbScore?.id).toBe(scoreId);
148
    expect(dbScore?.traceId).toBe(dbGeneration[0]!.traceId!);
149
    expect(dbScore?.observationId).toBe(dbGeneration[0]!.id);
150
    expect(dbScore?.name).toBe("score-name");
151
    expect(dbScore?.value).toBe(100);
152
  });
153

154
  it("should upsert a score", async () => {
155
    await pruneDatabase();
156

157
    const traceId = uuidv4();
158

159
    await makeAPICall("POST", "/api/public/traces", {
160
      id: traceId,
161
      name: "trace-name",
162
      userId: "user-1",
163
      projectId: "7a88fb47-b4e2-43b8-a06c-a5ce950dc53a",
164
      metadata: { key: "value" },
165
      release: "1.0.0",
166
      version: "2.0.0",
167
    });
168

169
    const dbTrace = await prisma.trace.findMany({
170
      where: {
171
        id: traceId,
172
      },
173
    });
174

175
    expect(dbTrace.length).toBeGreaterThan(0);
176
    expect(dbTrace[0]?.id).toBe(traceId);
177

178
    const generationId = uuidv4();
179
    await makeAPICall("POST", "/api/public/generations", {
180
      id: generationId,
181
      name: "generation-name",
182
      traceId,
183
      startTime: "2021-01-01T00:00:00.000Z",
184
      endTime: "2021-01-01T00:00:00.000Z",
185
      model: "model-name",
186
      modelParameters: { key: "value" },
187
      prompt: { key: "value" },
188
      metadata: { key: "value" },
189
      version: "2.0.0",
190
    });
191

192
    const dbGeneration = await prisma.observation.findMany({
193
      where: {
194
        id: generationId,
195
      },
196
    });
197

198
    expect(dbGeneration.length).toBeGreaterThan(0);
199
    expect(dbGeneration[0]?.id).toBe(generationId);
200

201
    const scoreId = uuidv4();
202
    const createScore = await makeAPICall("POST", "/api/public/scores", {
203
      id: scoreId,
204
      name: "score-name",
205
      value: 100,
206
      traceId: traceId,
207
      comment: "comment",
208
    });
209
    expect(createScore.status).toBe(200);
210

211
    const upsertScore = await makeAPICall("POST", "/api/public/scores", {
212
      id: scoreId,
213
      traceId: traceId,
214
      name: "score-name-updated",
215
      value: 200,
216
      comment: "comment-updated",
217
      observationId: dbGeneration[0]!.id,
218
    });
219
    expect(upsertScore.status).toBe(200);
220

221
    const dbScore = await prisma.score.findUnique({
222
      where: {
223
        id: scoreId,
224
      },
225
    });
226

227
    expect(dbScore?.id).toBe(scoreId);
228
    expect(dbScore?.traceId).toBe(traceId);
229
    expect(dbScore?.name).toBe("score-name-updated");
230
    expect(dbScore?.value).toBe(200);
231
    expect(dbScore?.comment).toBe("comment-updated");
232
    expect(dbScore?.observationId).toBe(dbGeneration[0]!.id);
233
  });
234

235
  it("should delete a score", async () => {
236
    await pruneDatabase();
237

238
    const traceId = uuidv4();
239

240
    await makeAPICall("POST", "/api/public/traces", {
241
      id: traceId,
242
    });
243

244
    const scoreId = uuidv4();
245
    const createScore = await makeAPICall("POST", "/api/public/scores", {
246
      id: scoreId,
247
      name: "score-name",
248
      value: 100.5,
249
      traceId: traceId,
250
      comment: "comment",
251
    });
252

253
    expect(createScore.status).toBe(200);
254
    const dbScore = await prisma.score.findUnique({
255
      where: {
256
        id: scoreId,
257
      },
258
    });
259
    expect(dbScore?.id).toBe(scoreId);
260

261
    const deleteScore = await makeAPICall(
262
      "DELETE",
263
      `/api/public/scores/${scoreId}`,
264
    );
265
    expect(deleteScore.status).toBe(200);
266
    const deletedScore = await prisma.score.findUnique({
267
      where: {
268
        id: scoreId,
269
      },
270
    });
271
    expect(deletedScore).toBeNull();
272
  });
273

274
  it("should GET a score", async () => {
275
    await pruneDatabase();
276

277
    const traceId = uuidv4();
278

279
    await makeAPICall("POST", "/api/public/traces", {
280
      id: traceId,
281
    });
282
    const generationId = uuidv4();
283
    await makeAPICall("POST", "/api/public/generations", {
284
      id: generationId,
285
    });
286

287
    const scoreId = uuidv4();
288
    await makeAPICall("POST", "/api/public/scores", {
289
      id: scoreId,
290
      observationId: generationId,
291
      name: "score-name",
292
      value: 100.5,
293
      traceId: traceId,
294
      comment: "comment",
295
    });
296

297
    const getScore = await makeAPICall<{
298
      id: string;
299
      name: string;
300
      value: number;
301
      comment: string;
302
      traceId: string;
303
      observationId: string;
304
    }>("GET", `/api/public/scores/${scoreId}`);
305

306
    expect(getScore.status).toBe(200);
307
    expect(getScore.body).toMatchObject({
308
      id: scoreId,
309
      name: "score-name",
310
      value: 100.5,
311
      comment: "comment",
312
      traceId,
313
      observationId: generationId,
314
    });
315
  });
316
});
317