From a3b2e0bd27d333734b4763ee1fc32aef21a29e87 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Fri, 26 Apr 2024 23:15:59 +0200 Subject: [PATCH 01/19] add clickhouse docker --- docker-compose.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 071a43cc..178dd7ac 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -34,6 +34,13 @@ services: ports: - "1025:1025" - "8025:8025" + clickhouse: + image: clickhouse/clickhouse-server + restart: always + ports: + - "8123:8123" + - "9000:9000" + - "9009:9009" # Names our volume volumes: my-db: From 33c2a91b5a61f7982dfb0ad89b15b4d9c44c9a4f Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Fri, 5 Jul 2024 01:20:27 +0200 Subject: [PATCH 02/19] wip --- apps/web/clickhouse/createDatabase.ts | 34 +++++ apps/web/clickhouse/setupDB.ts | 123 ++++++++++++++++++ apps/web/package.json | 5 +- apps/web/src/components/Test/Section.tsx | 2 + .../projects/[projectId]/tests/[testId].tsx | 2 + apps/web/src/server/db/clickhouseClient.ts | 14 ++ .../server/services/ClickHouseEventService.ts | 51 ++++++++ apps/web/src/server/services/EventService.ts | 16 ++- pnpm-lock.yaml | 14 ++ 9 files changed, 259 insertions(+), 2 deletions(-) create mode 100644 apps/web/clickhouse/createDatabase.ts create mode 100644 apps/web/clickhouse/setupDB.ts create mode 100644 apps/web/src/server/db/clickhouseClient.ts create mode 100644 apps/web/src/server/services/ClickHouseEventService.ts diff --git a/apps/web/clickhouse/createDatabase.ts b/apps/web/clickhouse/createDatabase.ts new file mode 100644 index 00000000..99504489 --- /dev/null +++ b/apps/web/clickhouse/createDatabase.ts @@ -0,0 +1,34 @@ +import { createClient } from "@clickhouse/client"; + +const client = createClient({ + url: "http://localhost:8123", +}); + +client + .command({ + query: "CREATE DATABASE IF NOT EXISTS abby", + }) + .then((res) => { + client.command({ + query: ` + DROP TABLE IF EXISTS abby.Event; + `, + }); + }) + .then((res) => { + client.command({ + query: ` + CREATE TABLE IF NOT EXISTS abby.Event ( + id String, + project_id String, + testName String, + type Int, + selectedVariant String, + createdAt DateTime DEFAULT toDateTime(now()) NOT NULL, + + ) + ENGINE = MergeTree() + ORDER BY (project_id, testName) + `, + }); + }); diff --git a/apps/web/clickhouse/setupDB.ts b/apps/web/clickhouse/setupDB.ts new file mode 100644 index 00000000..84604a1e --- /dev/null +++ b/apps/web/clickhouse/setupDB.ts @@ -0,0 +1,123 @@ +import { ResultSet, createClient } from "@clickhouse/client"; +import { AbbyEvent, AbbyEventType } from "@tryabby/core"; +import { coolGray } from "tailwindcss/colors"; +import { z } from "zod"; + +const client = createClient({ + url: "http://localhost:8123", +}); +async function getVariantCount(projectId: string, testId: string) { + const res = await client.query({ + query: `SELECT COUNT(selectedVariant) AS variant_count, testName, selectedVariant, type + FROM abby.Event + WHERE project_id = '${projectId}' + + GROUP BY testName, selectedVariant, type + `, + format: "JSONEachRow", + }); + return res; +} + +let dataToInsert = [ + { + project_id: "12321", + testName: "footer", + type: AbbyEventType.ACT, + selectedVariant: "A", + }, + { + project_id: "12321", + testName: "footer", + type: AbbyEventType.PING, + selectedVariant: "A", + }, + { + project_id: "12321", + testName: "footer", + type: AbbyEventType.ACT, + selectedVariant: "B", + }, + { + project_id: "12321", + testName: "footer", + type: AbbyEventType.PING, + selectedVariant: "B", + }, + { + project_id: "123", + testName: "footer", + type: AbbyEventType.ACT, + selectedVariant: "A", + }, + { + project_id: "123", + testName: "footer", + type: AbbyEventType.PING, + selectedVariant: "A", + }, + { + project_id: "123", + testName: "footer", + type: AbbyEventType.ACT, + selectedVariant: "B", + }, + { + project_id: "123", + testName: "footer", + type: AbbyEventType.PING, + selectedVariant: "B", + }, +]; + +for (let i = 0; i < 12; i++) { + dataToInsert = dataToInsert.concat(dataToInsert); +} + +console.log(dataToInsert.length); + +async function test() { + const beforeCount = await client + .query({ + query: "SELECT COUNT(*) FROM abby.Event", + format: "JSONEachRow", + }) + .then((data) => data.json()) + .then((data) => data) + .catch((err) => console.log(err)); + + console.log("before", beforeCount); + console.time("insert"); + + for (let x = 0; x < 10_000; x++) { + await client.insert({ + table: "abby.Event", + values: dataToInsert, + format: "JSONEachRow", + }); + } + + console.timeEnd("insert"); + console.log("done insert"); + + const afterCount = await client + .query({ + query: "SELECT COUNT(*) FROM abby.Event", + format: "JSONEachRow", + }) + .then((data) => data.json()) + .then((data) => data) + .catch((err) => console.log(err)); + + console.log("after", afterCount); +} + +// console.time("insert"); +// test(); +// console.timeEnd("insert"); + +console.time("query"); +getVariantCount("12321", "footer").then((data) => { + data.json().then((data) => console.log(data)); + console.timeEnd("query"); +}); diff --git a/apps/web/package.json b/apps/web/package.json index 9b1fb62b..231b8103 100644 --- a/apps/web/package.json +++ b/apps/web/package.json @@ -14,12 +14,15 @@ "seed:events": "ts-node --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} prisma/seedEvents.ts", "db:migrate": "prisma migrate dev", "generate:coupons": "pnpm ts-node -r tsconfig-paths/register --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} prisma/generateCoupons.ts", + "clickhouse:migrate": "pnpm ts-node -r tsconfig-paths/register --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} clickhouse/setupDB.ts", + "clickhouse:create": "pnpm ts-node -r tsconfig-paths/register --compiler-options {\\\"module\\\":\\\"CommonJS\\\"} clickhouse/createDatabase.ts", "mailhog:up": "docker-compose -f docker-compose.mailhog.yaml up", "mailhog:down": "docker-compose -f docker-compose.mailhog.yaml down", "test": "vitest", "start:docker": "npx prisma migrate deploy || { echo 'Migration failed, exiting'; exit 1; } && node server.js" }, "dependencies": { + "@clickhouse/client": "^1.0.1", "@code-hike/mdx": "0.9.0", "@databases/cache": "^1.0.0", "@dnd-kit/core": "^6.0.8", @@ -152,4 +155,4 @@ "ct3aMetadata": { "initVersion": "6.11.1" } -} \ No newline at end of file +} diff --git a/apps/web/src/components/Test/Section.tsx b/apps/web/src/components/Test/Section.tsx index 5b435185..03c7209b 100644 --- a/apps/web/src/components/Test/Section.tsx +++ b/apps/web/src/components/Test/Section.tsx @@ -154,6 +154,8 @@ const Section = ({ }, }); + console.log(events); + return (
diff --git a/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx b/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx index 03b8fcca..5ed76021 100644 --- a/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx +++ b/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx @@ -141,6 +141,8 @@ const TestDetailPage: NextPageWithLayout = () => { }); }, [eventsByVariant, interval]); + console.log("formattedEvents", formattedEvents); + const viewEvents = useMemo( () => ({ labels, diff --git a/apps/web/src/server/db/clickhouseClient.ts b/apps/web/src/server/db/clickhouseClient.ts new file mode 100644 index 00000000..5032801b --- /dev/null +++ b/apps/web/src/server/db/clickhouseClient.ts @@ -0,0 +1,14 @@ +import { NodeClickHouseClient } from "@clickhouse/client/dist/client"; +import { env } from "../../env/server.mjs"; +import { createClient } from "@clickhouse/client"; + +declare global { + // eslint-disable-next-line no-var + var clickhouseClient: NodeClickHouseClient | undefined; +} + +export const clickhouseClient = global.clickhouseClient || createClient(); + +if (env.NODE_ENV !== "production") { + global.clickhouseClient = clickhouseClient; +} diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts new file mode 100644 index 00000000..4b2cbd43 --- /dev/null +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -0,0 +1,51 @@ +import { AbbyEventType } from "@tryabby/core"; +import { Limit } from "server/common/plans"; +import { EventService, EventServiceInterface } from "./EventService"; +import { clickhouseClient } from "server/db/clickhouseClient"; + +export abstract class ClickHouseEventService implements EventServiceInterface { + async createEvent(event: { + type: AbbyEventType; + projectId: string; + testName: string; + selectedVariant: string; + }) { + const insertedEvent = await clickhouseClient.insert({ + table: "events", + values: { + type: event.type, + projectId: event.projectId, + testName: event.testName, + selectedVariant: event.selectedVariant, + }, + }); + + return event; + } + async getEventsByProjectId(projectId: string) { + const res = await clickhouseClient.query({ + query: `SELECT * FROM events WHERE projectId = '${projectId}'`, + }); + + const resultData = await res.json().then((data) => data); + + return resultData as any[]; + } + getEventsByTestId(testId: string, timeInterval: string) { + throw new Error("Method not implemented."); + } + getEventsForCurrentPeriod(projectId: string): Promise<{ + events: number; + planLimits: Limit; + plan: + | "STARTUP" + | "PRO" + | "ENTERPRISE" + | "BETA" + | "STARTUP_LIFETIME" + | undefined; + is80PercentOfLimit: boolean; + }> { + throw new Error("Method not implemented."); + } +} diff --git a/apps/web/src/server/services/EventService.ts b/apps/web/src/server/services/EventService.ts index dba9ace5..ab5933aa 100644 --- a/apps/web/src/server/services/EventService.ts +++ b/apps/web/src/server/services/EventService.ts @@ -5,11 +5,25 @@ import { SpecialTimeInterval, } from "lib/events"; import ms from "ms"; -import { getLimitByPlan, PlanName, PLANS } from "server/common/plans"; +import { getLimitByPlan, Limit, PlanName, PLANS } from "server/common/plans"; import { prisma } from "server/db/client"; import { AbbyEvent } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; +type EventPeriodData = { + events: number; + planLimits: Limit; + plan: PlanName | undefined; + is80PercentOfLimit: boolean; +}; + +export interface EventServiceInterface { + createEvent(event: AbbyEvent): Promise; + getEventsByProjectId(projectId: string): Promise; + getEventsByTestId(testId: string, timeInterval: string): Promise; + getEventsForCurrentPeriod(projectId: string): Promise; +} + export abstract class EventService { static async createEvent({ projectId, diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 993b502d..81cd21f4 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -188,6 +188,9 @@ importers: apps/web: dependencies: + '@clickhouse/client': + specifier: ^1.0.1 + version: 1.0.1 '@code-hike/mdx': specifier: 0.9.0 version: 0.9.0(react@18.2.0) @@ -5427,6 +5430,17 @@ packages: prettier: 2.8.8 dev: true + /@clickhouse/client-common@1.0.1: + resolution: {integrity: sha512-3L6e0foP6VOktScoi6XWMjJyOpKCWgLUYgPVxP2c7gm6Kotq+iRmmmXtXTSg7B7uozcLZycTtPfIw2d80SYsYw==} + dev: false + + /@clickhouse/client@1.0.1: + resolution: {integrity: sha512-fluUNnE2R7COJ6rn6DorYfi4D+AQn3t2qeBtEq37bQV3pD4EbKrBfKAwJ13e1lmMWdQ2B9bJUTMqGsRIDdWhJw==} + engines: {node: '>=16'} + dependencies: + '@clickhouse/client-common': 1.0.1 + dev: false + /@cloudflare/kv-asset-handler@0.2.0: resolution: {integrity: sha512-MVbXLbTcAotOPUj0pAMhVtJ+3/kFkwJqc5qNOleOZTv6QkZZABDMS21dSrSlVswEHwrpWC03e4fWytjqKvuE2A==} dependencies: From 0d5aac60b16ac80983b0c7c88ef7dcb6e19a9844 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Fri, 5 Jul 2024 01:57:47 +0200 Subject: [PATCH 03/19] Save event in clickhouse --- apps/web/clickhouse/setupDB.ts | 123 ----------------- apps/web/src/components/Test/Section.tsx | 2 - apps/web/src/server/queue/event.ts | 7 +- .../server/services/ClickHouseEventService.ts | 128 +++++++++++++----- apps/web/src/server/services/EventService.ts | 13 +- 5 files changed, 103 insertions(+), 170 deletions(-) delete mode 100644 apps/web/clickhouse/setupDB.ts diff --git a/apps/web/clickhouse/setupDB.ts b/apps/web/clickhouse/setupDB.ts deleted file mode 100644 index 84604a1e..00000000 --- a/apps/web/clickhouse/setupDB.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { ResultSet, createClient } from "@clickhouse/client"; -import { AbbyEvent, AbbyEventType } from "@tryabby/core"; -import { coolGray } from "tailwindcss/colors"; -import { z } from "zod"; - -const client = createClient({ - url: "http://localhost:8123", -}); -async function getVariantCount(projectId: string, testId: string) { - const res = await client.query({ - query: `SELECT COUNT(selectedVariant) AS variant_count, testName, selectedVariant, type - FROM abby.Event - WHERE project_id = '${projectId}' - - GROUP BY testName, selectedVariant, type - `, - format: "JSONEachRow", - }); - return res; -} - -let dataToInsert = [ - { - project_id: "12321", - testName: "footer", - type: AbbyEventType.ACT, - selectedVariant: "A", - }, - { - project_id: "12321", - testName: "footer", - type: AbbyEventType.PING, - selectedVariant: "A", - }, - { - project_id: "12321", - testName: "footer", - type: AbbyEventType.ACT, - selectedVariant: "B", - }, - { - project_id: "12321", - testName: "footer", - type: AbbyEventType.PING, - selectedVariant: "B", - }, - { - project_id: "123", - testName: "footer", - type: AbbyEventType.ACT, - selectedVariant: "A", - }, - { - project_id: "123", - testName: "footer", - type: AbbyEventType.PING, - selectedVariant: "A", - }, - { - project_id: "123", - testName: "footer", - type: AbbyEventType.ACT, - selectedVariant: "B", - }, - { - project_id: "123", - testName: "footer", - type: AbbyEventType.PING, - selectedVariant: "B", - }, -]; - -for (let i = 0; i < 12; i++) { - dataToInsert = dataToInsert.concat(dataToInsert); -} - -console.log(dataToInsert.length); - -async function test() { - const beforeCount = await client - .query({ - query: "SELECT COUNT(*) FROM abby.Event", - format: "JSONEachRow", - }) - .then((data) => data.json()) - .then((data) => data) - .catch((err) => console.log(err)); - - console.log("before", beforeCount); - console.time("insert"); - - for (let x = 0; x < 10_000; x++) { - await client.insert({ - table: "abby.Event", - values: dataToInsert, - format: "JSONEachRow", - }); - } - - console.timeEnd("insert"); - console.log("done insert"); - - const afterCount = await client - .query({ - query: "SELECT COUNT(*) FROM abby.Event", - format: "JSONEachRow", - }) - .then((data) => data.json()) - .then((data) => data) - .catch((err) => console.log(err)); - - console.log("after", afterCount); -} - -// console.time("insert"); -// test(); -// console.timeEnd("insert"); - -console.time("query"); -getVariantCount("12321", "footer").then((data) => { - data.json().then((data) => console.log(data)); - console.timeEnd("query"); -}); diff --git a/apps/web/src/components/Test/Section.tsx b/apps/web/src/components/Test/Section.tsx index 03c7209b..5b435185 100644 --- a/apps/web/src/components/Test/Section.tsx +++ b/apps/web/src/components/Test/Section.tsx @@ -154,8 +154,6 @@ const Section = ({ }, }); - console.log(events); - return (
diff --git a/apps/web/src/server/queue/event.ts b/apps/web/src/server/queue/event.ts index 3215e6a2..9bbc7982 100644 --- a/apps/web/src/server/queue/event.ts +++ b/apps/web/src/server/queue/event.ts @@ -7,6 +7,7 @@ import { eventQueue, getQueueingRedisConnection } from "./queues"; import { AbbyEvent, AbbyEventType } from "@tryabby/core"; import { env } from "env/server.mjs"; import { ApiRequestType } from "@prisma/client"; +import { ClickHouseEventService } from "server/services/ClickHouseEventService"; export type EventJobPayload = AbbyEvent & { functionDuration: number; @@ -24,7 +25,11 @@ const eventWorker = new Worker( switch (event.type) { case AbbyEventType.PING: case AbbyEventType.ACT: { - await EventService.createEvent(event); + await Promise.all([ + EventService.createEvent(event), + ClickHouseEventService.createEvent(event), + ]); + break; } default: { diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index 4b2cbd43..d2361684 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -1,51 +1,111 @@ -import { AbbyEventType } from "@tryabby/core"; -import { Limit } from "server/common/plans"; -import { EventService, EventServiceInterface } from "./EventService"; +import dayjs from "dayjs"; +import { + getMSFromSpecialTimeInterval, + isSpecialTimeInterval, + SpecialTimeInterval, +} from "lib/events"; +import ms from "ms"; +import { getLimitByPlan, Limit, PlanName, PLANS } from "server/common/plans"; +import { prisma } from "server/db/client"; +import { AbbyEvent, AbbyEventType } from "@tryabby/core"; +import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; -export abstract class ClickHouseEventService implements EventServiceInterface { - async createEvent(event: { +export abstract class ClickHouseEventService { + static async createEvent(event: { type: AbbyEventType; projectId: string; testName: string; selectedVariant: string; }) { const insertedEvent = await clickhouseClient.insert({ - table: "events", - values: { - type: event.type, - projectId: event.projectId, - testName: event.testName, - selectedVariant: event.selectedVariant, - }, + table: "abby.events", + format: "JSONEachRow", + values: [ + { + type: event.type, + projectId: event.projectId, + testName: event.testName, + selectedVariant: event.selectedVariant, + }, + ], }); - return event; + return insertedEvent; } - async getEventsByProjectId(projectId: string) { - const res = await clickhouseClient.query({ - query: `SELECT * FROM events WHERE projectId = '${projectId}'`, + + static async getEventsByProjectId(projectId: string) { + return prisma.event.findMany({ + where: { + test: { + projectId, + }, + }, }); + } - const resultData = await res.json().then((data) => data); + static async getEventsByTestId(testId: string, timeInterval: string) { + const now = new Date().getTime(); - return resultData as any[]; - } - getEventsByTestId(testId: string, timeInterval: string) { - throw new Error("Method not implemented."); + if (isSpecialTimeInterval(timeInterval)) { + const specialIntervalInMs = getMSFromSpecialTimeInterval(timeInterval); + return prisma.event.findMany({ + where: { + testId, + ...(specialIntervalInMs !== Infinity && + timeInterval !== SpecialTimeInterval.DAY && { + createdAt: { + gte: new Date(now - getMSFromSpecialTimeInterval(timeInterval)), + }, + }), + // Special case for day, since we want to include the current day + ...(timeInterval === SpecialTimeInterval.DAY && { + createdAt: { + gte: dayjs().startOf("day").toDate(), + }, + }), + }, + }); + } + + const parsedInterval = ms(timeInterval) as number | undefined; + + if (parsedInterval === undefined) { + throw new Error("Invalid time interval"); + } + + return prisma.event.findMany({ + where: { + testId, + createdAt: { + gte: new Date(now - ms(timeInterval)), + }, + }, + }); } - getEventsForCurrentPeriod(projectId: string): Promise<{ - events: number; - planLimits: Limit; - plan: - | "STARTUP" - | "PRO" - | "ENTERPRISE" - | "BETA" - | "STARTUP_LIFETIME" - | undefined; - is80PercentOfLimit: boolean; - }> { - throw new Error("Method not implemented."); + + static async getEventsForCurrentPeriod(projectId: string) { + const [project, eventCount] = await Promise.all([ + prisma.project.findUnique({ + where: { id: projectId }, + select: { stripePriceId: true }, + }), + RequestCache.get(projectId), + ]); + + if (!project) throw new Error("Project not found"); + + const plan = Object.keys(PLANS).find( + (plan) => PLANS[plan as PlanName] === project.stripePriceId + ) as PlanName | undefined; + + const planLimits = getLimitByPlan(plan ?? null); + + return { + events: eventCount, + planLimits, + plan, + is80PercentOfLimit: planLimits.eventsPerMonth * 0.8 === eventCount, + }; } } diff --git a/apps/web/src/server/services/EventService.ts b/apps/web/src/server/services/EventService.ts index ab5933aa..7c5ac41f 100644 --- a/apps/web/src/server/services/EventService.ts +++ b/apps/web/src/server/services/EventService.ts @@ -10,18 +10,11 @@ import { prisma } from "server/db/client"; import { AbbyEvent } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; -type EventPeriodData = { - events: number; - planLimits: Limit; - plan: PlanName | undefined; - is80PercentOfLimit: boolean; -}; - export interface EventServiceInterface { createEvent(event: AbbyEvent): Promise; - getEventsByProjectId(projectId: string): Promise; - getEventsByTestId(testId: string, timeInterval: string): Promise; - getEventsForCurrentPeriod(projectId: string): Promise; + // getEventsByProjectId(projectId: string): Promise; + // getEventsByTestId(testId: string, timeInterval: string): Promise; + // getEventsForCurrentPeriod(projectId: string): Promise; } export abstract class EventService { From e9083a2fed9652834dd54a14911f277f9deb9dc7 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Fri, 5 Jul 2024 02:39:06 +0200 Subject: [PATCH 04/19] Add get Event by projectID --- .../server/services/ClickHouseEventService.ts | 20 ++++++++++++------- apps/web/src/server/services/EventService.ts | 8 -------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index d2361684..e48b78c4 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -34,14 +34,20 @@ export abstract class ClickHouseEventService { return insertedEvent; } - static async getEventsByProjectId(projectId: string) { - return prisma.event.findMany({ - where: { - test: { - projectId, - }, - }, + static async getEventsByProjectId(projectId: string): Promise< + { + id: string; + testId: string; + type: number; + selectedVariant: string; + createdAt: Date; + }[] + > { + const queryResult = await clickhouseClient.query({ + query: `SELECT * FROM abby.events WHERE projectId = '${"clvh4sv5n0001furg6tj08z63"}'`, }); + + return (await queryResult.json()).data as any; } static async getEventsByTestId(testId: string, timeInterval: string) { diff --git a/apps/web/src/server/services/EventService.ts b/apps/web/src/server/services/EventService.ts index 7c5ac41f..9ea5c051 100644 --- a/apps/web/src/server/services/EventService.ts +++ b/apps/web/src/server/services/EventService.ts @@ -9,14 +9,6 @@ import { getLimitByPlan, Limit, PlanName, PLANS } from "server/common/plans"; import { prisma } from "server/db/client"; import { AbbyEvent } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; - -export interface EventServiceInterface { - createEvent(event: AbbyEvent): Promise; - // getEventsByProjectId(projectId: string): Promise; - // getEventsByTestId(testId: string, timeInterval: string): Promise; - // getEventsForCurrentPeriod(projectId: string): Promise; -} - export abstract class EventService { static async createEvent({ projectId, From 0fbc9735a651fbcf09a5058f54ed9324e334e901 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Fri, 5 Jul 2024 03:26:54 +0200 Subject: [PATCH 05/19] Save events in prisma and clickhouse with same id --- apps/web/clickhouse/createDatabase.ts | 26 ++++++++++--------- apps/web/src/server/queue/event.ts | 5 ++-- .../server/services/ClickHouseEventService.ts | 22 ++++++++-------- apps/web/src/server/services/EventService.ts | 12 ++++----- 4 files changed, 34 insertions(+), 31 deletions(-) diff --git a/apps/web/clickhouse/createDatabase.ts b/apps/web/clickhouse/createDatabase.ts index 99504489..7bca3be5 100644 --- a/apps/web/clickhouse/createDatabase.ts +++ b/apps/web/clickhouse/createDatabase.ts @@ -11,24 +11,26 @@ client .then((res) => { client.command({ query: ` - DROP TABLE IF EXISTS abby.Event; + DROP TABLE IF EXISTS abby.Event; `, }); }) .then((res) => { client.command({ query: ` - CREATE TABLE IF NOT EXISTS abby.Event ( - id String, - project_id String, - testName String, - type Int, - selectedVariant String, - createdAt DateTime DEFAULT toDateTime(now()) NOT NULL, - - ) - ENGINE = MergeTree() - ORDER BY (project_id, testName) + CREATE TABLE IF NOT EXISTS abby.Event ( + id UUID, + project_id String, + testName String, + type Int, + selectedVariant String, + createdAt DateTime DEFAULT toDateTime(now()) NOT NULL, + ) + ENGINE = MergeTree() + ORDER BY (project_id, testName) `, }); + }) + .catch((error) => { + console.error("Error creating table:", error); }); diff --git a/apps/web/src/server/queue/event.ts b/apps/web/src/server/queue/event.ts index 9bbc7982..e0a7b225 100644 --- a/apps/web/src/server/queue/event.ts +++ b/apps/web/src/server/queue/event.ts @@ -25,9 +25,10 @@ const eventWorker = new Worker( switch (event.type) { case AbbyEventType.PING: case AbbyEventType.ACT: { + const id = crypto.randomUUID(); await Promise.all([ - EventService.createEvent(event), - ClickHouseEventService.createEvent(event), + EventService.createEvent(event, id), + ClickHouseEventService.createEvent(event, id), ]); break; diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index e48b78c4..e2a57d41 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -12,21 +12,21 @@ import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; export abstract class ClickHouseEventService { - static async createEvent(event: { - type: AbbyEventType; - projectId: string; - testName: string; - selectedVariant: string; - }) { + static async createEvent( + { projectId, selectedVariant, testName, type }: AbbyEvent, + id: string + ) { + console.log("clickhouse", id); const insertedEvent = await clickhouseClient.insert({ - table: "abby.events", + table: "abby.Event", format: "JSONEachRow", values: [ { - type: event.type, - projectId: event.projectId, - testName: event.testName, - selectedVariant: event.selectedVariant, + id, + project_id: projectId, + testName: testName, + type: 0, + selectedVariant: selectedVariant, }, ], }); diff --git a/apps/web/src/server/services/EventService.ts b/apps/web/src/server/services/EventService.ts index 9ea5c051..f9b88864 100644 --- a/apps/web/src/server/services/EventService.ts +++ b/apps/web/src/server/services/EventService.ts @@ -10,14 +10,14 @@ import { prisma } from "server/db/client"; import { AbbyEvent } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; export abstract class EventService { - static async createEvent({ - projectId, - selectedVariant, - testName, - type, - }: AbbyEvent) { + static async createEvent( + { projectId, selectedVariant, testName, type }: AbbyEvent, + id: string + ) { + console.log("prisma", id); return prisma.event.create({ data: { + id, selectedVariant, type, test: { From 429e83cb289155a4a2f1dcd16410ae246ff84152 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sun, 7 Jul 2024 04:33:12 +0200 Subject: [PATCH 06/19] Feat: calculate test data on server --- apps/web/src/components/Test/Metrics.tsx | 27 +++------ apps/web/src/components/Test/Section.tsx | 57 ++++++++----------- apps/web/src/components/Test/Serves.tsx | 39 ++++++------- .../src/pages/projects/[projectId]/index.tsx | 15 ++++- apps/web/src/server/trpc/router/project.ts | 50 +++++++++++++--- 5 files changed, 102 insertions(+), 86 deletions(-) diff --git a/apps/web/src/components/Test/Metrics.tsx b/apps/web/src/components/Test/Metrics.tsx index e2421cb2..5463715f 100644 --- a/apps/web/src/components/Test/Metrics.tsx +++ b/apps/web/src/components/Test/Metrics.tsx @@ -42,26 +42,14 @@ export const OPTIONS: ChartOptions<"bar"> = { }; const Metrics = ({ - pingEvents, - options, + visitData, }: { - pingEvents: Event[]; - options: ClientOption[]; + visitData: (ClientOption & { actEventCount: number })[]; }) => { - const labels = options.map((option) => option.identifier); - const actualData = useMemo(() => { - return options.map((option) => { - return { - pings: pingEvents.filter( - (event) => event.selectedVariant === option.identifier - ).length, - weight: option.chance, - }; - }); - }, [options, pingEvents]); + const labels = visitData.map((data) => data.identifier); - const absPings = actualData.reduce((accumulator, value) => { - return accumulator + value.pings; + const absPings = visitData.reduce((accumulator, value) => { + return accumulator + value.actEventCount; }, 0); return ( @@ -74,12 +62,13 @@ const Metrics = ({ datasets: [ { label: "Actual", - data: actualData.map((d) => d.pings), + data: visitData.map((data) => data.actEventCount), + backgroundColor: "#A9E4EF", }, { label: "Expected", - data: actualData.map((data) => absPings * data.weight), + data: visitData.map((data) => absPings * data.chance), backgroundColor: "#f472b6", }, ], diff --git a/apps/web/src/components/Test/Section.tsx b/apps/web/src/components/Test/Section.tsx index 5b435185..27a9ced3 100644 --- a/apps/web/src/components/Test/Section.tsx +++ b/apps/web/src/components/Test/Section.tsx @@ -18,16 +18,10 @@ import { TitleEdit } from "components/TitleEdit"; import { Modal } from "components/Modal"; import { cn } from "lib/utils"; -function getBestVariant({ - absPings, - options, -}: { - absPings: number; - options: ClientOption[]; -}) { - const bestVariant = options.reduce( +function getBestVariant(visitData: VisitData) { + const bestVariant = visitData.reduce( (accumulator, option) => { - const pings = absPings * option.chance; + const pings = option.actEventCount; if (pings > accumulator.pings) { return { pings, @@ -39,7 +33,7 @@ function getBestVariant({ { pings: 0, identifier: "" } ); - return bestVariant; + return bestVariant.identifier; } const DeleteTestModal = ({ @@ -125,24 +119,31 @@ export const Card = ({ ); }; +export type VisitData = { + visitedEventCount: number; + actEventCount: number; + id: string; + identifier: string; + testId: string; + chance: number; + + variantName: string; +}[]; + const Section = ({ name, - options = [], - events = [], id, -}: Test & { - options: ClientOption[]; - events: Event[]; + visitData, +}: { + name: string; + id: string; + visitData: VisitData; }) => { const router = useRouter(); const [isDeleteModalOpen, setIsDeleteModalOpen] = useState(false); const trpcContext = trpc.useContext(); const showAdvancedTestStats = useFeatureFlag("AdvancedTestStats"); - - const bestVariant = getBestVariant({ - absPings: events.filter((event) => event.type === AbbyEventType.ACT).length, - options, - }).identifier; + const bestVariant = getBestVariant(visitData); const { mutate: updateTestName } = trpc.tests.updateName.useMutation({ onSuccess() { @@ -190,7 +191,7 @@ const Section = ({

} > - + } > - event.type === AbbyEventType.PING - )} - /> + } > - event.type === AbbyEventType.ACT - )} - /> +
diff --git a/apps/web/src/components/Test/Serves.tsx b/apps/web/src/components/Test/Serves.tsx index 8934cf37..716efd5f 100644 --- a/apps/web/src/components/Test/Serves.tsx +++ b/apps/web/src/components/Test/Serves.tsx @@ -57,28 +57,21 @@ export const OPTIONS: ChartOptions<"bar"> = { }; const Serves = ({ - pingEvents, - options, + visitData, }: { - pingEvents: Event[]; - options: ClientOption[]; + visitData: (ClientOption & { visitedEventCount: number })[]; }) => { - const labels = options.map((option) => option.identifier); - - const actualData = useMemo(() => { - return options.map((option) => { - return pingEvents.filter( - (event) => event.selectedVariant === option.identifier - ).length; - }); - }, [options, pingEvents]); - - const absPings = actualData.reduce((accumulator, value) => { - return accumulator + value; + const labels = visitData.map((data) => data.identifier); + const absPings = visitData.reduce((accumulator, value) => { + return accumulator + value.visitedEventCount; }, 0); + visitData.map((data) => { + console.log((data.visitedEventCount / absPings) * 249); + }); + return ( -
+
parseFloat(option.chance.toString()) * 100 - ), + data: visitData.map((data) => { + return parseFloat(data.chance.toString()) * 100; + }), backgroundColor: "#A9E4EF", }, { label: "Actual", - data: actualData.map((data) => - Math.round((data / absPings) * 100) - ), + data: visitData.map((data) => { + return Math.round((data.visitedEventCount / absPings) * 100); + }), backgroundColor: "#f472b6", }, ], diff --git a/apps/web/src/pages/projects/[projectId]/index.tsx b/apps/web/src/pages/projects/[projectId]/index.tsx index fe5c2819..b1a84ef8 100644 --- a/apps/web/src/pages/projects/[projectId]/index.tsx +++ b/apps/web/src/pages/projects/[projectId]/index.tsx @@ -10,6 +10,7 @@ import { AiOutlinePlus } from "react-icons/ai"; import { trpc } from "utils/trpc"; import { Button } from "components/ui/button"; import { GetStaticProps, GetStaticPaths } from "next"; +import { AbbyEventType } from "@tryabby/core"; const Projects: NextPageWithLayout = () => { const [isCreateTestModalOpen, setIsCreateTestModalOpen] = useState(false); @@ -58,9 +59,17 @@ const Projects: NextPageWithLayout = () => { />
- {data?.project?.tests.map((test) => ( -
- ))} + {data?.project?.tests.map((test) => { + console.log("test", test); + return ( +
+ ); + })}
); diff --git a/apps/web/src/server/trpc/router/project.ts b/apps/web/src/server/trpc/router/project.ts index 408c73f7..f40e196b 100644 --- a/apps/web/src/server/trpc/router/project.ts +++ b/apps/web/src/server/trpc/router/project.ts @@ -5,7 +5,7 @@ import { stripe } from "server/common/stripe"; import { EventService } from "server/services/EventService"; import { ProjectService } from "server/services/ProjectService"; import { generateCodeSnippets } from "utils/snippets"; -import { z } from "zod"; +import { ParseStatus, z } from "zod"; export type ClientOption = Omit & { chance: number; @@ -13,6 +13,7 @@ export type ClientOption = Omit & { import { updateProjectsOnSession } from "utils/updateSession"; import { protectedProcedure, router } from "../trpc"; +import { AbbyEventType } from "@tryabby/core"; export const projectRouter = router({ getProjectData: protectedProcedure @@ -43,17 +44,50 @@ export const projectRouter = router({ const { events: eventsThisPeriod } = await EventService.getEventsForCurrentPeriod(project.id); + const tests = await Promise.all( + project.tests.map(async (test) => { + const visitData = await Promise.all( + test.options.map(async (option) => { + const [visitedEventCount, actEventCount] = await Promise.all([ + ctx.prisma.event.count({ + where: { + testId: test.id, + selectedVariant: option.identifier, + type: AbbyEventType.PING, + }, + }), + ctx.prisma.event.count({ + where: { + testId: test.id, + selectedVariant: option.identifier, + type: AbbyEventType.ACT, + }, + }), + ]); + + return { + variantName: option.identifier, + ...option, + chance: option.chance.toNumber(), + visitedEventCount, + actEventCount, + }; + }) + ); + + return { + id: test.id, + name: test.name, + visitData, + }; + }) + ); + return { project: { ...project, eventsThisPeriod, - tests: project.tests.map((test) => ({ - ...test, - options: test.options.map((option) => ({ - ...option, - chance: option.chance.toNumber(), - })), - })), + tests, }, }; }), From e632f0909c70b30145b0efc83d5b75fffc924d8e Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sun, 7 Jul 2024 04:45:49 +0200 Subject: [PATCH 07/19] Fix: best variant calculation --- apps/web/src/components/Test/Section.tsx | 33 +++++++++++-------- .../src/pages/projects/[projectId]/index.tsx | 1 - apps/web/src/server/trpc/router/project.ts | 2 +- 3 files changed, 20 insertions(+), 16 deletions(-) diff --git a/apps/web/src/components/Test/Section.tsx b/apps/web/src/components/Test/Section.tsx index 27a9ced3..c4539650 100644 --- a/apps/web/src/components/Test/Section.tsx +++ b/apps/web/src/components/Test/Section.tsx @@ -19,21 +19,26 @@ import { Modal } from "components/Modal"; import { cn } from "lib/utils"; function getBestVariant(visitData: VisitData) { - const bestVariant = visitData.reduce( - (accumulator, option) => { - const pings = option.actEventCount; - if (pings > accumulator.pings) { - return { - pings, - identifier: option.identifier, - }; - } - return accumulator; - }, - { pings: 0, identifier: "" } - ); + const absPings = visitData + .map((data) => data.actEventCount) + .reduce((acc, curr) => acc + curr); + + const diffToExpected = visitData.map((data) => { + return { + variantName: data.variantName, + difference: data.actEventCount - data.chance * absPings, + }; + }); + + let currentMax = diffToExpected[0]; + + diffToExpected.forEach((diff) => { + if (currentMax!.difference < diff.difference) { + currentMax = diff; + } + }); - return bestVariant.identifier; + return currentMax?.variantName; } const DeleteTestModal = ({ diff --git a/apps/web/src/pages/projects/[projectId]/index.tsx b/apps/web/src/pages/projects/[projectId]/index.tsx index b1a84ef8..b810a61a 100644 --- a/apps/web/src/pages/projects/[projectId]/index.tsx +++ b/apps/web/src/pages/projects/[projectId]/index.tsx @@ -60,7 +60,6 @@ const Projects: NextPageWithLayout = () => {
{data?.project?.tests.map((test) => { - console.log("test", test); return (
Date: Sun, 7 Jul 2024 05:58:39 +0200 Subject: [PATCH 08/19] FEAT: load test data from clickhouse --- apps/web/src/components/Test/Serves.tsx | 5 +-- .../server/services/ClickHouseEventService.ts | 40 +++++++++++++++++++ apps/web/src/server/trpc/router/project.ts | 31 ++++++-------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/apps/web/src/components/Test/Serves.tsx b/apps/web/src/components/Test/Serves.tsx index 716efd5f..ea05dd0b 100644 --- a/apps/web/src/components/Test/Serves.tsx +++ b/apps/web/src/components/Test/Serves.tsx @@ -63,12 +63,11 @@ const Serves = ({ }) => { const labels = visitData.map((data) => data.identifier); const absPings = visitData.reduce((accumulator, value) => { + console.log(value.visitedEventCount); return accumulator + value.visitedEventCount; }, 0); - visitData.map((data) => { - console.log((data.visitedEventCount / absPings) * 249); - }); + console.log("abs", absPings); return (
diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index e2a57d41..6178bac3 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -10,6 +10,7 @@ import { prisma } from "server/db/client"; import { AbbyEvent, AbbyEventType } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; +import { count } from "node:console"; export abstract class ClickHouseEventService { static async createEvent( @@ -50,6 +51,45 @@ export abstract class ClickHouseEventService { return (await queryResult.json()).data as any; } + static async getGroupedEventsByTestId( + test: { + options: { + id: string; + identifier: string; + testId: string; + }[]; + } & { + id: string; + projectId: string; + createdAt: Date; + updatedAt: Date; + name: string; + } + ) { + const queryResult = await clickhouseClient.query({ + query: `select count(*) as count, type, selectedVariant from abby.Event + where testName ='${test.id}' + group by type, selectedVariant; + `, + }); + + //TODO add validation with id + const parsedRes = (await queryResult.json()).data as { + selectedVariant: string; + type: string; + count: string; + }[]; + + return parsedRes.map((row) => { + console.log(typeof row.count); + return { + variant: row.selectedVariant, + type: parseInt(row.type) == 0 ? AbbyEventType.PING : AbbyEventType.ACT, + count: parseInt(row.count), + }; + }); + } + static async getEventsByTestId(testId: string, timeInterval: string) { const now = new Date().getTime(); diff --git a/apps/web/src/server/trpc/router/project.ts b/apps/web/src/server/trpc/router/project.ts index a65d2dd9..68638948 100644 --- a/apps/web/src/server/trpc/router/project.ts +++ b/apps/web/src/server/trpc/router/project.ts @@ -14,6 +14,7 @@ export type ClientOption = Omit & { import { updateProjectsOnSession } from "utils/updateSession"; import { protectedProcedure, router } from "../trpc"; import { AbbyEventType } from "@tryabby/core"; +import { ClickHouseEventService } from "server/services/ClickHouseEventService"; export const projectRouter = router({ getProjectData: protectedProcedure @@ -48,29 +49,23 @@ export const projectRouter = router({ project.tests.map(async (test) => { const visitData = await Promise.all( test.options.map(async (option) => { - const [visitedEventCount, actEventCount] = await Promise.all([ - ctx.prisma.event.count({ - where: { - testId: test.id, - selectedVariant: option.identifier, - type: AbbyEventType.PING, - }, - }), - ctx.prisma.event.count({ - where: { - testId: test.id, - selectedVariant: option.identifier, - type: AbbyEventType.ACT, - }, - }), - ]); + const clickhouseResult = + await ClickHouseEventService.getGroupedEventsByTestId(test); return { variantName: option.identifier, ...option, chance: option.chance.toNumber(), - visitedEventCount, - actEventCount, + visitedEventCount: clickhouseResult.find( + (res) => + res.variant == option.identifier && + res.type == AbbyEventType.PING + )?.count, + actEventCount: clickhouseResult.find( + (res) => + res.variant == option.identifier && + res.type == AbbyEventType.ACT + )?.count, }; }) ); From 916e23da58fc23545593363235da72f46372210e Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Mon, 8 Jul 2024 22:23:20 +0200 Subject: [PATCH 09/19] Feat: parse clickhouse result with zod --- .../server/services/ClickHouseEventService.ts | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index 6178bac3..172918bd 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -10,23 +10,31 @@ import { prisma } from "server/db/client"; import { AbbyEvent, AbbyEventType } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; -import { count } from "node:console"; +import { z } from "zod"; + +const GroupedTestQueryResultSchema = z.object({ + selectedVariant: z.string(), + type: z.number(), + count: z.string(), +}); + +type GroupedTestQueryResult = z.infer; export abstract class ClickHouseEventService { - static async createEvent( - { projectId, selectedVariant, testName, type }: AbbyEvent, - id: string - ) { - console.log("clickhouse", id); + static async createEvent({ + projectId, + selectedVariant, + testName, + type, + }: AbbyEvent) { const insertedEvent = await clickhouseClient.insert({ table: "abby.Event", format: "JSONEachRow", values: [ { - id, project_id: projectId, testName: testName, - type: 0, + type, selectedVariant: selectedVariant, }, ], @@ -45,7 +53,7 @@ export abstract class ClickHouseEventService { }[] > { const queryResult = await clickhouseClient.query({ - query: `SELECT * FROM abby.events WHERE projectId = '${"clvh4sv5n0001furg6tj08z63"}'`, + query: `SELECT * FROM abby.events WHERE projectId = '${projectId}'`, }); return (await queryResult.json()).data as any; @@ -72,20 +80,15 @@ export abstract class ClickHouseEventService { group by type, selectedVariant; `, }); + const parsedJson = (await queryResult.json()).data; - //TODO add validation with id - const parsedRes = (await queryResult.json()).data as { - selectedVariant: string; - type: string; - count: string; - }[]; - - return parsedRes.map((row) => { - console.log(typeof row.count); + return parsedJson.map((row) => { + const { count, selectedVariant, type } = + GroupedTestQueryResultSchema.parse(row); return { - variant: row.selectedVariant, - type: parseInt(row.type) == 0 ? AbbyEventType.PING : AbbyEventType.ACT, - count: parseInt(row.count), + variant: selectedVariant, + type: type === 0 ? AbbyEventType.PING : AbbyEventType.ACT, + count: parseInt(count), }; }); } From 84cda10e7350edd93dcb46e02dab9846f427cd65 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Tue, 9 Jul 2024 03:29:29 +0200 Subject: [PATCH 10/19] Prevent oom with 1mio events in db --- apps/web/src/components/Test/Serves.tsx | 3 -- .../projects/[projectId]/tests/[testId].tsx | 2 - .../server/services/ClickHouseEventService.ts | 52 ++++++------------- apps/web/src/server/services/EventService.ts | 2 + apps/web/src/server/trpc/router/events.ts | 11 ++++ 5 files changed, 29 insertions(+), 41 deletions(-) diff --git a/apps/web/src/components/Test/Serves.tsx b/apps/web/src/components/Test/Serves.tsx index ea05dd0b..d00010a5 100644 --- a/apps/web/src/components/Test/Serves.tsx +++ b/apps/web/src/components/Test/Serves.tsx @@ -63,12 +63,9 @@ const Serves = ({ }) => { const labels = visitData.map((data) => data.identifier); const absPings = visitData.reduce((accumulator, value) => { - console.log(value.visitedEventCount); return accumulator + value.visitedEventCount; }, 0); - console.log("abs", absPings); - return (
{ }); }, [eventsByVariant, interval]); - console.log("formattedEvents", formattedEvents); - const viewEvents = useMemo( () => ({ labels, diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index 6178bac3..b59d02d6 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -10,14 +10,12 @@ import { prisma } from "server/db/client"; import { AbbyEvent, AbbyEventType } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; -import { count } from "node:console"; export abstract class ClickHouseEventService { static async createEvent( { projectId, selectedVariant, testName, type }: AbbyEvent, id: string ) { - console.log("clickhouse", id); const insertedEvent = await clickhouseClient.insert({ table: "abby.Event", format: "JSONEachRow", @@ -81,53 +79,35 @@ export abstract class ClickHouseEventService { }[]; return parsedRes.map((row) => { - console.log(typeof row.count); return { variant: row.selectedVariant, - type: parseInt(row.type) == 0 ? AbbyEventType.PING : AbbyEventType.ACT, + type: parseInt(row.type) === 0 ? AbbyEventType.PING : AbbyEventType.ACT, count: parseInt(row.count), }; }); } + //brauchen wir das? static async getEventsByTestId(testId: string, timeInterval: string) { const now = new Date().getTime(); - if (isSpecialTimeInterval(timeInterval)) { - const specialIntervalInMs = getMSFromSpecialTimeInterval(timeInterval); - return prisma.event.findMany({ - where: { - testId, - ...(specialIntervalInMs !== Infinity && - timeInterval !== SpecialTimeInterval.DAY && { - createdAt: { - gte: new Date(now - getMSFromSpecialTimeInterval(timeInterval)), - }, - }), - // Special case for day, since we want to include the current day - ...(timeInterval === SpecialTimeInterval.DAY && { - createdAt: { - gte: dayjs().startOf("day").toDate(), - }, - }), - }, + console.log("hier2"); + try { + const result = await clickhouseClient.query({ + query: `SELECT + toStartOfInterval(toTimeZone(createdAt, 'UTC'), toIntervalHour(3)) AS bucket_start, + count(*) AS bucket_count +FROM abby.Event +WHERE testName = '${testId}' +GROUP BY toStartOfInterval(toTimeZone(createdAt, 'UTC'), toIntervalHour(3)) +ORDER BY bucket_start ASC; +`, }); - } - - const parsedInterval = ms(timeInterval) as number | undefined; - if (parsedInterval === undefined) { - throw new Error("Invalid time interval"); + console.log("result", result); + } catch (e) { + console.log("error", e); } - - return prisma.event.findMany({ - where: { - testId, - createdAt: { - gte: new Date(now - ms(timeInterval)), - }, - }, - }); } static async getEventsForCurrentPeriod(projectId: string) { diff --git a/apps/web/src/server/services/EventService.ts b/apps/web/src/server/services/EventService.ts index f9b88864..abce9c2b 100644 --- a/apps/web/src/server/services/EventService.ts +++ b/apps/web/src/server/services/EventService.ts @@ -48,6 +48,7 @@ export abstract class EventService { if (isSpecialTimeInterval(timeInterval)) { const specialIntervalInMs = getMSFromSpecialTimeInterval(timeInterval); return prisma.event.findMany({ + take: 1000, where: { testId, ...(specialIntervalInMs !== Infinity && @@ -79,6 +80,7 @@ export abstract class EventService { gte: new Date(now - ms(timeInterval)), }, }, + take: 1000, }); } diff --git a/apps/web/src/server/trpc/router/events.ts b/apps/web/src/server/trpc/router/events.ts index 991c601d..cbcc55b8 100644 --- a/apps/web/src/server/trpc/router/events.ts +++ b/apps/web/src/server/trpc/router/events.ts @@ -3,6 +3,7 @@ import { EventService } from "server/services/EventService"; import { ProjectService } from "server/services/ProjectService"; import { z } from "zod"; import { protectedProcedure, router } from "../trpc"; +import { ClickHouseEventService } from "server/services/ClickHouseEventService"; export const eventRouter = router({ getEvents: protectedProcedure @@ -27,6 +28,8 @@ export const eventRouter = router({ }) ) .query(async ({ ctx, input }) => { + console.log("hier"); + const currentTest = await ctx.prisma.test.count({ where: { id: input.testId, @@ -41,6 +44,7 @@ export const eventRouter = router({ }); if (!currentTest) { + console.log("erro"); throw new TRPCError({ code: "UNAUTHORIZED" }); } @@ -49,6 +53,13 @@ export const eventRouter = router({ input.interval ); + // console.log("clickhouse"); + // await ClickHouseEventService.getEventsByTestId( + // input.testId, + // input.interval + // ); + console.log("lickhouse end"); + return tests; }), }); From 4030222ed32e7563f4157706637762081fde3303 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Tue, 9 Jul 2024 03:35:43 +0200 Subject: [PATCH 11/19] Get aggregated Events via clickhouse --- .../server/services/ClickHouseEventService.ts | 16 ++++++++++------ apps/web/src/server/trpc/router/events.ts | 10 +++++----- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index 2d147320..c976c1de 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -100,17 +100,21 @@ export abstract class ClickHouseEventService { console.log("hier2"); try { const result = await clickhouseClient.query({ - query: `SELECT - toStartOfInterval(toTimeZone(createdAt, 'UTC'), toIntervalHour(3)) AS bucket_start, - count(*) AS bucket_count + query: ` + SELECT + toStartOfHour(createdAt) AS startTime, + Count(selectedVariant) AS countSelectedVariant, + selectedVariant, + type FROM abby.Event WHERE testName = '${testId}' -GROUP BY toStartOfInterval(toTimeZone(createdAt, 'UTC'), toIntervalHour(3)) -ORDER BY bucket_start ASC; +GROUP BY startTime, selectedVariant, type +ORDER BY startTime ASC; + `, }); - console.log("result", result); + console.log("result", (await result.json()).data); } catch (e) { console.log("error", e); } diff --git a/apps/web/src/server/trpc/router/events.ts b/apps/web/src/server/trpc/router/events.ts index cbcc55b8..faf97cbd 100644 --- a/apps/web/src/server/trpc/router/events.ts +++ b/apps/web/src/server/trpc/router/events.ts @@ -53,11 +53,11 @@ export const eventRouter = router({ input.interval ); - // console.log("clickhouse"); - // await ClickHouseEventService.getEventsByTestId( - // input.testId, - // input.interval - // ); + console.log("clickhouse"); + await ClickHouseEventService.getEventsByTestId( + input.testId, + input.interval + ); console.log("lickhouse end"); return tests; From 0ffa4476ff51e70466404d24c5ddf66a3d9b42a6 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Thu, 11 Jul 2024 01:22:18 +0200 Subject: [PATCH 12/19] wip --- apps/web/clickhouse/createDatabase.ts | 86 +++++++++++------- apps/web/src/lib/events.ts | 54 +++++++----- .../projects/[projectId]/tests/[testId].tsx | 87 ++++++++----------- .../server/services/ClickHouseEventService.ts | 65 +++++++++++--- apps/web/src/server/trpc/router/events.ts | 17 +--- 5 files changed, 176 insertions(+), 133 deletions(-) diff --git a/apps/web/clickhouse/createDatabase.ts b/apps/web/clickhouse/createDatabase.ts index 7bca3be5..921e7ae7 100644 --- a/apps/web/clickhouse/createDatabase.ts +++ b/apps/web/clickhouse/createDatabase.ts @@ -1,36 +1,64 @@ import { createClient } from "@clickhouse/client"; +import { Prisma, PrismaClient } from "@prisma/client"; +import { AbbyEventType } from "@tryabby/core"; const client = createClient({ url: "http://localhost:8123", }); -client - .command({ - query: "CREATE DATABASE IF NOT EXISTS abby", - }) - .then((res) => { - client.command({ - query: ` - DROP TABLE IF EXISTS abby.Event; - `, - }); - }) - .then((res) => { - client.command({ - query: ` - CREATE TABLE IF NOT EXISTS abby.Event ( - id UUID, - project_id String, - testName String, - type Int, - selectedVariant String, - createdAt DateTime DEFAULT toDateTime(now()) NOT NULL, - ) - ENGINE = MergeTree() - ORDER BY (project_id, testName) - `, +// client +// .command({ +// query: "CREATE DATABASE IF NOT EXISTS abby", +// }) +// .then((res) => { +// client.command({ +// query: ` +// DROP TABLE IF EXISTS abby.Event; +// `, +// }); +// }) +// .then((res) => { +// client.command({ +// query: ` +// CREATE TABLE IF NOT EXISTS abby.Event ( +// id UUID, +// project_id String, +// testName String, +// type Int, +// selectedVariant String, +// createdAt DateTime DEFAULT toDateTime(now()) NOT NULL, +// ) +// ENGINE = MergeTree() +// ORDER BY (project_id, testName) +// `, +// }); +// }) +// .catch((error) => { +// console.error("Error creating table:", error); +// }); + +async function insertEvents() { + const projectId = "clvh4sv5n0001furg6tj08z63"; + const testName = "clyetopos0001yd6wa4yybkvw"; + + for (let i = 0; i < 100_000; i++) { + await client.insert({ + table: "abby.Event", + format: "JSONEachRow", + values: [ + { + project_id: projectId, + testName: testName, + type: Math.random() < 0.5 ? AbbyEventType.PING : AbbyEventType.ACT, + selectedVariant: + Math.random() < 0.5 ? "New Variant 1" : "New Variant 2", + createdAt: + Math.floor(Date.now() / 1000) - + Math.floor(Math.random() * 24 * 60 * 60), + }, + ], }); - }) - .catch((error) => { - console.error("Error creating table:", error); - }); + } +} + +insertEvents(); diff --git a/apps/web/src/lib/events.ts b/apps/web/src/lib/events.ts index bdb5233c..93cb0808 100644 --- a/apps/web/src/lib/events.ts +++ b/apps/web/src/lib/events.ts @@ -1,8 +1,9 @@ +import { assertUnreachable } from "@tryabby/core"; import dayjs from "dayjs"; export enum SpecialTimeInterval { DAY = "day", - MONTH_TO_DATE = "month", + Last30DAYS = "30d", ALL_TIME = "all", } @@ -11,22 +12,10 @@ export const INTERVALS = [ label: "Today", value: SpecialTimeInterval.DAY, }, - // { - // label: "Last 7 days", - // value: "7d", - // }, { label: "Last 30 days", - value: "30d", + value: SpecialTimeInterval.Last30DAYS, }, - // { - // label: "Year to Date", - // value: SpecialTimeInterval.MONTH_TO_DATE, - // }, - // { - // label: "Last 12 months", - // value: "12mo", - // }, { label: "All Time", value: SpecialTimeInterval.ALL_TIME, @@ -54,12 +43,14 @@ export function getMSFromSpecialTimeInterval( case SpecialTimeInterval.DAY: { return 1000 * 60 * 60 * 24; } - case SpecialTimeInterval.MONTH_TO_DATE: { - return new Date().getTime() - new Date().setDate(1); + case SpecialTimeInterval.Last30DAYS: { + return 1000 * 60 * 60 * 24 * 30; } case SpecialTimeInterval.ALL_TIME: { return Infinity; } + default: + assertUnreachable(timeInterval); } } @@ -80,26 +71,41 @@ export function getFormattingByInterval(interval: INTERVAL) { export function getLabelsByInterval( interval: (typeof INTERVALS)[number]["value"], fistEventDate: Date -): Array { +): { labels: Array; dates: Array } { const formatting = getFormattingByInterval(interval); switch (interval) { case SpecialTimeInterval.DAY: { const baseData = dayjs().set("minute", 0); - return [0, 3, 6, 9, 12, 15, 18, 21].map((hour) => - baseData.set("hour", hour).format(formatting) + const dateArray = [0, 3, 6, 9, 12, 15, 18, 21].map((hour) => + baseData.set("hour", hour) ); + return { + labels: dateArray.map((date) => date.format(formatting)), + dates: dateArray.map((date) => date.toDate()), + }; } - case "30d": { - return Array.from({ length: 30 }, (_, i) => - dayjs().subtract(i, "day").format(formatting) + + case SpecialTimeInterval.Last30DAYS: { + const dateArray = Array.from({ length: 30 }, (_, i) => + dayjs().subtract(i, "day") ).reverse(); + return { + labels: dateArray.map((date) => date.format(formatting)), + dates: dateArray.map((date) => date.set("minute", 0).toDate()), + }; } case SpecialTimeInterval.ALL_TIME: { const diff = dayjs().diff(dayjs(fistEventDate), "month"); - return Array.from({ length: Math.max(diff, 6) }, (_, i) => - dayjs(fistEventDate).add(i, "month").format(formatting) + const dateArray = Array.from({ length: Math.max(diff, 6) }, (_, i) => + dayjs(fistEventDate).add(i, "month") ).reverse(); + return { + labels: dateArray.map((date) => date.format(formatting)), + dates: dateArray.map((date) => date.toDate()), + }; } + default: + return assertUnreachable(interval); } } diff --git a/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx b/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx index 03b8fcca..e94b889a 100644 --- a/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx +++ b/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx @@ -29,6 +29,7 @@ import { Legend, Filler, ChartOptions, + ChartData, } from "chart.js"; import colors from "tailwindcss/colors"; import { useMemo } from "react"; @@ -100,8 +101,8 @@ const TestDetailPage: NextPageWithLayout = () => { enabled: !!testId, } ); - - const { data: events } = trpc.events.getEventsByTestId.useQuery( + //TODO beide zusammen fassen + const { data } = trpc.events.getEventsByTestId.useQuery( { testId, interval, @@ -111,70 +112,50 @@ const TestDetailPage: NextPageWithLayout = () => { } ); - const eventsByVariant = useMemo(() => { - const eventsByVariant = groupBy(events, (e) => e.selectedVariant); - // make sure all variants are present - test?.options.map((option) => { - eventsByVariant[option.identifier] ??= []; - }); - return eventsByVariant; - }, [events, test?.options]); + const events = data ?? []; const labels = getLabelsByInterval( interval, - minBy(events, "createdAt")?.createdAt! + minBy(events, "createdAt")?.startTime! ); - const formattedEvents = useMemo(() => { - return Object.entries(eventsByVariant).map(([variant, events], i) => { - const eventsByDate = groupBy(events, (e) => { - const date = dayjs(e.createdAt); - // round by 3 hours - const hour = Math.floor(date.hour() / 3) * 3; - - return date - .set("hour", hour) - .set("minute", 0) - .format(getFormattingByInterval(interval)); - }); - return { eventsByDate, variant }; - }); - }, [eventsByVariant, interval]); - - const viewEvents = useMemo( + const viewEvents: ChartData<"line", number[], unknown> = useMemo( () => ({ - labels, - datasets: formattedEvents.map(({ eventsByDate, variant }, i) => { - return { - data: labels.map( - (label) => - eventsByDate[label]?.filter((e) => e.type === AbbyEventType.PING) - ?.length ?? 0 - ), - ...getChartOptions(i, variant), - }; - }), + labelsAndDates: labels.labels, + datasets: events + .filter((event) => event.type == AbbyEventType.PING) + .map((event, i) => { + return { + data: labels.dates.map((date) => { + console.log("Label", date, event.startTime); + return events.find((e) => e.startTime === date)?.count ?? 0; + }), + ...getChartOptions(i, event.selectedVariant), + }; + }), }), - [formattedEvents, labels] + [events, labels] ); - const actEvents = useMemo( + console.log(viewEvents); + + const actEvents: ChartData<"line", number[], unknown> = useMemo( () => ({ - labels, - datasets: formattedEvents.map(({ eventsByDate, variant }, i) => { - return { - data: labels.map( - (label) => - eventsByDate[label]?.filter((e) => e.type === AbbyEventType.ACT) - ?.length ?? 0 - ), - ...getChartOptions(i, variant), - }; - }), + labels: labels.labels, + datasets: events + .filter((event) => event.type === AbbyEventType.ACT) + .map((event, i) => { + return { + data: labels.dates.map((date) => event.count), + ...getChartOptions(i, event.selectedVariant), + }; + }), }), - [formattedEvents, labels] + [events, labels] ); + if (!events) return ; + if (isTestLoading || isTestError) { return ; } diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index c976c1de..c4416968 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -7,7 +7,7 @@ import { import ms from "ms"; import { getLimitByPlan, Limit, PlanName, PLANS } from "server/common/plans"; import { prisma } from "server/db/client"; -import { AbbyEvent, AbbyEventType } from "@tryabby/core"; +import { AbbyEvent, AbbyEventType, assertUnreachable } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; import { z } from "zod"; @@ -18,6 +18,14 @@ const GroupedTestQueryResultSchema = z.object({ count: z.string(), }); +const GroupedTestQueryResultSchemaWithTimeSchema = z.intersection( + GroupedTestQueryResultSchema, + z.object({ startTime: z.string() }) +); + +type GroupedTestQueryResultSchemaWithTime = z.infer< + typeof GroupedTestQueryResultSchemaWithTimeSchema +>; type GroupedTestQueryResult = z.infer; export abstract class ClickHouseEventService { @@ -94,27 +102,41 @@ export abstract class ClickHouseEventService { } //brauchen wir das? - static async getEventsByTestId(testId: string, timeInterval: string) { - const now = new Date().getTime(); + static async getEventsByTestId( + testId: string, + timeInterval: SpecialTimeInterval + ) { + const computedBucketSize = this.computeBucketSize(timeInterval); - console.log("hier2"); try { const result = await clickhouseClient.query({ query: ` SELECT - toStartOfHour(createdAt) AS startTime, - Count(selectedVariant) AS countSelectedVariant, - selectedVariant, - type -FROM abby.Event -WHERE testName = '${testId}' -GROUP BY startTime, selectedVariant, type -ORDER BY startTime ASC; - + ${computedBucketSize} AS startTime, + Count(selectedVariant) AS count, + selectedVariant, + type + FROM abby.Event + WHERE testName = '${testId}' + GROUP BY startTime, selectedVariant, type + ORDER BY startTime ASC; `, }); - console.log("result", (await result.json()).data); + const parsedJson = (await result.json()).data; + console.log(parsedJson); + const parsedRes = parsedJson.map((row) => { + const { count, selectedVariant, type, startTime } = + GroupedTestQueryResultSchemaWithTimeSchema.parse(row); + return { + startTime: new Date(startTime), + selectedVariant, + type: type === 0 ? AbbyEventType.PING : AbbyEventType.ACT, + count: parseInt(count), + }; + }); + + return parsedRes; } catch (e) { console.log("error", e); } @@ -144,4 +166,19 @@ ORDER BY startTime ASC; is80PercentOfLimit: planLimits.eventsPerMonth * 0.8 === eventCount, }; } + + static computeBucketSize(timeInterval: SpecialTimeInterval) { + switch (timeInterval) { + case SpecialTimeInterval.DAY: { + return "toStartOfHour(createdAt)"; + } + case SpecialTimeInterval.MONTH_TO_DATE: + case SpecialTimeInterval.ALL_TIME: + case SpecialTimeInterval.Last30DAYS: { + return "toStartOfDay(createdAt)"; + } + default: + return assertUnreachable(timeInterval); + } + } } diff --git a/apps/web/src/server/trpc/router/events.ts b/apps/web/src/server/trpc/router/events.ts index faf97cbd..4d9ca3c7 100644 --- a/apps/web/src/server/trpc/router/events.ts +++ b/apps/web/src/server/trpc/router/events.ts @@ -4,6 +4,7 @@ import { ProjectService } from "server/services/ProjectService"; import { z } from "zod"; import { protectedProcedure, router } from "../trpc"; import { ClickHouseEventService } from "server/services/ClickHouseEventService"; +import { SpecialTimeInterval } from "lib/events"; export const eventRouter = router({ getEvents: protectedProcedure @@ -24,12 +25,10 @@ export const eventRouter = router({ .input( z.object({ testId: z.string(), - interval: z.string(), + interval: z.nativeEnum(SpecialTimeInterval), }) ) .query(async ({ ctx, input }) => { - console.log("hier"); - const currentTest = await ctx.prisma.test.count({ where: { id: input.testId, @@ -44,22 +43,14 @@ export const eventRouter = router({ }); if (!currentTest) { - console.log("erro"); throw new TRPCError({ code: "UNAUTHORIZED" }); } - const tests = await EventService.getEventsByTestId( - input.testId, - input.interval - ); - - console.log("clickhouse"); - await ClickHouseEventService.getEventsByTestId( + const clickhouseEvents = await ClickHouseEventService.getEventsByTestId( input.testId, input.interval ); - console.log("lickhouse end"); - return tests; + return clickhouseEvents; }), }); From e7ca898b06efdc13d2b7e625399b835c2fb5172e Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sat, 20 Jul 2024 20:13:24 +0200 Subject: [PATCH 13/19] Fix scale for high test volumes --- apps/web/src/components/Test/Metrics.tsx | 6 ++++-- apps/web/src/server/trpc/router/project.ts | 22 ++++++++++++---------- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/apps/web/src/components/Test/Metrics.tsx b/apps/web/src/components/Test/Metrics.tsx index 5463715f..3fa8c246 100644 --- a/apps/web/src/components/Test/Metrics.tsx +++ b/apps/web/src/components/Test/Metrics.tsx @@ -30,10 +30,10 @@ export const OPTIONS: ChartOptions<"bar"> = { maintainAspectRatio: false, scales: { y: { - min: 0, - max: 100, + beginAtZero: true, }, }, + plugins: { legend: { position: "top" as const, @@ -52,6 +52,8 @@ const Metrics = ({ return accumulator + value.actEventCount; }, 0); + console.log(absPings, visitData); + return (
- res.variant == option.identifier && - res.type == AbbyEventType.PING - )?.count, - actEventCount: clickhouseResult.find( - (res) => - res.variant == option.identifier && - res.type == AbbyEventType.ACT - )?.count, + visitedEventCount: + clickhouseResult.find( + (res) => + res.variant == option.identifier && + res.type == AbbyEventType.PING + )?.count ?? 0, + actEventCount: + clickhouseResult.find( + (res) => + res.variant == option.identifier && + res.type == AbbyEventType.ACT + )?.count ?? 0, }; }) ); From 4cae247f10395a8543fad4e9103e65fd896064a0 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sat, 20 Jul 2024 21:16:33 +0200 Subject: [PATCH 14/19] Remove Event Service --- apps/web/src/api/routes/v1_event.ts | 5 +- apps/web/src/server/queue/AfterDataRequest.ts | 4 +- apps/web/src/server/queue/event.ts | 9 +- .../server/services/ClickHouseEventService.ts | 1 - apps/web/src/server/services/EventService.ts | 111 ------------------ apps/web/src/server/trpc/router/events.ts | 1 - apps/web/src/server/trpc/router/project.ts | 3 +- apps/web/src/server/trpc/router/tests.ts | 4 - 8 files changed, 6 insertions(+), 132 deletions(-) delete mode 100644 apps/web/src/server/services/EventService.ts diff --git a/apps/web/src/api/routes/v1_event.ts b/apps/web/src/api/routes/v1_event.ts index ec41592f..07324a97 100644 --- a/apps/web/src/api/routes/v1_event.ts +++ b/apps/web/src/api/routes/v1_event.ts @@ -1,11 +1,8 @@ import { zValidator } from "@hono/zod-validator"; -import { abbyEventSchema, AbbyEventType } from "@tryabby/core"; +import { abbyEventSchema } from "@tryabby/core"; import { Hono } from "hono"; import isbot from "isbot"; -import { EventService } from "server/services/EventService"; -import { RequestCache } from "server/services/RequestCache"; -import { RequestService } from "server/services/RequestService"; import { checkRateLimit } from "server/common/ratelimit"; import { eventQueue } from "server/queue/queues"; diff --git a/apps/web/src/server/queue/AfterDataRequest.ts b/apps/web/src/server/queue/AfterDataRequest.ts index 7a91b874..0392a0ae 100644 --- a/apps/web/src/server/queue/AfterDataRequest.ts +++ b/apps/web/src/server/queue/AfterDataRequest.ts @@ -3,9 +3,9 @@ import { ApiVersion } from "@prisma/client"; import { trackPlanOverage } from "lib/logsnag"; import { RequestCache } from "server/services/RequestCache"; import { RequestService } from "server/services/RequestService"; -import { EventService } from "server/services/EventService"; import { afterDataRequestQueue, getQueueingRedisConnection } from "./queues"; import { env } from "env/server.mjs"; +import { ClickHouseEventService } from "server/services/ClickHouseEventService"; export type AfterRequestJobPayload = { functionDuration: number; @@ -17,7 +17,7 @@ const afterDataRequestWorker = new Worker( afterDataRequestQueue.name, async ({ data: { apiVersion, functionDuration, projectId } }) => { const { events, planLimits, plan, is80PercentOfLimit } = - await EventService.getEventsForCurrentPeriod(projectId); + await ClickHouseEventService.getEventsForCurrentPeriod(projectId); if (events > planLimits.eventsPerMonth) { // TODO: send email diff --git a/apps/web/src/server/queue/event.ts b/apps/web/src/server/queue/event.ts index e0a7b225..b5d84f8e 100644 --- a/apps/web/src/server/queue/event.ts +++ b/apps/web/src/server/queue/event.ts @@ -2,7 +2,6 @@ import { Worker } from "bullmq"; import { trackPlanOverage } from "lib/logsnag"; import { RequestCache } from "server/services/RequestCache"; import { RequestService } from "server/services/RequestService"; -import { EventService } from "server/services/EventService"; import { eventQueue, getQueueingRedisConnection } from "./queues"; import { AbbyEvent, AbbyEventType } from "@tryabby/core"; import { env } from "env/server.mjs"; @@ -25,11 +24,7 @@ const eventWorker = new Worker( switch (event.type) { case AbbyEventType.PING: case AbbyEventType.ACT: { - const id = crypto.randomUUID(); - await Promise.all([ - EventService.createEvent(event, id), - ClickHouseEventService.createEvent(event, id), - ]); + await ClickHouseEventService.createEvent(event); break; } @@ -38,7 +33,7 @@ const eventWorker = new Worker( } } const { events, planLimits, plan, is80PercentOfLimit } = - await EventService.getEventsForCurrentPeriod(event.projectId); + await ClickHouseEventService.getEventsForCurrentPeriod(event.projectId); if (events > planLimits.eventsPerMonth) { // TODO: send email diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index c4416968..47791f70 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -172,7 +172,6 @@ export abstract class ClickHouseEventService { case SpecialTimeInterval.DAY: { return "toStartOfHour(createdAt)"; } - case SpecialTimeInterval.MONTH_TO_DATE: case SpecialTimeInterval.ALL_TIME: case SpecialTimeInterval.Last30DAYS: { return "toStartOfDay(createdAt)"; diff --git a/apps/web/src/server/services/EventService.ts b/apps/web/src/server/services/EventService.ts deleted file mode 100644 index abce9c2b..00000000 --- a/apps/web/src/server/services/EventService.ts +++ /dev/null @@ -1,111 +0,0 @@ -import dayjs from "dayjs"; -import { - getMSFromSpecialTimeInterval, - isSpecialTimeInterval, - SpecialTimeInterval, -} from "lib/events"; -import ms from "ms"; -import { getLimitByPlan, Limit, PlanName, PLANS } from "server/common/plans"; -import { prisma } from "server/db/client"; -import { AbbyEvent } from "@tryabby/core"; -import { RequestCache } from "./RequestCache"; -export abstract class EventService { - static async createEvent( - { projectId, selectedVariant, testName, type }: AbbyEvent, - id: string - ) { - console.log("prisma", id); - return prisma.event.create({ - data: { - id, - selectedVariant, - type, - test: { - connect: { - projectId_name: { - projectId, - name: testName, - }, - }, - }, - }, - }); - } - - static async getEventsByProjectId(projectId: string) { - return prisma.event.findMany({ - where: { - test: { - projectId, - }, - }, - }); - } - - static async getEventsByTestId(testId: string, timeInterval: string) { - const now = new Date().getTime(); - - if (isSpecialTimeInterval(timeInterval)) { - const specialIntervalInMs = getMSFromSpecialTimeInterval(timeInterval); - return prisma.event.findMany({ - take: 1000, - where: { - testId, - ...(specialIntervalInMs !== Infinity && - timeInterval !== SpecialTimeInterval.DAY && { - createdAt: { - gte: new Date(now - getMSFromSpecialTimeInterval(timeInterval)), - }, - }), - // Special case for day, since we want to include the current day - ...(timeInterval === SpecialTimeInterval.DAY && { - createdAt: { - gte: dayjs().startOf("day").toDate(), - }, - }), - }, - }); - } - - const parsedInterval = ms(timeInterval) as number | undefined; - - if (parsedInterval === undefined) { - throw new Error("Invalid time interval"); - } - - return prisma.event.findMany({ - where: { - testId, - createdAt: { - gte: new Date(now - ms(timeInterval)), - }, - }, - take: 1000, - }); - } - - static async getEventsForCurrentPeriod(projectId: string) { - const [project, eventCount] = await Promise.all([ - prisma.project.findUnique({ - where: { id: projectId }, - select: { stripePriceId: true }, - }), - RequestCache.get(projectId), - ]); - - if (!project) throw new Error("Project not found"); - - const plan = Object.keys(PLANS).find( - (plan) => PLANS[plan as PlanName] === project.stripePriceId - ) as PlanName | undefined; - - const planLimits = getLimitByPlan(plan ?? null); - - return { - events: eventCount, - planLimits, - plan, - is80PercentOfLimit: planLimits.eventsPerMonth * 0.8 === eventCount, - }; - } -} diff --git a/apps/web/src/server/trpc/router/events.ts b/apps/web/src/server/trpc/router/events.ts index 4d9ca3c7..f536c83b 100644 --- a/apps/web/src/server/trpc/router/events.ts +++ b/apps/web/src/server/trpc/router/events.ts @@ -1,5 +1,4 @@ import { TRPCError } from "@trpc/server"; -import { EventService } from "server/services/EventService"; import { ProjectService } from "server/services/ProjectService"; import { z } from "zod"; import { protectedProcedure, router } from "../trpc"; diff --git a/apps/web/src/server/trpc/router/project.ts b/apps/web/src/server/trpc/router/project.ts index 1a3fa1da..c66312a7 100644 --- a/apps/web/src/server/trpc/router/project.ts +++ b/apps/web/src/server/trpc/router/project.ts @@ -2,7 +2,6 @@ import { Option, ROLE } from "@prisma/client"; import { TRPCError } from "@trpc/server"; import { PLANS, planNameSchema } from "server/common/plans"; import { stripe } from "server/common/stripe"; -import { EventService } from "server/services/EventService"; import { ProjectService } from "server/services/ProjectService"; import { generateCodeSnippets } from "utils/snippets"; import { ParseStatus, z } from "zod"; @@ -43,7 +42,7 @@ export const projectRouter = router({ throw new TRPCError({ code: "UNAUTHORIZED" }); } const { events: eventsThisPeriod } = - await EventService.getEventsForCurrentPeriod(project.id); + await ClickHouseEventService.getEventsForCurrentPeriod(project.id); const tests = await Promise.all( project.tests.map(async (test) => { diff --git a/apps/web/src/server/trpc/router/tests.ts b/apps/web/src/server/trpc/router/tests.ts index 5d1b5606..d4b94d5c 100644 --- a/apps/web/src/server/trpc/router/tests.ts +++ b/apps/web/src/server/trpc/router/tests.ts @@ -1,11 +1,7 @@ import { TRPCError } from "@trpc/server"; -import { ProjectService } from "server/services/ProjectService"; import { z } from "zod"; import { protectedProcedure, router } from "../trpc"; import { prisma } from "server/db/client"; -import { getLimitByPlan } from "server/common/plans"; -import { getProjectPaidPlan } from "lib/stripe"; -import { EventService } from "server/services/EventService"; import { TestService } from "server/services/TestService"; import { ConfigCache } from "server/common/config-cache"; From 9431e1558b762db1cec42beec3adf8837fe44120 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sun, 21 Jul 2024 00:14:25 +0200 Subject: [PATCH 15/19] clean up --- .../src/pages/projects/[projectId]/tests/[testId].tsx | 4 ++-- apps/web/src/server/services/ClickHouseEventService.ts | 10 ++-------- apps/web/src/server/trpc/router/events.ts | 2 +- apps/web/src/server/trpc/router/project.ts | 8 ++++---- 4 files changed, 9 insertions(+), 15 deletions(-) diff --git a/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx b/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx index e94b889a..8c8bfc03 100644 --- a/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx +++ b/apps/web/src/pages/projects/[projectId]/tests/[testId].tsx @@ -112,7 +112,7 @@ const TestDetailPage: NextPageWithLayout = () => { } ); - const events = data ?? []; + const events = useMemo(() => data ?? [], [data]); const labels = getLabelsByInterval( interval, @@ -123,7 +123,7 @@ const TestDetailPage: NextPageWithLayout = () => { () => ({ labelsAndDates: labels.labels, datasets: events - .filter((event) => event.type == AbbyEventType.PING) + .filter((event) => event.type === AbbyEventType.PING) .map((event, i) => { return { data: labels.dates.map((date) => { diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index 47791f70..b129e14d 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -1,11 +1,5 @@ -import dayjs from "dayjs"; -import { - getMSFromSpecialTimeInterval, - isSpecialTimeInterval, - SpecialTimeInterval, -} from "lib/events"; -import ms from "ms"; -import { getLimitByPlan, Limit, PlanName, PLANS } from "server/common/plans"; +import { SpecialTimeInterval } from "lib/events"; +import { getLimitByPlan, PlanName, PLANS } from "server/common/plans"; import { prisma } from "server/db/client"; import { AbbyEvent, AbbyEventType, assertUnreachable } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; diff --git a/apps/web/src/server/trpc/router/events.ts b/apps/web/src/server/trpc/router/events.ts index f536c83b..cf09aca7 100644 --- a/apps/web/src/server/trpc/router/events.ts +++ b/apps/web/src/server/trpc/router/events.ts @@ -18,7 +18,7 @@ export const eventRouter = router({ throw new TRPCError({ code: "UNAUTHORIZED" }); } - return EventService.getEventsByProjectId(input.projectId); + return ClickHouseEventService.getEventsByProjectId(input.projectId); }), getEventsByTestId: protectedProcedure .input( diff --git a/apps/web/src/server/trpc/router/project.ts b/apps/web/src/server/trpc/router/project.ts index c66312a7..3b4787a1 100644 --- a/apps/web/src/server/trpc/router/project.ts +++ b/apps/web/src/server/trpc/router/project.ts @@ -58,14 +58,14 @@ export const projectRouter = router({ visitedEventCount: clickhouseResult.find( (res) => - res.variant == option.identifier && - res.type == AbbyEventType.PING + res.variant === option.identifier && + res.type === AbbyEventType.PING )?.count ?? 0, actEventCount: clickhouseResult.find( (res) => - res.variant == option.identifier && - res.type == AbbyEventType.ACT + res.variant === option.identifier && + res.type === AbbyEventType.ACT )?.count ?? 0, }; }) From 61de3daeb668cc9334dd439ad01dc9c30a1fc199 Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sun, 28 Jul 2024 02:46:46 +0200 Subject: [PATCH 16/19] add csv generation for sample data --- generate_csv/generate_csv.js | 210 ++++++++++++++++++++++++++++++ generate_csv/generate_csv.ts | 66 ++++++++++ generate_csv/insertInBatch.js | 232 ++++++++++++++++++++++++++++++++++ generate_csv/insertInBatch.ts | 175 +++++++++++++++++++++++++ generate_csv/package.json | 24 ++++ generate_csv/tsconfig.json | 23 ++++ 6 files changed, 730 insertions(+) create mode 100644 generate_csv/generate_csv.js create mode 100644 generate_csv/generate_csv.ts create mode 100644 generate_csv/insertInBatch.js create mode 100644 generate_csv/insertInBatch.ts create mode 100644 generate_csv/package.json create mode 100644 generate_csv/tsconfig.json diff --git a/generate_csv/generate_csv.js b/generate_csv/generate_csv.js new file mode 100644 index 00000000..7692ce5e --- /dev/null +++ b/generate_csv/generate_csv.js @@ -0,0 +1,210 @@ +"use strict"; +var __awaiter = + (this && this.__awaiter) || + function (thisArg, _arguments, P, generator) { + function adopt(value) { + return value instanceof P + ? value + : new P(function (resolve) { + resolve(value); + }); + } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { + try { + step(generator.next(value)); + } catch (e) { + reject(e); + } + } + function rejected(value) { + try { + step(generator["throw"](value)); + } catch (e) { + reject(e); + } + } + function step(result) { + result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); + } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); + }; +var __generator = + (this && this.__generator) || + function (thisArg, body) { + var _ = { + label: 0, + sent: function () { + if (t[0] & 1) throw t[1]; + return t[1]; + }, + trys: [], + ops: [], + }, + f, + y, + t, + g; + return ( + (g = { next: verb(0), throw: verb(1), return: verb(2) }), + typeof Symbol === "function" && + (g[Symbol.iterator] = function () { + return this; + }), + g + ); + function verb(n) { + return function (v) { + return step([n, v]); + }; + } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while ((g && ((g = 0), op[0] && (_ = 0)), _)) + try { + if ( + ((f = 1), + y && + (t = + op[0] & 2 + ? y["return"] + : op[0] + ? y["throw"] || ((t = y["return"]) && t.call(y), 0) + : y.next) && + !(t = t.call(y, op[1])).done) + ) + return t; + if (((y = 0), t)) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: + case 1: + t = op; + break; + case 4: + _.label++; + return { value: op[1], done: false }; + case 5: + _.label++; + y = op[1]; + op = [0]; + continue; + case 7: + op = _.ops.pop(); + _.trys.pop(); + continue; + default: + if ( + !((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && + (op[0] === 6 || op[0] === 2) + ) { + _ = 0; + continue; + } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { + _.label = op[1]; + break; + } + if (op[0] === 6 && _.label < t[1]) { + _.label = t[1]; + t = op; + break; + } + if (t && _.label < t[2]) { + _.label = t[2]; + _.ops.push(op); + break; + } + if (t[2]) _.ops.pop(); + _.trys.pop(); + continue; + } + op = body.call(thisArg, _); + } catch (e) { + op = [6, e]; + y = 0; + } finally { + f = t = 0; + } + if (op[0] & 5) throw op[1]; + return { value: op[0] ? op[1] : void 0, done: true }; + } + }; +Object.defineProperty(exports, "__esModule", { value: true }); +var csv_writer_1 = require("csv-writer"); +var uuid_1 = require("uuid"); +// Constants +var NUM_ENTRIES = 10000000; +var PROJECT_COUNT = 100; +var API_VERSION = "V0"; +var TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; +// Generate a random date between today and one year ago +function randomDate() { + var today = new Date(); + var oneYearAgo = new Date(today); + oneYearAgo.setFullYear(today.getFullYear() - 1); + var randomTime = new Date( + oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime()) + ); + return randomTime.toISOString().replace("T", " ").substring(0, 23); +} +// Generate project IDs +var projectIds = ["clvh4sv5n0001furg6tj08z63"]; +// Setup CSV writer +var csvWriter = (0, csv_writer_1.createObjectCsvWriter)({ + path: "ApiRequestFürMeinProject.csv", + header: [ + { id: "id", title: "id" }, + { id: "createdAt", title: "createdAt" }, + { id: "type", title: "type" }, + { id: "durationInMs", title: "durationInMs" }, + { id: "apiVersion", title: "apiVersion" }, + { id: "projectId", title: "projectId" }, + ], +}); +function generateCSV() { + return __awaiter(this, void 0, void 0, function () { + var records, i, record; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + records = []; + i = 0; + _a.label = 1; + case 1: + if (!(i < NUM_ENTRIES)) return [3 /*break*/, 4]; + record = { + id: (0, uuid_1.v4)(), + createdAt: randomDate(), + type: TYPES[Math.floor(Math.random() * TYPES.length)], + durationInMs: Math.floor(Math.random() * 1000) + 1, + apiVersion: API_VERSION, + projectId: "clvh4sv5n0001furg6tj08z63", + // projectIds[Math.floor(Math.random() * PROJECT_COUNT)], + }; + records.push(record); + if (!(records.length === 100000)) return [3 /*break*/, 3]; + return [4 /*yield*/, csvWriter.writeRecords(records)]; + case 2: + _a.sent(); + records.length = 0; // Clear the array + _a.label = 3; + case 3: + i++; + return [3 /*break*/, 1]; + case 4: + if (!(records.length > 0)) return [3 /*break*/, 6]; + return [4 /*yield*/, csvWriter.writeRecords(records)]; + case 5: + _a.sent(); + _a.label = 6; + case 6: + console.log("CSV file generated successfully!"); + return [2 /*return*/]; + } + }); + }); +} +generateCSV().catch(function (err) { + console.error("Error generating CSV:", err); +}); diff --git a/generate_csv/generate_csv.ts b/generate_csv/generate_csv.ts new file mode 100644 index 00000000..33a5aacf --- /dev/null +++ b/generate_csv/generate_csv.ts @@ -0,0 +1,66 @@ +import { createObjectCsvWriter } from "csv-writer"; +import { v4 as uuidv4 } from "uuid"; +import * as fs from "fs"; + +// Constants +const NUM_ENTRIES = 10_000_000; +const PROJECT_COUNT = 100; +const API_VERSION = "V0"; +const TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; + +// Generate a random date between today and one year ago +function randomDate(): string { + const today = new Date(); + const oneYearAgo = new Date(today); + oneYearAgo.setFullYear(today.getFullYear() - 1); + const randomTime = new Date( + oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime()) + ); + return randomTime.toISOString().replace("T", " ").substring(0, 23); +} + +// Generate project IDs +const projectIds = ["clvh4sv5n0001furg6tj08z63"]; + +// Setup CSV writer +const csvWriter = createObjectCsvWriter({ + path: "ApiRequest.csv", + header: [ + { id: "id", title: "id" }, + { id: "createdAt", title: "createdAt" }, + { id: "type", title: "type" }, + { id: "durationInMs", title: "durationInMs" }, + { id: "apiVersion", title: "apiVersion" }, + { id: "projectId", title: "projectId" }, + ], +}); + +async function generateCSV() { + const records: any = []; + for (let i = 0; i < NUM_ENTRIES; i++) { + const record = { + id: uuidv4(), + createdAt: randomDate(), + type: TYPES[Math.floor(Math.random() * TYPES.length)], + durationInMs: Math.floor(Math.random() * 1000) + 1, + apiVersion: API_VERSION, + projectId: "clvh4sv5n0001furg6tj08z63", + // projectIds[Math.floor(Math.random() * PROJECT_COUNT)], + }; + records.push(record); + + // Write in chunks to avoid memory issues + if (records.length === 100000) { + await csvWriter.writeRecords(records); + records.length = 0; // Clear the array + } + } + if (records.length > 0) { + await csvWriter.writeRecords(records); + } + console.log("CSV file generated successfully!"); +} + +generateCSV().catch((err) => { + console.error("Error generating CSV:", err); +}); diff --git a/generate_csv/insertInBatch.js b/generate_csv/insertInBatch.js new file mode 100644 index 00000000..e296c8ce --- /dev/null +++ b/generate_csv/insertInBatch.js @@ -0,0 +1,232 @@ +"use strict"; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __generator = (this && this.__generator) || function (thisArg, body) { + var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; + return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; + function verb(n) { return function (v) { return step([n, v]); }; } + function step(op) { + if (f) throw new TypeError("Generator is already executing."); + while (g && (g = 0, op[0] && (_ = 0)), _) try { + if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; + if (y = 0, t) op = [op[0] & 2, t.value]; + switch (op[0]) { + case 0: case 1: t = op; break; + case 4: _.label++; return { value: op[1], done: false }; + case 5: _.label++; y = op[1]; op = [0]; continue; + case 7: op = _.ops.pop(); _.trys.pop(); continue; + default: + if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } + if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } + if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } + if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } + if (t[2]) _.ops.pop(); + _.trys.pop(); continue; + } + op = body.call(thisArg, _); + } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } + if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; + } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.clickhouseClient = void 0; +var uuid_1 = require("uuid"); +var client_1 = require("@clickhouse/client"); +exports.clickhouseClient = (0, client_1.createClient)(); +// Constants +var NUM_ENTRIES = 10000000 / 10000; // Number of entries you want to insert +var PROJECT_COUNT = 100; +var API_VERSION = "V0"; +var TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; +var BATCH_SIZE = 10000 / 10; // Size of each batch +// Generate a random date between today and one year ago +function randomDate() { + var today = new Date(); + var oneYearAgo = new Date(today); + oneYearAgo.setFullYear(today.getFullYear() - 1); + var randomTime = new Date(oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime())); + return randomTime.toISOString().replace("T", " ").substring(0, 19); +} +var projectIds = [ + "clvh4sv5n0001furg6tj08z63", + // "8e0ad326-c7d4-4daa-a004-585ca96d8e71", + // "4e236852-6ef1-44e2-a15e-9463fed3921f", + // "d4c607ea-d9e8-44a3-b12e-749225cb0599", + // "c4111aec-5677-4d2d-8d71-de8e49f79c26", + // "e4b69866-a985-4100-8bdc-64672e2ebefe", + // "8716a142-b424-48b6-8051-5ab5ce6b06cb", + // "20895eff-723c-4c76-8334-cf32ce72501a", + // "e00683fb-eca8-419c-a9e6-f4526ad93aa4", + // "43e225d6-d5ef-4c0c-a435-7ea1ebe2f74c", + // "ed4ffef2-790d-4850-b075-e589978a56c4", + // "f13778dd-b360-48be-ab7a-d97c048d99ba", + // "097cd30d-e654-44f5-b379-7a69c594e986", + // "d1e049e5-823d-44b3-845d-1ca9609cb657", + // "e23231e3-a18b-48ae-a496-a625c29ecdfe", + // "2f01a0b2-cc78-4f8e-b77e-322a3321ea44", + // "76a50abb-7908-4cce-b168-a8709965b5f9", + // "a3660a3f-525d-4d66-bf90-0bf2313ffbfc", + // "d9d3736e-1ba2-43bf-8c1c-b1c33436ceba", + // "e73dbf0d-4919-47d6-a63f-1fc1258a5673", + // "e859f1e1-f004-4a02-aecb-2ecd413aca3f", + // "0e3687cb-1cbc-44dd-a057-c0e35d9de8c2", + // "9868de0f-6e53-4e21-b480-cf97a8f6cb7d", + // "83a051fa-e4b3-4965-b0b9-a850ccc2d78d", + // "6d473975-5d1c-4ac5-9ba6-068cdca0c77e", + // "3b9fa79c-c190-4869-b15f-731f0167d453", + // "3c1c27db-ce33-4d21-9108-81b3b7f11a54", + // "28834d98-282b-49bd-978d-692f3da4003e", + // "cd64cfcc-02df-495c-a0a0-d9fe9a6ca474", + // "242d0023-50d7-467f-ac96-602b938c03a8", + // "56c4ccb6-f81d-4443-86d9-537e3bc341ed", + // "c743ef78-262c-4a37-ab7f-ea3178f3d2a5", + // "27ac8acf-b401-422d-93ce-6b6f861d9b19", + // "a09f6643-1a40-40d1-b9cd-e14c7d8a8584", + // "01b84b17-00fa-45e5-ac2f-2780c5476fe6", + // "98509dc4-4c90-4fb7-a473-f2db017f21ca", + // "37f0a317-c7ce-4e26-b190-e17191dc1b8a", + // "db0ec165-3fa3-4f0d-82aa-0fccb737f9ad", + // "ad8e5c90-f946-44a8-89e5-a34df667840d", + // "11dbb0f1-1b94-41c2-8737-116f2849a9ae", + // "d5c7330c-d959-47fc-b02f-39926595ccd2", + // "34fb504d-27b1-4053-af62-20767115278c", + // "e86905a1-5fbe-463c-ae35-274d6516df95", + // "2530fba6-cae4-4907-9369-5616b1aa29c2", + // "470d8dda-2e6c-40de-bfd8-6eccdb21169b", + // "83c04edc-659b-476b-8cf3-e0cad8c78c1d", + // "1f8937ba-f6fa-4dd6-ad4d-2fe88c7beed1", + // "573aa5e6-e66f-4c57-8de0-eb4af5be80f7", + // "2839c716-1bdf-4d1d-bcde-c85fd7879e47", + // "f6450cfc-2937-4d1c-8054-2a2ab56af0c8", + // "ac761d46-a959-435c-98ed-72e86f42dd91", + // "8ab88073-88f2-4fa1-9ece-2e77daf22900", + // "8f72ec17-2c08-452c-9f82-f8270b8c211f", + // "cf2cf72f-61f4-48da-b9d6-03860cdd1a51", + // "6887099b-a5ca-42f3-830f-6b9e17bcc405", + // "1408e392-4e39-4841-a722-dfc467b5aea7", + // "a7648ad8-bd9b-4def-ad30-8f6a8ffe790c", + // "dc1cdf87-7e65-4e35-8c84-53cc45061e5d", + // "a62d6dbe-df0b-40ca-93d9-0b175671f858", + // "51449cc3-25e0-46d4-b1db-afc21d274a77", + // "5581bc22-3f75-49dd-bad3-7b39c6a6fa47", + // "a9999a00-0a6f-43d8-b277-96fb027361fe", + // "cea38acb-76d5-407d-8c41-43faab812206", + // "9db2bb0e-e227-4527-8d91-5f6dff7f3e67", + // "72a47b93-313a-42f4-a51e-b850eb18e3fd", + // "05c44513-0a4b-4b98-8ffc-ae44dafa3a54", + // "4622ec76-0fd7-4226-bdaf-eceb7e2444f2", + // "bf49128d-7899-43d9-a2b0-64297df9f057", + // "313e1413-b3b5-4952-bd08-5f34fb7e5744", + // "d28ccf5b-3ea1-46a0-8ed1-56db7a52de7c", + // "3ebad352-ef41-4f15-931d-6f077c4e6307", + // "0f969120-52c3-436f-b027-9723e10c1780", + // "78462cc2-ef73-446e-a244-bf786ac481c2", + // "06be3cee-0603-41cb-affd-3d1c15056586", + // "26f0887c-617d-46eb-8629-eb0690a85387", + // "14778b80-cd6c-43df-9010-ef8c9bcf801b", + // "93db3706-6682-43f7-9720-60056e96a898", + // "ad5ed7d9-40a8-4b63-8ccd-346fe618744c", + // "6d6adca8-153c-44dc-ac99-e42ee41d2883", + // "dff18ffe-0ef4-484b-8354-583252291f39", + // "89befa8a-a98e-4912-9843-f861633c2ab8", + // "835c14d1-399e-4740-9e55-4fa83b6ce045", + // "7c96f67f-8b72-43fb-8129-29f168fa0dd8", + // "f16211d7-b2b9-4d02-9f76-d8c23853a91f", + // "7b1a000a-c0c9-411c-a5dc-622297a99b12", + // "0213a399-d761-434b-ae6e-7faa6b53809b", + // "05456e2e-dcd3-40de-ba43-f8dd4c8c34b9", + // "9efce5d6-818a-4d14-961f-c74111b4aab2", + // "0072cf6a-aef2-4fe6-b069-eec7c1f13c93", + // "40aa3f60-0532-4628-9871-baa54ddc76c0", + // "877b1e77-8bea-4163-81ce-7b7d110557f4", + // "806592b9-0683-4fb4-9afc-aee808941de0", + // "fb89e622-be0d-4e38-8f8a-687b118d9fbb", + // "32ab0418-63e6-4ef0-8e18-d830ded901c0", + // "468a66dc-f550-4c76-be49-2040c381f42f", + // "c8f66b78-3675-4d66-928e-ae37ea3d89fa", + // "0ea8c1bb-4ae7-4af6-b684-535e7cece274", + // "c7b06ef9-b9ad-481e-b61b-25bc4edfafd0", + // "a8414689-91b6-4d35-a53e-b49fafd39fc2", + // "79db2c3f-f544-46cb-b5e3-fb1a752880b8", + // "86584601-342c-4f1c-b739-fb20187e9c52", +]; +function insertBatch(records) { + return __awaiter(this, void 0, void 0, function () { + var query, error_1; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + query = "\n INSERT INTO abby.ApiRequest (id, createdAt, type, durationInMs, apiVersion, projectId)\n VALUES ".concat(records.map(function (record) { return "('".concat(record.id, "', '").concat(record.createdAt, "', '").concat(record.type, "', ").concat(record.durationInMs, ", '").concat(record.apiVersion, "', '").concat(record.projectId, "')"); }).join(","), "\n "); + _a.label = 1; + case 1: + _a.trys.push([1, 3, , 4]); + return [4 /*yield*/, exports.clickhouseClient.query({ + query: query, + })]; + case 2: + _a.sent(); + return [3 /*break*/, 4]; + case 3: + error_1 = _a.sent(); + console.error("Error inserting batch:", error_1); + return [3 /*break*/, 4]; + case 4: return [2 /*return*/]; + } + }); + }); +} +function generateAndInsertRecords() { + return __awaiter(this, void 0, void 0, function () { + var records, batchCount, i, record; + return __generator(this, function (_a) { + switch (_a.label) { + case 0: + records = []; + batchCount = 0; + i = 0; + _a.label = 1; + case 1: + if (!(i < NUM_ENTRIES)) return [3 /*break*/, 4]; + record = { + id: (0, uuid_1.v4)(), + createdAt: randomDate(), + type: TYPES[Math.floor(Math.random() * TYPES.length)], + durationInMs: Math.floor(Math.random() * 1000) + 1, + apiVersion: API_VERSION, + projectId: projectIds[Math.floor(Math.random() * PROJECT_COUNT)], + }; + records.push(record); + if (!(records.length >= BATCH_SIZE)) return [3 /*break*/, 3]; + console.log("Inserting batch ".concat(++batchCount)); + return [4 /*yield*/, insertBatch(records)]; + case 2: + _a.sent(); + records.length = 0; // Clear the array + _a.label = 3; + case 3: + i++; + return [3 /*break*/, 1]; + case 4: + if (!(records.length > 0)) return [3 /*break*/, 6]; + console.log("Inserting final batch ".concat(++batchCount)); + return [4 /*yield*/, insertBatch(records)]; + case 5: + _a.sent(); + _a.label = 6; + case 6: + console.log("Data insertion completed successfully!"); + return [2 /*return*/]; + } + }); + }); +} +generateAndInsertRecords().catch(function (err) { + console.error("Error generating or inserting records:", err); +}); diff --git a/generate_csv/insertInBatch.ts b/generate_csv/insertInBatch.ts new file mode 100644 index 00000000..e0757925 --- /dev/null +++ b/generate_csv/insertInBatch.ts @@ -0,0 +1,175 @@ +import { v4 as uuidv4 } from "uuid"; +import { createClient } from "@clickhouse/client"; + +export const clickhouseClient = createClient(); + +// Constants +const NUM_ENTRIES = 10_000_000 / 10000; // Number of entries you want to insert +const PROJECT_COUNT = 100; +const API_VERSION = "V0"; +const TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; +const BATCH_SIZE = 10000 / 10; // Size of each batch + +// Generate a random date between today and one year ago +function randomDate(): string { + const today = new Date(); + const oneYearAgo = new Date(today); + oneYearAgo.setFullYear(today.getFullYear() - 1); + const randomTime = new Date( + oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime()) + ); + return randomTime.toISOString().replace("T", " ").substring(0, 19); +} + +const projectIds = [ + "clvh4sv5n0001furg6tj08z63", + // "8e0ad326-c7d4-4daa-a004-585ca96d8e71", + // "4e236852-6ef1-44e2-a15e-9463fed3921f", + // "d4c607ea-d9e8-44a3-b12e-749225cb0599", + // "c4111aec-5677-4d2d-8d71-de8e49f79c26", + // "e4b69866-a985-4100-8bdc-64672e2ebefe", + // "8716a142-b424-48b6-8051-5ab5ce6b06cb", + // "20895eff-723c-4c76-8334-cf32ce72501a", + // "e00683fb-eca8-419c-a9e6-f4526ad93aa4", + // "43e225d6-d5ef-4c0c-a435-7ea1ebe2f74c", + // "ed4ffef2-790d-4850-b075-e589978a56c4", + // "f13778dd-b360-48be-ab7a-d97c048d99ba", + // "097cd30d-e654-44f5-b379-7a69c594e986", + // "d1e049e5-823d-44b3-845d-1ca9609cb657", + // "e23231e3-a18b-48ae-a496-a625c29ecdfe", + // "2f01a0b2-cc78-4f8e-b77e-322a3321ea44", + // "76a50abb-7908-4cce-b168-a8709965b5f9", + // "a3660a3f-525d-4d66-bf90-0bf2313ffbfc", + // "d9d3736e-1ba2-43bf-8c1c-b1c33436ceba", + // "e73dbf0d-4919-47d6-a63f-1fc1258a5673", + // "e859f1e1-f004-4a02-aecb-2ecd413aca3f", + // "0e3687cb-1cbc-44dd-a057-c0e35d9de8c2", + // "9868de0f-6e53-4e21-b480-cf97a8f6cb7d", + // "83a051fa-e4b3-4965-b0b9-a850ccc2d78d", + // "6d473975-5d1c-4ac5-9ba6-068cdca0c77e", + // "3b9fa79c-c190-4869-b15f-731f0167d453", + // "3c1c27db-ce33-4d21-9108-81b3b7f11a54", + // "28834d98-282b-49bd-978d-692f3da4003e", + // "cd64cfcc-02df-495c-a0a0-d9fe9a6ca474", + // "242d0023-50d7-467f-ac96-602b938c03a8", + // "56c4ccb6-f81d-4443-86d9-537e3bc341ed", + // "c743ef78-262c-4a37-ab7f-ea3178f3d2a5", + // "27ac8acf-b401-422d-93ce-6b6f861d9b19", + // "a09f6643-1a40-40d1-b9cd-e14c7d8a8584", + // "01b84b17-00fa-45e5-ac2f-2780c5476fe6", + // "98509dc4-4c90-4fb7-a473-f2db017f21ca", + // "37f0a317-c7ce-4e26-b190-e17191dc1b8a", + // "db0ec165-3fa3-4f0d-82aa-0fccb737f9ad", + // "ad8e5c90-f946-44a8-89e5-a34df667840d", + // "11dbb0f1-1b94-41c2-8737-116f2849a9ae", + // "d5c7330c-d959-47fc-b02f-39926595ccd2", + // "34fb504d-27b1-4053-af62-20767115278c", + // "e86905a1-5fbe-463c-ae35-274d6516df95", + // "2530fba6-cae4-4907-9369-5616b1aa29c2", + // "470d8dda-2e6c-40de-bfd8-6eccdb21169b", + // "83c04edc-659b-476b-8cf3-e0cad8c78c1d", + // "1f8937ba-f6fa-4dd6-ad4d-2fe88c7beed1", + // "573aa5e6-e66f-4c57-8de0-eb4af5be80f7", + // "2839c716-1bdf-4d1d-bcde-c85fd7879e47", + // "f6450cfc-2937-4d1c-8054-2a2ab56af0c8", + // "ac761d46-a959-435c-98ed-72e86f42dd91", + // "8ab88073-88f2-4fa1-9ece-2e77daf22900", + // "8f72ec17-2c08-452c-9f82-f8270b8c211f", + // "cf2cf72f-61f4-48da-b9d6-03860cdd1a51", + // "6887099b-a5ca-42f3-830f-6b9e17bcc405", + // "1408e392-4e39-4841-a722-dfc467b5aea7", + // "a7648ad8-bd9b-4def-ad30-8f6a8ffe790c", + // "dc1cdf87-7e65-4e35-8c84-53cc45061e5d", + // "a62d6dbe-df0b-40ca-93d9-0b175671f858", + // "51449cc3-25e0-46d4-b1db-afc21d274a77", + // "5581bc22-3f75-49dd-bad3-7b39c6a6fa47", + // "a9999a00-0a6f-43d8-b277-96fb027361fe", + // "cea38acb-76d5-407d-8c41-43faab812206", + // "9db2bb0e-e227-4527-8d91-5f6dff7f3e67", + // "72a47b93-313a-42f4-a51e-b850eb18e3fd", + // "05c44513-0a4b-4b98-8ffc-ae44dafa3a54", + // "4622ec76-0fd7-4226-bdaf-eceb7e2444f2", + // "bf49128d-7899-43d9-a2b0-64297df9f057", + // "313e1413-b3b5-4952-bd08-5f34fb7e5744", + // "d28ccf5b-3ea1-46a0-8ed1-56db7a52de7c", + // "3ebad352-ef41-4f15-931d-6f077c4e6307", + // "0f969120-52c3-436f-b027-9723e10c1780", + // "78462cc2-ef73-446e-a244-bf786ac481c2", + // "06be3cee-0603-41cb-affd-3d1c15056586", + // "26f0887c-617d-46eb-8629-eb0690a85387", + // "14778b80-cd6c-43df-9010-ef8c9bcf801b", + // "93db3706-6682-43f7-9720-60056e96a898", + // "ad5ed7d9-40a8-4b63-8ccd-346fe618744c", + // "6d6adca8-153c-44dc-ac99-e42ee41d2883", + // "dff18ffe-0ef4-484b-8354-583252291f39", + // "89befa8a-a98e-4912-9843-f861633c2ab8", + // "835c14d1-399e-4740-9e55-4fa83b6ce045", + // "7c96f67f-8b72-43fb-8129-29f168fa0dd8", + // "f16211d7-b2b9-4d02-9f76-d8c23853a91f", + // "7b1a000a-c0c9-411c-a5dc-622297a99b12", + // "0213a399-d761-434b-ae6e-7faa6b53809b", + // "05456e2e-dcd3-40de-ba43-f8dd4c8c34b9", + // "9efce5d6-818a-4d14-961f-c74111b4aab2", + // "0072cf6a-aef2-4fe6-b069-eec7c1f13c93", + // "40aa3f60-0532-4628-9871-baa54ddc76c0", + // "877b1e77-8bea-4163-81ce-7b7d110557f4", + // "806592b9-0683-4fb4-9afc-aee808941de0", + // "fb89e622-be0d-4e38-8f8a-687b118d9fbb", + // "32ab0418-63e6-4ef0-8e18-d830ded901c0", + // "468a66dc-f550-4c76-be49-2040c381f42f", + // "c8f66b78-3675-4d66-928e-ae37ea3d89fa", + // "0ea8c1bb-4ae7-4af6-b684-535e7cece274", + // "c7b06ef9-b9ad-481e-b61b-25bc4edfafd0", + // "a8414689-91b6-4d35-a53e-b49fafd39fc2", + // "79db2c3f-f544-46cb-b5e3-fb1a752880b8", + // "86584601-342c-4f1c-b739-fb20187e9c52", +]; + +async function insertBatch(records: any[]) { + const query = ` + INSERT INTO abby.ApiRequest (id, createdAt, type, durationInMs, apiVersion, projectId) + VALUES ${records.map((record) => `('${record.id}', '${record.createdAt}', '${record.type}', ${record.durationInMs}, '${record.apiVersion}', '${record.projectId}')`).join(",")} + `; + try { + await clickhouseClient.query({ + query: query, + }); + } catch (error) { + console.error("Error inserting batch:", error); + } +} + +async function generateAndInsertRecords() { + const records: any[] = []; + let batchCount = 0; + + for (let i = 0; i < NUM_ENTRIES; i++) { + const record = { + id: uuidv4(), + createdAt: randomDate(), + type: TYPES[Math.floor(Math.random() * TYPES.length)], + durationInMs: Math.floor(Math.random() * 1000) + 1, + apiVersion: API_VERSION, + projectId: projectIds[Math.floor(Math.random() * PROJECT_COUNT)], + }; + records.push(record); + + // Write in batches + if (records.length >= BATCH_SIZE) { + console.log(`Inserting batch ${++batchCount}`); + await insertBatch(records); + records.length = 0; // Clear the array + } + } + + if (records.length > 0) { + console.log(`Inserting final batch ${++batchCount}`); + await insertBatch(records); + } + + console.log("Data insertion completed successfully!"); +} + +generateAndInsertRecords().catch((err) => { + console.error("Error generating or inserting records:", err); +}); diff --git a/generate_csv/package.json b/generate_csv/package.json new file mode 100644 index 00000000..462f1733 --- /dev/null +++ b/generate_csv/package.json @@ -0,0 +1,24 @@ +{ + "name": "generate_csv", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "@clickhouse/client": "^1.0.1", + "@types/uuid": "^10.0.0", + "clickhouse": "^2.6.0", + "csv-writer": "^1.6.0", + "ts-node": "^10.9.1", + "typescript": "^5.5.4", + "uuid": "^10.0.0" + }, + "devDependencies": { + "@types/node": "^20.14.12" + } +} diff --git a/generate_csv/tsconfig.json b/generate_csv/tsconfig.json new file mode 100644 index 00000000..8d285974 --- /dev/null +++ b/generate_csv/tsconfig.json @@ -0,0 +1,23 @@ +{ + // This is an alias to @tsconfig/node16: https://github.com/tsconfig/bases + "extends": "ts-node/node16/tsconfig.json", + // Most ts-node options can be specified here using their programmatic names. + "ts-node": { + // It is faster to skip typechecking. + // Remove if you want ts-node to do typechecking. + "transpileOnly": true, + "files": true, + "compilerOptions": { + // This is needed to use the `import.meta` syntax. + "module": "ESNext", + // This is needed to use the `import.meta` syntax. + "target": "ESNext", + // This is needed to use the `import.meta` syntax. + "moduleResolution": "node", + // This is needed to use the `import.meta` syntax. + "esModuleInterop": true, + // This is needed to use the `import.meta` syntax. + "skipLibCheck": true + } + } +} From 1adaa808951435793a565620f348e2cf9837c51d Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sun, 28 Jul 2024 02:47:56 +0200 Subject: [PATCH 17/19] Move ApiRequest into clickchouse --- apps/web/src/server/queue/event.ts | 2 ++ .../server/services/ClickHouseEventService.ts | 27 +++++++++++++++++-- .../web/src/server/services/RequestService.ts | 22 +++++++++++---- 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/apps/web/src/server/queue/event.ts b/apps/web/src/server/queue/event.ts index b5d84f8e..ab825e85 100644 --- a/apps/web/src/server/queue/event.ts +++ b/apps/web/src/server/queue/event.ts @@ -32,6 +32,8 @@ const eventWorker = new Worker( event.type satisfies never; } } + + //could be moved into a cron job and checked only once a hour const { events, planLimits, plan, is80PercentOfLimit } = await ClickHouseEventService.getEventsForCurrentPeriod(event.projectId); diff --git a/apps/web/src/server/services/ClickHouseEventService.ts b/apps/web/src/server/services/ClickHouseEventService.ts index b129e14d..a5052dc5 100644 --- a/apps/web/src/server/services/ClickHouseEventService.ts +++ b/apps/web/src/server/services/ClickHouseEventService.ts @@ -5,6 +5,7 @@ import { AbbyEvent, AbbyEventType, assertUnreachable } from "@tryabby/core"; import { RequestCache } from "./RequestCache"; import { clickhouseClient } from "server/db/clickhouseClient"; import { z } from "zod"; +import dayjs from "dayjs"; const GroupedTestQueryResultSchema = z.object({ selectedVariant: z.string(), @@ -17,6 +18,10 @@ const GroupedTestQueryResultSchemaWithTimeSchema = z.intersection( z.object({ startTime: z.string() }) ); +const EventCurrentPeriodQueryResultSchema = z.object({ + apiRequestCount: z.string(), +}); + type GroupedTestQueryResultSchemaWithTime = z.infer< typeof GroupedTestQueryResultSchemaWithTimeSchema >; @@ -140,13 +145,29 @@ export abstract class ClickHouseEventService { const [project, eventCount] = await Promise.all([ prisma.project.findUnique({ where: { id: projectId }, - select: { stripePriceId: true }, + select: { stripePriceId: true, currentPeriodEnd: true }, }), RequestCache.get(projectId), ]); if (!project) throw new Error("Project not found"); + const billingPeriodStartDate = dayjs(project.currentPeriodEnd) + .subtract(30, "days") + .format("YYYY-MM-DD"); + + const res = await clickhouseClient + .query({ + query: ` + SELECT + Count(*) as apiRequestCount + FROM abby.ApiRequest + WHERE projectId = '${projectId}' AND createdAt >= toDate('${billingPeriodStartDate}'); + +`, + }) + .then((res) => res.json()); + const plan = Object.keys(PLANS).find( (plan) => PLANS[plan as PlanName] === project.stripePriceId ) as PlanName | undefined; @@ -154,7 +175,9 @@ export abstract class ClickHouseEventService { const planLimits = getLimitByPlan(plan ?? null); return { - events: eventCount, + events: parseInt( + EventCurrentPeriodQueryResultSchema.parse(res.data[0]).apiRequestCount + ), planLimits, plan, is80PercentOfLimit: planLimits.eventsPerMonth * 0.8 === eventCount, diff --git a/apps/web/src/server/services/RequestService.ts b/apps/web/src/server/services/RequestService.ts index 79a11f92..f16da152 100644 --- a/apps/web/src/server/services/RequestService.ts +++ b/apps/web/src/server/services/RequestService.ts @@ -1,12 +1,24 @@ import { ApiRequest } from "@prisma/client"; import { prisma } from "server/db/client"; +import { clickhouseClient } from "server/db/clickhouseClient"; export abstract class RequestService { static async storeRequest(request: Omit) { - await prisma.apiRequest.create({ - data: { - ...request, - }, - }); + const {} = await Promise.all([ + await prisma.apiRequest.create({ + data: { + ...request, + }, + }), + await clickhouseClient.insert({ + table: "abby.Event", + format: "JSONEachRow", + values: [ + { + project_id: request.projectId, + }, + ], + }), + ]); } } From f69dabb7d2a9a970f50f91598f1bb9248fefd4aa Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Sun, 28 Jul 2024 23:41:06 +0200 Subject: [PATCH 18/19] . --- generate_csv/generate_csv.js | 210 ------------------------------ generate_csv/generate_csv.ts | 66 ---------- generate_csv/insertInBatch.js | 232 ---------------------------------- generate_csv/insertInBatch.ts | 175 ------------------------- generate_csv/package.json | 24 ---- generate_csv/tsconfig.json | 23 ---- 6 files changed, 730 deletions(-) delete mode 100644 generate_csv/generate_csv.js delete mode 100644 generate_csv/generate_csv.ts delete mode 100644 generate_csv/insertInBatch.js delete mode 100644 generate_csv/insertInBatch.ts delete mode 100644 generate_csv/package.json delete mode 100644 generate_csv/tsconfig.json diff --git a/generate_csv/generate_csv.js b/generate_csv/generate_csv.js deleted file mode 100644 index 7692ce5e..00000000 --- a/generate_csv/generate_csv.js +++ /dev/null @@ -1,210 +0,0 @@ -"use strict"; -var __awaiter = - (this && this.__awaiter) || - function (thisArg, _arguments, P, generator) { - function adopt(value) { - return value instanceof P - ? value - : new P(function (resolve) { - resolve(value); - }); - } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { - try { - step(generator.next(value)); - } catch (e) { - reject(e); - } - } - function rejected(value) { - try { - step(generator["throw"](value)); - } catch (e) { - reject(e); - } - } - function step(result) { - result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); - } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); - }; -var __generator = - (this && this.__generator) || - function (thisArg, body) { - var _ = { - label: 0, - sent: function () { - if (t[0] & 1) throw t[1]; - return t[1]; - }, - trys: [], - ops: [], - }, - f, - y, - t, - g; - return ( - (g = { next: verb(0), throw: verb(1), return: verb(2) }), - typeof Symbol === "function" && - (g[Symbol.iterator] = function () { - return this; - }), - g - ); - function verb(n) { - return function (v) { - return step([n, v]); - }; - } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while ((g && ((g = 0), op[0] && (_ = 0)), _)) - try { - if ( - ((f = 1), - y && - (t = - op[0] & 2 - ? y["return"] - : op[0] - ? y["throw"] || ((t = y["return"]) && t.call(y), 0) - : y.next) && - !(t = t.call(y, op[1])).done) - ) - return t; - if (((y = 0), t)) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: - case 1: - t = op; - break; - case 4: - _.label++; - return { value: op[1], done: false }; - case 5: - _.label++; - y = op[1]; - op = [0]; - continue; - case 7: - op = _.ops.pop(); - _.trys.pop(); - continue; - default: - if ( - !((t = _.trys), (t = t.length > 0 && t[t.length - 1])) && - (op[0] === 6 || op[0] === 2) - ) { - _ = 0; - continue; - } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { - _.label = op[1]; - break; - } - if (op[0] === 6 && _.label < t[1]) { - _.label = t[1]; - t = op; - break; - } - if (t && _.label < t[2]) { - _.label = t[2]; - _.ops.push(op); - break; - } - if (t[2]) _.ops.pop(); - _.trys.pop(); - continue; - } - op = body.call(thisArg, _); - } catch (e) { - op = [6, e]; - y = 0; - } finally { - f = t = 0; - } - if (op[0] & 5) throw op[1]; - return { value: op[0] ? op[1] : void 0, done: true }; - } - }; -Object.defineProperty(exports, "__esModule", { value: true }); -var csv_writer_1 = require("csv-writer"); -var uuid_1 = require("uuid"); -// Constants -var NUM_ENTRIES = 10000000; -var PROJECT_COUNT = 100; -var API_VERSION = "V0"; -var TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; -// Generate a random date between today and one year ago -function randomDate() { - var today = new Date(); - var oneYearAgo = new Date(today); - oneYearAgo.setFullYear(today.getFullYear() - 1); - var randomTime = new Date( - oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime()) - ); - return randomTime.toISOString().replace("T", " ").substring(0, 23); -} -// Generate project IDs -var projectIds = ["clvh4sv5n0001furg6tj08z63"]; -// Setup CSV writer -var csvWriter = (0, csv_writer_1.createObjectCsvWriter)({ - path: "ApiRequestFürMeinProject.csv", - header: [ - { id: "id", title: "id" }, - { id: "createdAt", title: "createdAt" }, - { id: "type", title: "type" }, - { id: "durationInMs", title: "durationInMs" }, - { id: "apiVersion", title: "apiVersion" }, - { id: "projectId", title: "projectId" }, - ], -}); -function generateCSV() { - return __awaiter(this, void 0, void 0, function () { - var records, i, record; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - records = []; - i = 0; - _a.label = 1; - case 1: - if (!(i < NUM_ENTRIES)) return [3 /*break*/, 4]; - record = { - id: (0, uuid_1.v4)(), - createdAt: randomDate(), - type: TYPES[Math.floor(Math.random() * TYPES.length)], - durationInMs: Math.floor(Math.random() * 1000) + 1, - apiVersion: API_VERSION, - projectId: "clvh4sv5n0001furg6tj08z63", - // projectIds[Math.floor(Math.random() * PROJECT_COUNT)], - }; - records.push(record); - if (!(records.length === 100000)) return [3 /*break*/, 3]; - return [4 /*yield*/, csvWriter.writeRecords(records)]; - case 2: - _a.sent(); - records.length = 0; // Clear the array - _a.label = 3; - case 3: - i++; - return [3 /*break*/, 1]; - case 4: - if (!(records.length > 0)) return [3 /*break*/, 6]; - return [4 /*yield*/, csvWriter.writeRecords(records)]; - case 5: - _a.sent(); - _a.label = 6; - case 6: - console.log("CSV file generated successfully!"); - return [2 /*return*/]; - } - }); - }); -} -generateCSV().catch(function (err) { - console.error("Error generating CSV:", err); -}); diff --git a/generate_csv/generate_csv.ts b/generate_csv/generate_csv.ts deleted file mode 100644 index 33a5aacf..00000000 --- a/generate_csv/generate_csv.ts +++ /dev/null @@ -1,66 +0,0 @@ -import { createObjectCsvWriter } from "csv-writer"; -import { v4 as uuidv4 } from "uuid"; -import * as fs from "fs"; - -// Constants -const NUM_ENTRIES = 10_000_000; -const PROJECT_COUNT = 100; -const API_VERSION = "V0"; -const TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; - -// Generate a random date between today and one year ago -function randomDate(): string { - const today = new Date(); - const oneYearAgo = new Date(today); - oneYearAgo.setFullYear(today.getFullYear() - 1); - const randomTime = new Date( - oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime()) - ); - return randomTime.toISOString().replace("T", " ").substring(0, 23); -} - -// Generate project IDs -const projectIds = ["clvh4sv5n0001furg6tj08z63"]; - -// Setup CSV writer -const csvWriter = createObjectCsvWriter({ - path: "ApiRequest.csv", - header: [ - { id: "id", title: "id" }, - { id: "createdAt", title: "createdAt" }, - { id: "type", title: "type" }, - { id: "durationInMs", title: "durationInMs" }, - { id: "apiVersion", title: "apiVersion" }, - { id: "projectId", title: "projectId" }, - ], -}); - -async function generateCSV() { - const records: any = []; - for (let i = 0; i < NUM_ENTRIES; i++) { - const record = { - id: uuidv4(), - createdAt: randomDate(), - type: TYPES[Math.floor(Math.random() * TYPES.length)], - durationInMs: Math.floor(Math.random() * 1000) + 1, - apiVersion: API_VERSION, - projectId: "clvh4sv5n0001furg6tj08z63", - // projectIds[Math.floor(Math.random() * PROJECT_COUNT)], - }; - records.push(record); - - // Write in chunks to avoid memory issues - if (records.length === 100000) { - await csvWriter.writeRecords(records); - records.length = 0; // Clear the array - } - } - if (records.length > 0) { - await csvWriter.writeRecords(records); - } - console.log("CSV file generated successfully!"); -} - -generateCSV().catch((err) => { - console.error("Error generating CSV:", err); -}); diff --git a/generate_csv/insertInBatch.js b/generate_csv/insertInBatch.js deleted file mode 100644 index e296c8ce..00000000 --- a/generate_csv/insertInBatch.js +++ /dev/null @@ -1,232 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __generator = (this && this.__generator) || function (thisArg, body) { - var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g; - return g = { next: verb(0), "throw": verb(1), "return": verb(2) }, typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g; - function verb(n) { return function (v) { return step([n, v]); }; } - function step(op) { - if (f) throw new TypeError("Generator is already executing."); - while (g && (g = 0, op[0] && (_ = 0)), _) try { - if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t; - if (y = 0, t) op = [op[0] & 2, t.value]; - switch (op[0]) { - case 0: case 1: t = op; break; - case 4: _.label++; return { value: op[1], done: false }; - case 5: _.label++; y = op[1]; op = [0]; continue; - case 7: op = _.ops.pop(); _.trys.pop(); continue; - default: - if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; } - if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; } - if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; } - if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; } - if (t[2]) _.ops.pop(); - _.trys.pop(); continue; - } - op = body.call(thisArg, _); - } catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; } - if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true }; - } -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.clickhouseClient = void 0; -var uuid_1 = require("uuid"); -var client_1 = require("@clickhouse/client"); -exports.clickhouseClient = (0, client_1.createClient)(); -// Constants -var NUM_ENTRIES = 10000000 / 10000; // Number of entries you want to insert -var PROJECT_COUNT = 100; -var API_VERSION = "V0"; -var TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; -var BATCH_SIZE = 10000 / 10; // Size of each batch -// Generate a random date between today and one year ago -function randomDate() { - var today = new Date(); - var oneYearAgo = new Date(today); - oneYearAgo.setFullYear(today.getFullYear() - 1); - var randomTime = new Date(oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime())); - return randomTime.toISOString().replace("T", " ").substring(0, 19); -} -var projectIds = [ - "clvh4sv5n0001furg6tj08z63", - // "8e0ad326-c7d4-4daa-a004-585ca96d8e71", - // "4e236852-6ef1-44e2-a15e-9463fed3921f", - // "d4c607ea-d9e8-44a3-b12e-749225cb0599", - // "c4111aec-5677-4d2d-8d71-de8e49f79c26", - // "e4b69866-a985-4100-8bdc-64672e2ebefe", - // "8716a142-b424-48b6-8051-5ab5ce6b06cb", - // "20895eff-723c-4c76-8334-cf32ce72501a", - // "e00683fb-eca8-419c-a9e6-f4526ad93aa4", - // "43e225d6-d5ef-4c0c-a435-7ea1ebe2f74c", - // "ed4ffef2-790d-4850-b075-e589978a56c4", - // "f13778dd-b360-48be-ab7a-d97c048d99ba", - // "097cd30d-e654-44f5-b379-7a69c594e986", - // "d1e049e5-823d-44b3-845d-1ca9609cb657", - // "e23231e3-a18b-48ae-a496-a625c29ecdfe", - // "2f01a0b2-cc78-4f8e-b77e-322a3321ea44", - // "76a50abb-7908-4cce-b168-a8709965b5f9", - // "a3660a3f-525d-4d66-bf90-0bf2313ffbfc", - // "d9d3736e-1ba2-43bf-8c1c-b1c33436ceba", - // "e73dbf0d-4919-47d6-a63f-1fc1258a5673", - // "e859f1e1-f004-4a02-aecb-2ecd413aca3f", - // "0e3687cb-1cbc-44dd-a057-c0e35d9de8c2", - // "9868de0f-6e53-4e21-b480-cf97a8f6cb7d", - // "83a051fa-e4b3-4965-b0b9-a850ccc2d78d", - // "6d473975-5d1c-4ac5-9ba6-068cdca0c77e", - // "3b9fa79c-c190-4869-b15f-731f0167d453", - // "3c1c27db-ce33-4d21-9108-81b3b7f11a54", - // "28834d98-282b-49bd-978d-692f3da4003e", - // "cd64cfcc-02df-495c-a0a0-d9fe9a6ca474", - // "242d0023-50d7-467f-ac96-602b938c03a8", - // "56c4ccb6-f81d-4443-86d9-537e3bc341ed", - // "c743ef78-262c-4a37-ab7f-ea3178f3d2a5", - // "27ac8acf-b401-422d-93ce-6b6f861d9b19", - // "a09f6643-1a40-40d1-b9cd-e14c7d8a8584", - // "01b84b17-00fa-45e5-ac2f-2780c5476fe6", - // "98509dc4-4c90-4fb7-a473-f2db017f21ca", - // "37f0a317-c7ce-4e26-b190-e17191dc1b8a", - // "db0ec165-3fa3-4f0d-82aa-0fccb737f9ad", - // "ad8e5c90-f946-44a8-89e5-a34df667840d", - // "11dbb0f1-1b94-41c2-8737-116f2849a9ae", - // "d5c7330c-d959-47fc-b02f-39926595ccd2", - // "34fb504d-27b1-4053-af62-20767115278c", - // "e86905a1-5fbe-463c-ae35-274d6516df95", - // "2530fba6-cae4-4907-9369-5616b1aa29c2", - // "470d8dda-2e6c-40de-bfd8-6eccdb21169b", - // "83c04edc-659b-476b-8cf3-e0cad8c78c1d", - // "1f8937ba-f6fa-4dd6-ad4d-2fe88c7beed1", - // "573aa5e6-e66f-4c57-8de0-eb4af5be80f7", - // "2839c716-1bdf-4d1d-bcde-c85fd7879e47", - // "f6450cfc-2937-4d1c-8054-2a2ab56af0c8", - // "ac761d46-a959-435c-98ed-72e86f42dd91", - // "8ab88073-88f2-4fa1-9ece-2e77daf22900", - // "8f72ec17-2c08-452c-9f82-f8270b8c211f", - // "cf2cf72f-61f4-48da-b9d6-03860cdd1a51", - // "6887099b-a5ca-42f3-830f-6b9e17bcc405", - // "1408e392-4e39-4841-a722-dfc467b5aea7", - // "a7648ad8-bd9b-4def-ad30-8f6a8ffe790c", - // "dc1cdf87-7e65-4e35-8c84-53cc45061e5d", - // "a62d6dbe-df0b-40ca-93d9-0b175671f858", - // "51449cc3-25e0-46d4-b1db-afc21d274a77", - // "5581bc22-3f75-49dd-bad3-7b39c6a6fa47", - // "a9999a00-0a6f-43d8-b277-96fb027361fe", - // "cea38acb-76d5-407d-8c41-43faab812206", - // "9db2bb0e-e227-4527-8d91-5f6dff7f3e67", - // "72a47b93-313a-42f4-a51e-b850eb18e3fd", - // "05c44513-0a4b-4b98-8ffc-ae44dafa3a54", - // "4622ec76-0fd7-4226-bdaf-eceb7e2444f2", - // "bf49128d-7899-43d9-a2b0-64297df9f057", - // "313e1413-b3b5-4952-bd08-5f34fb7e5744", - // "d28ccf5b-3ea1-46a0-8ed1-56db7a52de7c", - // "3ebad352-ef41-4f15-931d-6f077c4e6307", - // "0f969120-52c3-436f-b027-9723e10c1780", - // "78462cc2-ef73-446e-a244-bf786ac481c2", - // "06be3cee-0603-41cb-affd-3d1c15056586", - // "26f0887c-617d-46eb-8629-eb0690a85387", - // "14778b80-cd6c-43df-9010-ef8c9bcf801b", - // "93db3706-6682-43f7-9720-60056e96a898", - // "ad5ed7d9-40a8-4b63-8ccd-346fe618744c", - // "6d6adca8-153c-44dc-ac99-e42ee41d2883", - // "dff18ffe-0ef4-484b-8354-583252291f39", - // "89befa8a-a98e-4912-9843-f861633c2ab8", - // "835c14d1-399e-4740-9e55-4fa83b6ce045", - // "7c96f67f-8b72-43fb-8129-29f168fa0dd8", - // "f16211d7-b2b9-4d02-9f76-d8c23853a91f", - // "7b1a000a-c0c9-411c-a5dc-622297a99b12", - // "0213a399-d761-434b-ae6e-7faa6b53809b", - // "05456e2e-dcd3-40de-ba43-f8dd4c8c34b9", - // "9efce5d6-818a-4d14-961f-c74111b4aab2", - // "0072cf6a-aef2-4fe6-b069-eec7c1f13c93", - // "40aa3f60-0532-4628-9871-baa54ddc76c0", - // "877b1e77-8bea-4163-81ce-7b7d110557f4", - // "806592b9-0683-4fb4-9afc-aee808941de0", - // "fb89e622-be0d-4e38-8f8a-687b118d9fbb", - // "32ab0418-63e6-4ef0-8e18-d830ded901c0", - // "468a66dc-f550-4c76-be49-2040c381f42f", - // "c8f66b78-3675-4d66-928e-ae37ea3d89fa", - // "0ea8c1bb-4ae7-4af6-b684-535e7cece274", - // "c7b06ef9-b9ad-481e-b61b-25bc4edfafd0", - // "a8414689-91b6-4d35-a53e-b49fafd39fc2", - // "79db2c3f-f544-46cb-b5e3-fb1a752880b8", - // "86584601-342c-4f1c-b739-fb20187e9c52", -]; -function insertBatch(records) { - return __awaiter(this, void 0, void 0, function () { - var query, error_1; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - query = "\n INSERT INTO abby.ApiRequest (id, createdAt, type, durationInMs, apiVersion, projectId)\n VALUES ".concat(records.map(function (record) { return "('".concat(record.id, "', '").concat(record.createdAt, "', '").concat(record.type, "', ").concat(record.durationInMs, ", '").concat(record.apiVersion, "', '").concat(record.projectId, "')"); }).join(","), "\n "); - _a.label = 1; - case 1: - _a.trys.push([1, 3, , 4]); - return [4 /*yield*/, exports.clickhouseClient.query({ - query: query, - })]; - case 2: - _a.sent(); - return [3 /*break*/, 4]; - case 3: - error_1 = _a.sent(); - console.error("Error inserting batch:", error_1); - return [3 /*break*/, 4]; - case 4: return [2 /*return*/]; - } - }); - }); -} -function generateAndInsertRecords() { - return __awaiter(this, void 0, void 0, function () { - var records, batchCount, i, record; - return __generator(this, function (_a) { - switch (_a.label) { - case 0: - records = []; - batchCount = 0; - i = 0; - _a.label = 1; - case 1: - if (!(i < NUM_ENTRIES)) return [3 /*break*/, 4]; - record = { - id: (0, uuid_1.v4)(), - createdAt: randomDate(), - type: TYPES[Math.floor(Math.random() * TYPES.length)], - durationInMs: Math.floor(Math.random() * 1000) + 1, - apiVersion: API_VERSION, - projectId: projectIds[Math.floor(Math.random() * PROJECT_COUNT)], - }; - records.push(record); - if (!(records.length >= BATCH_SIZE)) return [3 /*break*/, 3]; - console.log("Inserting batch ".concat(++batchCount)); - return [4 /*yield*/, insertBatch(records)]; - case 2: - _a.sent(); - records.length = 0; // Clear the array - _a.label = 3; - case 3: - i++; - return [3 /*break*/, 1]; - case 4: - if (!(records.length > 0)) return [3 /*break*/, 6]; - console.log("Inserting final batch ".concat(++batchCount)); - return [4 /*yield*/, insertBatch(records)]; - case 5: - _a.sent(); - _a.label = 6; - case 6: - console.log("Data insertion completed successfully!"); - return [2 /*return*/]; - } - }); - }); -} -generateAndInsertRecords().catch(function (err) { - console.error("Error generating or inserting records:", err); -}); diff --git a/generate_csv/insertInBatch.ts b/generate_csv/insertInBatch.ts deleted file mode 100644 index e0757925..00000000 --- a/generate_csv/insertInBatch.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { v4 as uuidv4 } from "uuid"; -import { createClient } from "@clickhouse/client"; - -export const clickhouseClient = createClient(); - -// Constants -const NUM_ENTRIES = 10_000_000 / 10000; // Number of entries you want to insert -const PROJECT_COUNT = 100; -const API_VERSION = "V0"; -const TYPES = ["GET_CONFIG", "POST_UPDATE", "DELETE_RECORD", "PUT_RESOURCE"]; -const BATCH_SIZE = 10000 / 10; // Size of each batch - -// Generate a random date between today and one year ago -function randomDate(): string { - const today = new Date(); - const oneYearAgo = new Date(today); - oneYearAgo.setFullYear(today.getFullYear() - 1); - const randomTime = new Date( - oneYearAgo.getTime() + Math.random() * (today.getTime() - oneYearAgo.getTime()) - ); - return randomTime.toISOString().replace("T", " ").substring(0, 19); -} - -const projectIds = [ - "clvh4sv5n0001furg6tj08z63", - // "8e0ad326-c7d4-4daa-a004-585ca96d8e71", - // "4e236852-6ef1-44e2-a15e-9463fed3921f", - // "d4c607ea-d9e8-44a3-b12e-749225cb0599", - // "c4111aec-5677-4d2d-8d71-de8e49f79c26", - // "e4b69866-a985-4100-8bdc-64672e2ebefe", - // "8716a142-b424-48b6-8051-5ab5ce6b06cb", - // "20895eff-723c-4c76-8334-cf32ce72501a", - // "e00683fb-eca8-419c-a9e6-f4526ad93aa4", - // "43e225d6-d5ef-4c0c-a435-7ea1ebe2f74c", - // "ed4ffef2-790d-4850-b075-e589978a56c4", - // "f13778dd-b360-48be-ab7a-d97c048d99ba", - // "097cd30d-e654-44f5-b379-7a69c594e986", - // "d1e049e5-823d-44b3-845d-1ca9609cb657", - // "e23231e3-a18b-48ae-a496-a625c29ecdfe", - // "2f01a0b2-cc78-4f8e-b77e-322a3321ea44", - // "76a50abb-7908-4cce-b168-a8709965b5f9", - // "a3660a3f-525d-4d66-bf90-0bf2313ffbfc", - // "d9d3736e-1ba2-43bf-8c1c-b1c33436ceba", - // "e73dbf0d-4919-47d6-a63f-1fc1258a5673", - // "e859f1e1-f004-4a02-aecb-2ecd413aca3f", - // "0e3687cb-1cbc-44dd-a057-c0e35d9de8c2", - // "9868de0f-6e53-4e21-b480-cf97a8f6cb7d", - // "83a051fa-e4b3-4965-b0b9-a850ccc2d78d", - // "6d473975-5d1c-4ac5-9ba6-068cdca0c77e", - // "3b9fa79c-c190-4869-b15f-731f0167d453", - // "3c1c27db-ce33-4d21-9108-81b3b7f11a54", - // "28834d98-282b-49bd-978d-692f3da4003e", - // "cd64cfcc-02df-495c-a0a0-d9fe9a6ca474", - // "242d0023-50d7-467f-ac96-602b938c03a8", - // "56c4ccb6-f81d-4443-86d9-537e3bc341ed", - // "c743ef78-262c-4a37-ab7f-ea3178f3d2a5", - // "27ac8acf-b401-422d-93ce-6b6f861d9b19", - // "a09f6643-1a40-40d1-b9cd-e14c7d8a8584", - // "01b84b17-00fa-45e5-ac2f-2780c5476fe6", - // "98509dc4-4c90-4fb7-a473-f2db017f21ca", - // "37f0a317-c7ce-4e26-b190-e17191dc1b8a", - // "db0ec165-3fa3-4f0d-82aa-0fccb737f9ad", - // "ad8e5c90-f946-44a8-89e5-a34df667840d", - // "11dbb0f1-1b94-41c2-8737-116f2849a9ae", - // "d5c7330c-d959-47fc-b02f-39926595ccd2", - // "34fb504d-27b1-4053-af62-20767115278c", - // "e86905a1-5fbe-463c-ae35-274d6516df95", - // "2530fba6-cae4-4907-9369-5616b1aa29c2", - // "470d8dda-2e6c-40de-bfd8-6eccdb21169b", - // "83c04edc-659b-476b-8cf3-e0cad8c78c1d", - // "1f8937ba-f6fa-4dd6-ad4d-2fe88c7beed1", - // "573aa5e6-e66f-4c57-8de0-eb4af5be80f7", - // "2839c716-1bdf-4d1d-bcde-c85fd7879e47", - // "f6450cfc-2937-4d1c-8054-2a2ab56af0c8", - // "ac761d46-a959-435c-98ed-72e86f42dd91", - // "8ab88073-88f2-4fa1-9ece-2e77daf22900", - // "8f72ec17-2c08-452c-9f82-f8270b8c211f", - // "cf2cf72f-61f4-48da-b9d6-03860cdd1a51", - // "6887099b-a5ca-42f3-830f-6b9e17bcc405", - // "1408e392-4e39-4841-a722-dfc467b5aea7", - // "a7648ad8-bd9b-4def-ad30-8f6a8ffe790c", - // "dc1cdf87-7e65-4e35-8c84-53cc45061e5d", - // "a62d6dbe-df0b-40ca-93d9-0b175671f858", - // "51449cc3-25e0-46d4-b1db-afc21d274a77", - // "5581bc22-3f75-49dd-bad3-7b39c6a6fa47", - // "a9999a00-0a6f-43d8-b277-96fb027361fe", - // "cea38acb-76d5-407d-8c41-43faab812206", - // "9db2bb0e-e227-4527-8d91-5f6dff7f3e67", - // "72a47b93-313a-42f4-a51e-b850eb18e3fd", - // "05c44513-0a4b-4b98-8ffc-ae44dafa3a54", - // "4622ec76-0fd7-4226-bdaf-eceb7e2444f2", - // "bf49128d-7899-43d9-a2b0-64297df9f057", - // "313e1413-b3b5-4952-bd08-5f34fb7e5744", - // "d28ccf5b-3ea1-46a0-8ed1-56db7a52de7c", - // "3ebad352-ef41-4f15-931d-6f077c4e6307", - // "0f969120-52c3-436f-b027-9723e10c1780", - // "78462cc2-ef73-446e-a244-bf786ac481c2", - // "06be3cee-0603-41cb-affd-3d1c15056586", - // "26f0887c-617d-46eb-8629-eb0690a85387", - // "14778b80-cd6c-43df-9010-ef8c9bcf801b", - // "93db3706-6682-43f7-9720-60056e96a898", - // "ad5ed7d9-40a8-4b63-8ccd-346fe618744c", - // "6d6adca8-153c-44dc-ac99-e42ee41d2883", - // "dff18ffe-0ef4-484b-8354-583252291f39", - // "89befa8a-a98e-4912-9843-f861633c2ab8", - // "835c14d1-399e-4740-9e55-4fa83b6ce045", - // "7c96f67f-8b72-43fb-8129-29f168fa0dd8", - // "f16211d7-b2b9-4d02-9f76-d8c23853a91f", - // "7b1a000a-c0c9-411c-a5dc-622297a99b12", - // "0213a399-d761-434b-ae6e-7faa6b53809b", - // "05456e2e-dcd3-40de-ba43-f8dd4c8c34b9", - // "9efce5d6-818a-4d14-961f-c74111b4aab2", - // "0072cf6a-aef2-4fe6-b069-eec7c1f13c93", - // "40aa3f60-0532-4628-9871-baa54ddc76c0", - // "877b1e77-8bea-4163-81ce-7b7d110557f4", - // "806592b9-0683-4fb4-9afc-aee808941de0", - // "fb89e622-be0d-4e38-8f8a-687b118d9fbb", - // "32ab0418-63e6-4ef0-8e18-d830ded901c0", - // "468a66dc-f550-4c76-be49-2040c381f42f", - // "c8f66b78-3675-4d66-928e-ae37ea3d89fa", - // "0ea8c1bb-4ae7-4af6-b684-535e7cece274", - // "c7b06ef9-b9ad-481e-b61b-25bc4edfafd0", - // "a8414689-91b6-4d35-a53e-b49fafd39fc2", - // "79db2c3f-f544-46cb-b5e3-fb1a752880b8", - // "86584601-342c-4f1c-b739-fb20187e9c52", -]; - -async function insertBatch(records: any[]) { - const query = ` - INSERT INTO abby.ApiRequest (id, createdAt, type, durationInMs, apiVersion, projectId) - VALUES ${records.map((record) => `('${record.id}', '${record.createdAt}', '${record.type}', ${record.durationInMs}, '${record.apiVersion}', '${record.projectId}')`).join(",")} - `; - try { - await clickhouseClient.query({ - query: query, - }); - } catch (error) { - console.error("Error inserting batch:", error); - } -} - -async function generateAndInsertRecords() { - const records: any[] = []; - let batchCount = 0; - - for (let i = 0; i < NUM_ENTRIES; i++) { - const record = { - id: uuidv4(), - createdAt: randomDate(), - type: TYPES[Math.floor(Math.random() * TYPES.length)], - durationInMs: Math.floor(Math.random() * 1000) + 1, - apiVersion: API_VERSION, - projectId: projectIds[Math.floor(Math.random() * PROJECT_COUNT)], - }; - records.push(record); - - // Write in batches - if (records.length >= BATCH_SIZE) { - console.log(`Inserting batch ${++batchCount}`); - await insertBatch(records); - records.length = 0; // Clear the array - } - } - - if (records.length > 0) { - console.log(`Inserting final batch ${++batchCount}`); - await insertBatch(records); - } - - console.log("Data insertion completed successfully!"); -} - -generateAndInsertRecords().catch((err) => { - console.error("Error generating or inserting records:", err); -}); diff --git a/generate_csv/package.json b/generate_csv/package.json deleted file mode 100644 index 462f1733..00000000 --- a/generate_csv/package.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "name": "generate_csv", - "version": "1.0.0", - "description": "", - "main": "index.js", - "scripts": { - "test": "echo \"Error: no test specified\" && exit 1" - }, - "keywords": [], - "author": "", - "license": "ISC", - "dependencies": { - "@clickhouse/client": "^1.0.1", - "@types/uuid": "^10.0.0", - "clickhouse": "^2.6.0", - "csv-writer": "^1.6.0", - "ts-node": "^10.9.1", - "typescript": "^5.5.4", - "uuid": "^10.0.0" - }, - "devDependencies": { - "@types/node": "^20.14.12" - } -} diff --git a/generate_csv/tsconfig.json b/generate_csv/tsconfig.json deleted file mode 100644 index 8d285974..00000000 --- a/generate_csv/tsconfig.json +++ /dev/null @@ -1,23 +0,0 @@ -{ - // This is an alias to @tsconfig/node16: https://github.com/tsconfig/bases - "extends": "ts-node/node16/tsconfig.json", - // Most ts-node options can be specified here using their programmatic names. - "ts-node": { - // It is faster to skip typechecking. - // Remove if you want ts-node to do typechecking. - "transpileOnly": true, - "files": true, - "compilerOptions": { - // This is needed to use the `import.meta` syntax. - "module": "ESNext", - // This is needed to use the `import.meta` syntax. - "target": "ESNext", - // This is needed to use the `import.meta` syntax. - "moduleResolution": "node", - // This is needed to use the `import.meta` syntax. - "esModuleInterop": true, - // This is needed to use the `import.meta` syntax. - "skipLibCheck": true - } - } -} From 492c17a8fd7034f114897c89ecea92b7910748ef Mon Sep 17 00:00:00 2001 From: Tim Trost Date: Tue, 30 Jul 2024 20:34:10 +0200 Subject: [PATCH 19/19] init setup bullmq-receiver --- .vscode/settings.json | 3 +- apps/bullmq-receiver/package.json | 22 +++++++ apps/bullmq-receiver/src/index.ts | 1 + pnpm-lock.yaml | 100 ++++++++++++++++++++++++++++-- 4 files changed, 121 insertions(+), 5 deletions(-) create mode 100644 apps/bullmq-receiver/package.json create mode 100644 apps/bullmq-receiver/src/index.ts diff --git a/.vscode/settings.json b/.vscode/settings.json index 29519146..8e6f3602 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -7,5 +7,6 @@ "editor.defaultFormatter": "esbenp.prettier-vscode" }, "eslint.workingDirectories": ["apps", "packages"], - "prettier.enable": true + "prettier.enable": true, + "cSpell.words": ["clvh", "clyetopos"] } diff --git a/apps/bullmq-receiver/package.json b/apps/bullmq-receiver/package.json new file mode 100644 index 00000000..de3fb608 --- /dev/null +++ b/apps/bullmq-receiver/package.json @@ -0,0 +1,22 @@ +{ + "name": "bullmq-receiver", + "version": "1.0.0", + "description": "", + "main": "index.js", + "scripts": { + "dev": "ts-node-dev --respawn --transpile-only src/index.ts", + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "dependencies": { + "bullmq": "^5.8.2", + "ioredis": "^5.4.1" + }, + "devDependencies": { + "@types/node": "^14.14.31", + "ts-node-dev": "^1.1.6", + "typescript": "^4.2.3" + } +} diff --git a/apps/bullmq-receiver/src/index.ts b/apps/bullmq-receiver/src/index.ts new file mode 100644 index 00000000..d914c606 --- /dev/null +++ b/apps/bullmq-receiver/src/index.ts @@ -0,0 +1 @@ +console.log("hi"); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 387ca8a0..5a1a36dc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -124,6 +124,25 @@ importers: specifier: ~4.9.4 version: 4.9.5 + apps/bullmq-receiver: + dependencies: + bullmq: + specifier: ^5.8.2 + version: 5.8.2 + ioredis: + specifier: ^5.4.1 + version: 5.4.1 + devDependencies: + '@types/node': + specifier: ^14.14.31 + version: 14.18.63 + ts-node-dev: + specifier: ^1.1.6 + version: 1.1.8(typescript@4.9.5) + typescript: + specifier: ^4.2.3 + version: 4.9.5 + apps/cdn: dependencies: hono: @@ -12187,6 +12206,10 @@ packages: resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} dev: true + /@types/node@14.18.63: + resolution: {integrity: sha512-fAtCfv4jJg+ExtXhvCkCqUKZ+4ok/JQk01qDKhL5BDDoS3AxKXhV5/MAVUZyQnSEd2GT92fkgZl0pz0Q0AzcIQ==} + dev: true + /@types/node@16.18.35: resolution: {integrity: sha512-yqU2Rf94HFZqgHf6Tuyc/IqVD0l3U91KjvypSr1GtJKyrnl6L/kfnxVqN4QOwcF5Zx9tO/HKK+fozGr5AtqA+g==} dev: true @@ -12345,6 +12368,14 @@ packages: resolution: {integrity: sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==} dev: true + /@types/strip-bom@3.0.0: + resolution: {integrity: sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==} + dev: true + + /@types/strip-json-comments@0.0.30: + resolution: {integrity: sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==} + dev: true + /@types/stripe-v3@3.1.28: resolution: {integrity: sha512-5poJyz1QFXpi1hE2bAWy7gFMdj5Fgofm94DNCaTK9V2LeWPdhCQIaP/6qUagZwgCcTURXzih1J7f3sjXH1cOsw==} dev: false @@ -13096,7 +13127,6 @@ packages: resolution: {integrity: sha512-RTvkC4w+KNXrM39/lWCUaG0IbRkWdCv7W/IOW9oU6SawyxulvkQy5HQPVTKxEjczcUvapcrw3cFx/60VN/NRNw==} engines: {node: '>=0.4.0'} hasBin: true - dev: false /acorn@8.8.2: resolution: {integrity: sha512-xjIYgE8HBrkpd/sJqOGNspf8uHG+NOHGOw6a/Urj8taM2EXfdNAH2oFcPeIFfsv3+kz/mJrS5VuMqbNLjCa2vw==} @@ -15803,6 +15833,12 @@ packages: stream-shift: 1.0.1 dev: true + /dynamic-dedupe@0.3.0: + resolution: {integrity: sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==} + dependencies: + xtend: 4.0.2 + dev: true + /eastasianwidth@0.2.0: resolution: {integrity: sha512-I88TYZWc9XiYHRQ4/3c5rjjfgkjhLyW2luGIheGERbNQ6OY7yTybanSpDXZa8y7VUP9YmDcYa+eyq4ca7iLqWA==} dev: true @@ -24559,7 +24595,7 @@ packages: resolution: {integrity: sha512-HFM8rkZ+i3zrV+4LQjwQ0W+ez98pApMGM3HUrN04j3CqzPOzl9nmP15Y8YXNm8QHGv/eacOVEjqhmWpkRV0NAw==} engines: {node: '>= 0.10'} dependencies: - resolve: 1.22.2 + resolve: 1.22.8 dev: true /redent@3.0.0: @@ -24927,6 +24963,7 @@ packages: /rimraf@2.6.3: resolution: {integrity: sha512-mwqeW5XsA2qAejG46gYdENaxXjx9onRNCfn7L0duuP4hCuTIi/QO7PDK07KJfp1d+izWPrzEJDcSqBa0OZQriA==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.3 @@ -24934,6 +24971,7 @@ packages: /rimraf@2.7.1: resolution: {integrity: sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==} + deprecated: Rimraf versions prior to v4 are no longer supported hasBin: true dependencies: glob: 7.2.3 @@ -26128,6 +26166,11 @@ packages: min-indent: 1.0.1 dev: true + /strip-json-comments@2.0.1: + resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} + engines: {node: '>=0.10.0'} + dev: true + /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} engines: {node: '>=8'} @@ -26798,6 +26841,30 @@ packages: /ts-interface-checker@0.1.13: resolution: {integrity: sha512-Y/arvbn+rrz3JCKl9C4kVNfTfSm2/mEp5FSz5EsZSANGPSlQrpRI5M4PKF+mJnE52jOO90PnPSc3Ur3bTQw0gA==} + /ts-node-dev@1.1.8(typescript@4.9.5): + resolution: {integrity: sha512-Q/m3vEwzYwLZKmV6/0VlFxcZzVV/xcgOt+Tx/VjaaRHyiBcFlV0541yrT09QjzzCxlDZ34OzKjrFAynlmtflEg==} + engines: {node: '>=0.8.0'} + hasBin: true + peerDependencies: + node-notifier: '*' + typescript: '*' + peerDependenciesMeta: + node-notifier: + optional: true + dependencies: + chokidar: 3.5.3 + dynamic-dedupe: 0.3.0 + minimist: 1.2.8 + mkdirp: 1.0.4 + resolve: 1.22.8 + rimraf: 2.7.1 + source-map-support: 0.5.21 + tree-kill: 1.2.2 + ts-node: 9.1.1(typescript@4.9.5) + tsconfig: 7.0.0 + typescript: 4.9.5 + dev: true + /ts-node@10.9.1(@types/node@18.16.17)(typescript@4.9.5): resolution: {integrity: sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==} hasBin: true @@ -26848,8 +26915,8 @@ packages: '@tsconfig/node14': 1.0.3 '@tsconfig/node16': 1.0.4 '@types/node': 20.3.1 - acorn: 8.8.2 - acorn-walk: 8.2.0 + acorn: 8.12.0 + acorn-walk: 8.3.2 arg: 4.1.3 create-require: 1.1.1 diff: 4.0.2 @@ -26858,6 +26925,22 @@ packages: v8-compile-cache-lib: 3.0.1 yn: 3.1.1 + /ts-node@9.1.1(typescript@4.9.5): + resolution: {integrity: sha512-hPlt7ZACERQGf03M253ytLY3dHbGNGrAq9qIHWUY9XHYl1z7wYngSr3OQ5xmui8o2AaxsONxIzjafLUiWBo1Fg==} + engines: {node: '>=10.0.0'} + hasBin: true + peerDependencies: + typescript: '>=2.7' + dependencies: + arg: 4.1.3 + create-require: 1.1.1 + diff: 4.0.2 + make-error: 1.3.6 + source-map-support: 0.5.21 + typescript: 4.9.5 + yn: 3.1.1 + dev: true + /ts-pattern@4.3.0: resolution: {integrity: sha512-pefrkcd4lmIVR0LA49Imjf9DYLK8vtWhqBPA3Ya1ir8xCW0O2yjL9dsCVvI7pCodLC5q7smNpEtDR2yVulQxOg==} dev: false @@ -26904,6 +26987,15 @@ packages: strip-bom: 3.0.0 dev: true + /tsconfig@7.0.0: + resolution: {integrity: sha512-vZXmzPrL+EmC4T/4rVlT2jNVMWCi/O4DIiSj3UHg1OE5kCKbk4mfrXc6dZksLgRM/TZlKnousKH9bbTazUWRRw==} + dependencies: + '@types/strip-bom': 3.0.0 + '@types/strip-json-comments': 0.0.30 + strip-bom: 3.0.0 + strip-json-comments: 2.0.1 + dev: true + /tslib@1.14.1: resolution: {integrity: sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg==}