-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathdb.ts
More file actions
101 lines (88 loc) · 3.01 KB
/
db.ts
File metadata and controls
101 lines (88 loc) · 3.01 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
import { PrismaClient } from "@prisma/client";
import { alignDate, getFirstFreeSpot } from "./scheduler";
import { IS_DEV } from "./util";
export const prisma = new PrismaClient();
export async function getSetting<T = any>(key: string, ifNotExist?: T): Promise<T> {
let value = await prisma.settings.findUnique({
where: {
key: key,
},
});
if (value === null) {
return ifNotExist!;
} else {
return value.value as T;
}
}
export async function setSettingDefaultValue(key: string, defaultValue: any) {
try {
await prisma.settings.create({
data: {
key: key,
value: defaultValue,
},
});
} catch (ex) {}
}
export async function setSetting(key: string, value: any) {
await prisma.settings.update({
where: {
key: key,
},
data: {
value: value,
},
});
}
export const SETTING_MAX_TIME_BEFORE_APPROVAL = "maxTimeBeforeApprovalRequired";
export const SETTING_GPU_COUNT = "gpuCount";
export const SETTING_MULTI_GPU_APPROVAL = "approvalRequiredForMultiGpu";
setSettingDefaultValue(SETTING_MAX_TIME_BEFORE_APPROVAL, 1000 * 60 * 60 * 8);
setSettingDefaultValue(SETTING_GPU_COUNT, 2);
setSettingDefaultValue(SETTING_MULTI_GPU_APPROVAL, true);
export async function findScheduleSpot(trainMilliseconds: number, allGpus: boolean): Promise<{ date: Date; gpus: number[] }> {
let minDate = alignDate(new Date(), IS_DEV ? 1000 * 60 * 60 : 1000 * 60 * 60);
let nextTasks = await prisma.task.findMany({
where: {
endDate: {
gte: minDate,
},
},
orderBy: {
startDate: "asc",
},
});
let gpuCount = await getSetting<number>(SETTING_GPU_COUNT);
let scheduleDate: Date;
let scheduleGpus = [] as number[];
if (allGpus) {
// Find a spot where all the gpus aren't used
scheduleDate = getFirstFreeSpot(nextTasks, trainMilliseconds, minDate);
scheduleGpus = new Array(gpuCount).fill(0).map((_, i) => i);
} else {
// Find a spot on each specific gpu
let scheduleDateCandidates = new Array(gpuCount);
for (let g = 0; g < gpuCount; g++) {
let gpuSpecificScheduleDate = getFirstFreeSpot(
nextTasks.filter((e) => e.gpus.includes(g)),
trainMilliseconds,
minDate
);
scheduleDateCandidates[g] = gpuSpecificScheduleDate;
}
// Use the earliest available spot
scheduleDate = scheduleDateCandidates[0];
scheduleGpus = [0];
for (let g = 1; g < scheduleDateCandidates.length; g++) {
let gpuSpecificScheduleDate = scheduleDateCandidates[g];
if (gpuSpecificScheduleDate.getTime() < scheduleDate.getTime()) {
scheduleDate = gpuSpecificScheduleDate;
scheduleGpus = [g];
}
}
}
return {
date: scheduleDate,
gpus: scheduleGpus,
};
}