From 0adf9cb65f852dede4e6c683eab4c47128d10a41 Mon Sep 17 00:00:00 2001 From: William Phetsinorath Date: Wed, 25 Feb 2026 12:42:56 +0100 Subject: [PATCH] fix(gitlab): iterate over pagination Current implementation assume that the API return the whole list, which is not truth. Related: https://github.com/cloud-pi-native/console/issues/1916 Co-authored-by: William Phetsinorath Signed-off-by: William Phetsinorath --- plugins/gitlab/src/class.ts | 74 ++++++++++++++---------------- plugins/gitlab/src/user.ts | 31 ++++++------- plugins/gitlab/src/utils.ts | 71 +++++++++++++++++++++------- plugins/sonarqube/src/functions.ts | 9 ++-- plugins/sonarqube/src/user.ts | 26 ++++++++--- 5 files changed, 125 insertions(+), 86 deletions(-) diff --git a/plugins/gitlab/src/class.ts b/plugins/gitlab/src/class.ts index ce8d9e808..54a882aef 100644 --- a/plugins/gitlab/src/class.ts +++ b/plugins/gitlab/src/class.ts @@ -1,12 +1,11 @@ import { createHash } from 'node:crypto' import { PluginApi, type Project, type UniqueRepo } from '@cpn-console/hooks' -import type { AccessTokenScopes, CommitAction, GroupSchema, GroupStatisticsSchema, MemberSchema, ProjectVariableSchema, VariableSchema } from '@gitbeaker/rest' -import type { AllRepositoryTreesOptions, CondensedProjectSchema, Gitlab, PaginationRequestOptions, ProjectSchema, RepositoryFileExpandedSchema, RepositoryTreeSchema } from '@gitbeaker/core' +import type { AccessTokenScopes, CommitAction, GroupSchema, MemberSchema, ProjectVariableSchema, VariableSchema, AllRepositoryTreesOptions, CondensedProjectSchema, Gitlab, ProjectSchema, RepositoryFileExpandedSchema } from '@gitbeaker/core' import { AccessLevel } from '@gitbeaker/core' import type { VaultProjectApi } from '@cpn-console/vault-plugin/types/vault-project-api.js' import { objectEntries } from '@cpn-console/shared' import type { GitbeakerRequestError } from '@gitbeaker/requester-utils' -import { getApi, getGroupRootId, infraAppsRepoName, internalMirrorRepoName } from './utils.js' +import { find, getApi, getAll, getGroupRootId, infraAppsRepoName, internalMirrorRepoName, offsetPaginate } from './utils.js' import config from './config.js' type setVariableResult = 'created' | 'updated' | 'already up-to-date' @@ -69,8 +68,8 @@ export class GitlabApi extends PluginApi { ): Promise { let action: CommitAction['action'] = 'create' - const branches = await this.api.Branches.all(repoId) - if (branches.some(b => b.name === branch)) { + const existingBranch = await find(offsetPaginate(opts => this.api.Branches.all(repoId, opts)), b => b.name === branch) + if (existingBranch) { let actualFile: RepositoryFileExpandedSchema | undefined try { actualFile = await this.api.RepositoryFiles.show(repoId, filePath, branch) @@ -152,12 +151,12 @@ export class GitlabApi extends PluginApi { return filesUpdated } - public async listFiles(repoId: number, options: AllRepositoryTreesOptions & PaginationRequestOptions<'keyset'> = {}) { + public async listFiles(repoId: number, options: AllRepositoryTreesOptions = {}) { options.path = options?.path ?? '/' options.ref = options?.ref ?? 'main' options.recursive = options?.recursive ?? false try { - const files: RepositoryTreeSchema[] = await this.api.Repositories.allRepositoryTrees(repoId, options) + const files = await this.api.Repositories.allRepositoryTrees(repoId, options) // if (depth >= 0) { // for (const file of files) { // if (file.type !== 'tree') { @@ -199,8 +198,11 @@ export class GitlabZoneApi extends GitlabApi { public async getOrCreateInfraGroup(): Promise { const rootId = await getGroupRootId() // Get or create projects_root_dir/infra group - const searchResult = await this.api.Groups.search(infraGroupName) - const existingParentGroup = searchResult.find(group => group.parent_id === rootId && group.name === infraGroupName) + const existingParentGroup = await find(offsetPaginate(opts => this.api.Groups.all({ + search: infraGroupName, + orderBy: 'id', + ...opts, + })), group => group.parent_id === rootId && group.name === infraGroupName) return existingParentGroup || await this.api.Groups.create(infraGroupName, infraGroupPath, { parentId: rootId, projectCreationLevel: 'maintainer', @@ -216,18 +218,16 @@ export class GitlabZoneApi extends GitlabApi { } const infraGroup = await this.getOrCreateInfraGroup() // Get or create projects_root_dir/infra/zone - const infraProjects = await this.api.Groups.allProjects(infraGroup.id, { + const project = await find(offsetPaginate(opts => this.api.Groups.allProjects(infraGroup.id, { search: zone, simple: true, - perPage: 100, - }) - const project: ProjectSchema = infraProjects.find(repo => repo.name === zone) ?? await this.createEmptyRepository({ + ...opts, + })), repo => repo.name === zone) ?? await this.createEmptyRepository({ repoName: zone, groupId: infraGroup.id, description: 'Repository hosting deployment files for this zone.', createFirstCommit: true, - }, - ) + }) this.infraProjectsByZoneSlug.set(zone, project) return project } @@ -235,7 +235,7 @@ export class GitlabZoneApi extends GitlabApi { export class GitlabProjectApi extends GitlabApi { private project: Project | UniqueRepo - private gitlabGroup: GroupSchema & { statistics: GroupStatisticsSchema } | undefined + private gitlabGroup: GroupSchema | undefined private specialRepositories: string[] = [infraAppsRepoName, internalMirrorRepoName] private zoneApi: GitlabZoneApi @@ -248,9 +248,12 @@ export class GitlabProjectApi extends GitlabApi { // Group Project private async createProjectGroup(): Promise { - const searchResult = await this.api.Groups.search(this.project.slug) const parentId = await getGroupRootId() - const existingGroup = searchResult.find(group => group.parent_id === parentId && group.name === this.project.slug) + const existingGroup = await find(offsetPaginate(opts => this.api.Groups.all({ + search: this.project.slug, + orderBy: 'id', + ...opts, + })), group => group.parent_id === parentId && group.name === this.project.slug) if (existingGroup) return existingGroup @@ -265,8 +268,7 @@ export class GitlabProjectApi extends GitlabApi { public async getProjectGroup(): Promise { if (this.gitlabGroup) return this.gitlabGroup const parentId = await getGroupRootId() - const searchResult = await this.api.Groups.allSubgroups(parentId) - this.gitlabGroup = searchResult.find(group => group.name === this.project.slug) + this.gitlabGroup = await find(offsetPaginate(opts => this.api.Groups.allSubgroups(parentId, opts)), group => group.name === this.project.slug) return this.gitlabGroup } @@ -323,21 +325,15 @@ export class GitlabProjectApi extends GitlabApi { public async getProjectId(projectName: string) { const projectGroup = await this.getProjectGroup() - if (!projectGroup) { - throw new Error('Parent DSO Project group has not been created yet') - } - const projectsInGroup = await this.api.Groups.allProjects(projectGroup.id, { + if (!projectGroup) throw new Error(`Gitlab inaccessible, impossible de trouver le groupe ${this.project.slug}`) + + const project = await find(offsetPaginate(opts => this.api.Groups.allProjects(projectGroup.id, { search: projectName, simple: true, - perPage: 100, - }) - const project = projectsInGroup.find(p => p.path === projectName) + ...opts, + })), repo => repo.name === projectName) - if (!project) { - const pathProjectName = `${config().projectsRootDir}/${this.project.slug}/${projectName}` - throw new Error(`Gitlab project "${pathProjectName}" not found`) - } - return project.id + return project?.id } public async getProjectById(projectId: number) { @@ -351,8 +347,7 @@ export class GitlabProjectApi extends GitlabApi { public async getProjectToken(tokenName: string) { const group = await this.getProjectGroup() if (!group) throw new Error('Unable to retrieve gitlab project group') - const groupTokens = await this.api.GroupAccessTokens.all(group.id) - return groupTokens.find(token => token.name === tokenName) + return find(offsetPaginate(opts => this.api.GroupAccessTokens.all(group.id, opts)), token => token.name === tokenName) } public async createProjectToken(tokenName: string, scopes: AccessTokenScopes[]) { @@ -375,8 +370,7 @@ export class GitlabProjectApi extends GitlabApi { const gitlabRepositories = await this.listRepositories() const mirrorRepo = gitlabRepositories.find(repo => repo.name === internalMirrorRepoName) if (!mirrorRepo) throw new Error('Don\'t know how mirror repo could not exist') - const allTriggerTokens = await this.api.PipelineTriggerTokens.all(mirrorRepo.id) - const currentTriggerToken = allTriggerTokens.find(token => token.description === tokenDescription) + const currentTriggerToken = await find(offsetPaginate(opts => this.api.PipelineTriggerTokens.all(mirrorRepo.id, opts)), token => token.description === tokenDescription) const tokenVaultSecret = await vaultApi.read('GITLAB', { throwIfNoEntry: false }) @@ -398,7 +392,7 @@ export class GitlabProjectApi extends GitlabApi { public async listRepositories() { const group = await this.getOrCreateProjectGroup() - const projects = await this.api.Groups.allProjects(group.id, { simple: false }) // to refactor with https://github.com/jdalrymple/gitbeaker/pull/3624 + const projects = await getAll(offsetPaginate(opts => this.api.Groups.allProjects(group.id, { simple: false, ...opts }))) // to refactor with https://github.com/jdalrymple/gitbeaker/pull/3624 return Promise.all(projects.map(async (project) => { if (this.specialRepositories.includes(project.name) && (!project.topics || !project.topics.includes(pluginManagedTopic))) { return this.api.Projects.edit(project.id, { topics: project.topics ? [...project.topics, pluginManagedTopic] : [pluginManagedTopic] }) @@ -432,7 +426,7 @@ export class GitlabProjectApi extends GitlabApi { // Group members public async getGroupMembers() { const group = await this.getOrCreateProjectGroup() - return this.api.GroupMembers.all(group.id) + return getAll(offsetPaginate(opts => this.api.GroupMembers.all(group.id, opts))) } public async addGroupMember(userId: number, accessLevel: AccessLevelAllowed = AccessLevel.DEVELOPER): Promise { @@ -448,7 +442,7 @@ export class GitlabProjectApi extends GitlabApi { // CI Variables public async getGitlabGroupVariables(): Promise { const group = await this.getOrCreateProjectGroup() - return await this.api.GroupVariables.all(group.id) + return await getAll(offsetPaginate(opts => this.api.GroupVariables.all(group.id, opts))) } public async setGitlabGroupVariable(listVars: VariableSchema[], toSetVariable: VariableSchema): Promise { @@ -491,7 +485,7 @@ export class GitlabProjectApi extends GitlabApi { } public async getGitlabRepoVariables(repoId: number): Promise { - return await this.api.ProjectVariables.all(repoId) + return await getAll(offsetPaginate(opts => this.api.ProjectVariables.all(repoId, opts))) } public async setGitlabRepoVariable(repoId: number, listVars: VariableSchema[], toSetVariable: ProjectVariableSchema): Promise { diff --git a/plugins/gitlab/src/user.ts b/plugins/gitlab/src/user.ts index cb984c036..48efa5b49 100644 --- a/plugins/gitlab/src/user.ts +++ b/plugins/gitlab/src/user.ts @@ -1,28 +1,19 @@ import type { UserObject } from '@cpn-console/hooks' import type { CreateUserOptions, SimpleUserSchema } from '@gitbeaker/rest' -import { getApi } from './utils.js' +import { getApi, find, offsetPaginate } from './utils.js' export const createUsername = (email: string) => email.replace('@', '.') export async function getUser(user: { email: string, username: string, id: string }): Promise { const api = getApi() - let gitlabUser: SimpleUserSchema | undefined - - // test finding by extern_uid by searching with email - const usersByEmail = await api.Users.all({ search: user.email }) - gitlabUser = usersByEmail.find(gitlabUser => gitlabUser?.externUid === user.id) - if (gitlabUser) return gitlabUser - - // if not found, test finding by extern_uid by searching with username - const usersByUsername = await api.Users.all({ username: user.username }) - gitlabUser = usersByUsername.find(gitlabUser => gitlabUser?.externUid === user.id) - if (gitlabUser) return gitlabUser - - // if not found, test finding by email or username - const allUsers = [...usersByEmail, ...usersByUsername] - return allUsers.find(gitlabUser => gitlabUser.email === user.email) - || allUsers.find(gitlabUser => gitlabUser.username === user.username) + return find( + offsetPaginate(opts => api.Users.all({ ...opts, asAdmin: true })), + gitlabUser => + gitlabUser?.externUid === user.id + || gitlabUser.email === user.email + || gitlabUser.username === user.username, + ) } export async function upsertUser(user: UserObject): Promise { @@ -57,7 +48,11 @@ export async function upsertUser(user: UserObject): Promise { console.log(`Gitlab plugin: Updating user: ${user.email}`) console.log(incorrectProps) } - await api.Users.edit(existingUser.id, userDefinitionBase) + try { + await api.Users.edit(existingUser.id, userDefinitionBase) + } catch (err) { + console.error(`Gitlab plugin: Failed to update user: ${user.email} for ${err}`) + } } return existingUser } diff --git a/plugins/gitlab/src/utils.ts b/plugins/gitlab/src/utils.ts index 8a9463814..1a085cc10 100644 --- a/plugins/gitlab/src/utils.ts +++ b/plugins/gitlab/src/utils.ts @@ -1,5 +1,5 @@ import { Gitlab } from '@gitbeaker/rest' -import type { Gitlab as IGitlab } from '@gitbeaker/core' +import type { Gitlab as IGitlab, BaseRequestOptions, PaginationRequestOptions, OffsetPagination } from '@gitbeaker/core' import { GitbeakerRequestError } from '@gitbeaker/requester-utils' import config from './config.js' @@ -13,8 +13,12 @@ export async function getGroupRootId(throwIfNotFound?: boolean): Promise grp.full_path === projectRootDir))?.id + const groupRoot = await find(offsetPaginate(opts => gitlabApi.Groups.all({ + search: projectRootDir, + orderBy: 'id', + ...opts, + })), grp => grp.full_path === projectRootDir) + const searchId = groupRoot?.id if (typeof searchId === 'undefined') { if (throwIfNotFound) { throw new Error(`Gitlab inaccessible, impossible de trouver le groupe ${projectRootDir}`) @@ -35,9 +39,11 @@ async function createGroupRoot(): Promise { throw new Error('No projectRootDir available') } - let parentGroup = (await gitlabApi.Groups.search(rootGroupPath)) - .find(grp => grp.full_path === rootGroupPath) - ?? await gitlabApi.Groups.create(rootGroupPath, rootGroupPath) + let parentGroup = await find(offsetPaginate(opts => gitlabApi.Groups.all({ + search: rootGroupPath, + orderBy: 'id', + ...opts, + })), grp => grp.full_path === rootGroupPath) ?? await gitlabApi.Groups.create(rootGroupPath, rootGroupPath) if (parentGroup.full_path === projectRootDir) { return parentGroup.id @@ -45,9 +51,11 @@ async function createGroupRoot(): Promise { for (const path of projectRootDirArray) { const futureFullPath = `${parentGroup.full_path}/${path}` - parentGroup = (await gitlabApi.Groups.search(futureFullPath)) - .find(grp => grp.full_path === futureFullPath) - ?? await gitlabApi.Groups.create(path, path, { parentId: parentGroup.id, visibility: 'internal' }) + parentGroup = await find(offsetPaginate(opts => gitlabApi.Groups.all({ + search: futureFullPath, + orderBy: 'id', + ...opts, + })), grp => grp.full_path === futureFullPath) ?? await gitlabApi.Groups.create(path, path, { parentId: parentGroup.id, visibility: 'internal' }) if (parentGroup.full_path === projectRootDir) { return parentGroup.id @@ -57,17 +65,11 @@ async function createGroupRoot(): Promise { } export async function getOrCreateGroupRoot(): Promise { - let rootId = await getGroupRootId(false) - if (typeof rootId === 'undefined') { - rootId = await createGroupRoot() - } - return rootId + return await getGroupRootId(false) ?? createGroupRoot() } export function getApi(): IGitlab { - if (!api) { - api = new Gitlab({ token: config().token, host: config().internalUrl }) - } + api ??= new Gitlab({ token: config().token, host: config().internalUrl }) return api } @@ -89,3 +91,38 @@ export function cleanGitlabError(error: T): T { } return error } + +export async function* offsetPaginate( + request: (options: PaginationRequestOptions<'offset'> & BaseRequestOptions) => Promise<{ data: T[], paginationInfo: OffsetPagination }>, +): AsyncGenerator { + let page: number | null = 1 + while (page !== null) { + const { data, paginationInfo } = await request({ page, showExpanded: true, pagination: 'offset' }) + for (const item of data) { + yield item + } + page = paginationInfo.next + } +} + +export async function getAll( + iterable: AsyncIterable, +): Promise { + const items: T[] = [] + for await (const item of iterable) { + items.push(item) + } + return items +} + +export async function find( + iterable: AsyncIterable, + predicate: (item: T) => boolean, +): Promise { + for await (const item of iterable) { + if (predicate(item)) { + return item + } + } + return undefined +} diff --git a/plugins/sonarqube/src/functions.ts b/plugins/sonarqube/src/functions.ts index b31747c7b..9f00644c6 100644 --- a/plugins/sonarqube/src/functions.ts +++ b/plugins/sonarqube/src/functions.ts @@ -119,13 +119,13 @@ export const upsertProject: StepCall = async (payload) => { // Remove excess repositories ...sonarRepositories - .filter(sonarRepository => !project.repositories.find(repo => repo.internalRepoName === sonarRepository.repository)) + .filter(sonarRepository => !project.repositories.some(repo => repo.internalRepoName === sonarRepository.repository)) .map(sonarRepository => deleteDsoRepository(sonarRepository.key)), // Create or configure needed repos ...project.repositories.map(async (repository) => { const projectKey = generateProjectKey(projectSlug, repository.internalRepoName) - if (!sonarRepositories.find(sonarRepository => sonarRepository.repository === repository.internalRepoName)) { + if (!sonarRepositories.some(sonarRepository => sonarRepository.repository === repository.internalRepoName)) { await createDsoRepository(projectSlug, repository.internalRepoName) } await ensureRepositoryConfiguration(projectKey, username, keycloakGroupPath) @@ -166,6 +166,7 @@ export const setVariables: StepCall = async (payload) => { ...project.repositories.map(async (repo) => { const projectKey = generateProjectKey(projectSlug, repo.internalRepoName) const repoId = await payload.apis.gitlab.getProjectId(repo.internalRepoName) + if (!repoId) return const listVars = await gitlabApi.getGitlabRepoVariables(repoId) return [ await gitlabApi.setGitlabRepoVariable(repoId, listVars, { @@ -193,9 +194,9 @@ export const setVariables: StepCall = async (payload) => { environment_scope: '*', }), ] - }).flat(), + }), // Sonar vars saving in CI (group) - await gitlabApi.setGitlabGroupVariable(listGroupVars, { + gitlabApi.setGitlabGroupVariable(listGroupVars, { key: 'SONAR_TOKEN', masked: true, protected: false, diff --git a/plugins/sonarqube/src/user.ts b/plugins/sonarqube/src/user.ts index 5402ccc84..f5d98bbfc 100644 --- a/plugins/sonarqube/src/user.ts +++ b/plugins/sonarqube/src/user.ts @@ -62,13 +62,25 @@ export async function changeToken(username: string) { export async function getUser(username: string): Promise { const axiosInstance = getAxiosInstance() - const users: { paging: SonarPaging, users: SonarUser[] } = (await axiosInstance({ - url: 'users/search', - params: { - q: username, - }, - }))?.data - return users.users.find(u => u.login === username) + let page = 1 + const pageSize = 100 + while (true) { + const response = await axiosInstance({ + url: 'users/search', + params: { + q: username, + ps: pageSize, + p: page, + }, + }) + const users: { paging: SonarPaging, users: SonarUser[] } = response.data + const found = users.users.find(user => user.login === username) + if (found) return found + if (!users.users.length || users.paging.pageIndex * users.paging.pageSize >= users.paging.total) { + break + } + page += 1 + } } export async function ensureUserExists(username: string, projectSlug: string, vaultUserSecret: VaultSonarSecret | undefined): Promise {