fix(gitlab): iterate over pagination

Current implementation assume that the API return the whole list, which is not truth.

Related: #1916
Co-authored-by: William Phetsinorath <william.phetsinorath@shikanime.studio>
Signed-off-by: William Phetsinorath <william.phetsinorath-open@interieur.gouv.fr>

Author: shikanime

plugins/gitlab/src/class.ts

@@ -1,12 +1,11 @@
 import { createHash } from 'node:crypto'
 import { PluginApi, type Project, type UniqueRepo } from '@cpn-console/hooks'
-import type { AccessTokenScopes, CommitAction, GroupSchema, GroupStatisticsSchema, MemberSchema, ProjectVariableSchema, VariableSchema } from '@gitbeaker/rest'
-import type { AllRepositoryTreesOptions, CondensedProjectSchema, Gitlab, PaginationRequestOptions, ProjectSchema, RepositoryFileExpandedSchema, RepositoryTreeSchema } from '@gitbeaker/core'
+import type { AccessTokenScopes, CommitAction, GroupSchema, MemberSchema, ProjectVariableSchema, VariableSchema, AllRepositoryTreesOptions, CondensedProjectSchema, Gitlab, ProjectSchema, RepositoryFileExpandedSchema } from '@gitbeaker/core'
 import { AccessLevel } from '@gitbeaker/core'
 import type { VaultProjectApi } from '@cpn-console/vault-plugin/types/vault-project-api.js'
 import { objectEntries } from '@cpn-console/shared'
 import type { GitbeakerRequestError } from '@gitbeaker/requester-utils'
-import { getApi, getGroupRootId, infraAppsRepoName, internalMirrorRepoName } from './utils.js'
+import { find, getApi, getAll, getGroupRootId, infraAppsRepoName, internalMirrorRepoName, offsetPaginate } from './utils.js'
 import config from './config.js'
 
 type setVariableResult = 'created' | 'updated' | 'already up-to-date'
@@ -69,8 +68,8 @@ export class GitlabApi extends PluginApi {
   ): Promise<boolean> {
     let action: CommitAction['action'] = 'create'
 
-    const branches = await this.api.Branches.all(repoId)
-    if (branches.some(b => b.name === branch)) {
+    const existingBranch = await find(offsetPaginate(opts => this.api.Branches.all(repoId, opts)), b => b.name === branch)
+    if (existingBranch) {
       let actualFile: RepositoryFileExpandedSchema | undefined
       try {
         actualFile = await this.api.RepositoryFiles.show(repoId, filePath, branch)
@@ -152,12 +151,12 @@ export class GitlabApi extends PluginApi {
     return filesUpdated
   }
 
-  public async listFiles(repoId: number, options: AllRepositoryTreesOptions & PaginationRequestOptions<'keyset'> = {}) {
+  public async listFiles(repoId: number, options: AllRepositoryTreesOptions = {}) {
     options.path = options?.path ?? '/'
     options.ref = options?.ref ?? 'main'
     options.recursive = options?.recursive ?? false
     try {
-      const files: RepositoryTreeSchema[] = await this.api.Repositories.allRepositoryTrees(repoId, options)
+      const files = await this.api.Repositories.allRepositoryTrees(repoId, options)
       // if (depth >= 0) {
       //   for (const file of files) {
       //     if (file.type !== 'tree') {
@@ -199,8 +198,11 @@ export class GitlabZoneApi extends GitlabApi {
   public async getOrCreateInfraGroup(): Promise<GroupSchema> {
     const rootId = await getGroupRootId()
     // Get or create projects_root_dir/infra group
-    const searchResult = await this.api.Groups.search(infraGroupName)
-    const existingParentGroup = searchResult.find(group => group.parent_id === rootId && group.name === infraGroupName)
+    const existingParentGroup = await find(offsetPaginate(opts => this.api.Groups.all({
+      search: infraGroupName,
+      orderBy: 'id',
+      ...opts,
+    })), group => group.parent_id === rootId && group.name === infraGroupName)
     return existingParentGroup || await this.api.Groups.create(infraGroupName, infraGroupPath, {
       parentId: rootId,
       projectCreationLevel: 'maintainer',
@@ -216,26 +218,24 @@ export class GitlabZoneApi extends GitlabApi {
     }
     const infraGroup = await this.getOrCreateInfraGroup()
     // Get or create projects_root_dir/infra/zone
-    const infraProjects = await this.api.Groups.allProjects(infraGroup.id, {
+    const project = await find(offsetPaginate(opts => this.api.Groups.allProjects(infraGroup.id, {
       search: zone,
       simple: true,
-      perPage: 100,
-    })
-    const project: ProjectSchema = infraProjects.find(repo => repo.name === zone) ?? await this.createEmptyRepository({
+      ...opts,
+    })), repo => repo.name === zone) ?? await this.createEmptyRepository({
       repoName: zone,
       groupId: infraGroup.id,
       description: 'Repository hosting deployment files for this zone.',
       createFirstCommit: true,
-    },
-    )
+    })
     this.infraProjectsByZoneSlug.set(zone, project)
     return project
   }
 }
 
 export class GitlabProjectApi extends GitlabApi {
   private project: Project | UniqueRepo
-  private gitlabGroup: GroupSchema & { statistics: GroupStatisticsSchema } | undefined
+  private gitlabGroup: GroupSchema | undefined
   private specialRepositories: string[] = [infraAppsRepoName, internalMirrorRepoName]
   private zoneApi: GitlabZoneApi
 
@@ -248,9 +248,12 @@ export class GitlabProjectApi extends GitlabApi {
 
   // Group Project
   private async createProjectGroup(): Promise<GroupSchema> {
-    const searchResult = await this.api.Groups.search(this.project.slug)
     const parentId = await getGroupRootId()
-    const existingGroup = searchResult.find(group => group.parent_id === parentId && group.name === this.project.slug)
+    const existingGroup = await find(offsetPaginate(opts => this.api.Groups.all({
+      search: this.project.slug,
+      orderBy: 'id',
+      ...opts,
+    })), group => group.parent_id === parentId && group.name === this.project.slug)
 
     if (existingGroup) return existingGroup
 
@@ -265,8 +268,7 @@ export class GitlabProjectApi extends GitlabApi {
   public async getProjectGroup(): Promise<GroupSchema | undefined> {
     if (this.gitlabGroup) return this.gitlabGroup
     const parentId = await getGroupRootId()
-    const searchResult = await this.api.Groups.allSubgroups(parentId)
-    this.gitlabGroup = searchResult.find(group => group.name === this.project.slug)
+    this.gitlabGroup = await find(offsetPaginate(opts => this.api.Groups.allSubgroups(parentId, opts)), group => group.name === this.project.slug)
     return this.gitlabGroup
   }
 
@@ -323,21 +325,15 @@ export class GitlabProjectApi extends GitlabApi {
 
   public async getProjectId(projectName: string) {
     const projectGroup = await this.getProjectGroup()
-    if (!projectGroup) {
-      throw new Error('Parent DSO Project group has not been created yet')
-    }
-    const projectsInGroup = await this.api.Groups.allProjects(projectGroup.id, {
+    if (!projectGroup) throw new Error(`Gitlab inaccessible, impossible de trouver le groupe ${this.project.slug}`)
+
+    const project = await find(offsetPaginate(opts => this.api.Groups.allProjects(projectGroup.id, {
       search: projectName,
       simple: true,
-      perPage: 100,
-    })
-    const project = projectsInGroup.find(p => p.path === projectName)
+      ...opts,
+    })), repo => repo.name === projectName)
 
-    if (!project) {
-      const pathProjectName = `${config().projectsRootDir}/${this.project.slug}/${projectName}`
-      throw new Error(`Gitlab project "${pathProjectName}" not found`)
-    }
-    return project.id
+    return project?.id
   }
 
   public async getProjectById(projectId: number) {
@@ -351,8 +347,7 @@ export class GitlabProjectApi extends GitlabApi {
   public async getProjectToken(tokenName: string) {
     const group = await this.getProjectGroup()
     if (!group) throw new Error('Unable to retrieve gitlab project group')
-    const groupTokens = await this.api.GroupAccessTokens.all(group.id)
-    return groupTokens.find(token => token.name === tokenName)
+    return find(offsetPaginate(opts => this.api.GroupAccessTokens.all(group.id, opts)), token => token.name === tokenName)
   }
 
   public async createProjectToken(tokenName: string, scopes: AccessTokenScopes[]) {
@@ -375,8 +370,7 @@ export class GitlabProjectApi extends GitlabApi {
     const gitlabRepositories = await this.listRepositories()
     const mirrorRepo = gitlabRepositories.find(repo => repo.name === internalMirrorRepoName)
     if (!mirrorRepo) throw new Error('Don\'t know how mirror repo could not exist')
-    const allTriggerTokens = await this.api.PipelineTriggerTokens.all(mirrorRepo.id)
-    const currentTriggerToken = allTriggerTokens.find(token => token.description === tokenDescription)
+    const currentTriggerToken = await find(offsetPaginate(opts => this.api.PipelineTriggerTokens.all(mirrorRepo.id, opts)), token => token.description === tokenDescription)
 
     const tokenVaultSecret = await vaultApi.read('GITLAB', { throwIfNoEntry: false })
 
@@ -398,7 +392,7 @@ export class GitlabProjectApi extends GitlabApi {
 
   public async listRepositories() {
     const group = await this.getOrCreateProjectGroup()
-    const projects = await this.api.Groups.allProjects(group.id, { simple: false }) // to refactor with https://github.com/jdalrymple/gitbeaker/pull/3624
+    const projects = await getAll(offsetPaginate(opts => this.api.Groups.allProjects(group.id, { simple: false, ...opts }))) // to refactor with https://github.com/jdalrymple/gitbeaker/pull/3624
     return Promise.all(projects.map(async (project) => {
       if (this.specialRepositories.includes(project.name) && (!project.topics || !project.topics.includes(pluginManagedTopic))) {
         return this.api.Projects.edit(project.id, { topics: project.topics ? [...project.topics, pluginManagedTopic] : [pluginManagedTopic] })
@@ -432,7 +426,7 @@ export class GitlabProjectApi extends GitlabApi {
   // Group members
   public async getGroupMembers() {
     const group = await this.getOrCreateProjectGroup()
-    return this.api.GroupMembers.all(group.id)
+    return getAll(offsetPaginate(opts => this.api.GroupMembers.all(group.id, opts)))
   }
 
   public async addGroupMember(userId: number, accessLevel: AccessLevelAllowed = AccessLevel.DEVELOPER): Promise<MemberSchema> {
@@ -448,7 +442,7 @@ export class GitlabProjectApi extends GitlabApi {
   // CI Variables
   public async getGitlabGroupVariables(): Promise<VariableSchema[]> {
     const group = await this.getOrCreateProjectGroup()
-    return await this.api.GroupVariables.all(group.id)
+    return await getAll(offsetPaginate(opts => this.api.GroupVariables.all(group.id, opts)))
   }
 
   public async setGitlabGroupVariable(listVars: VariableSchema[], toSetVariable: VariableSchema): Promise<setVariableResult> {
@@ -491,7 +485,7 @@ export class GitlabProjectApi extends GitlabApi {
   }
 
   public async getGitlabRepoVariables(repoId: number): Promise<VariableSchema[]> {
-    return await this.api.ProjectVariables.all(repoId)
+    return await getAll(offsetPaginate(opts => this.api.ProjectVariables.all(repoId, opts)))
   }
 
   public async setGitlabRepoVariable(repoId: number, listVars: VariableSchema[], toSetVariable: ProjectVariableSchema): Promise<setVariableResult | 'repository not found'> {

plugins/gitlab/src/user.ts

@@ -1,28 +1,19 @@
 import type { UserObject } from '@cpn-console/hooks'
 import type { CreateUserOptions, SimpleUserSchema } from '@gitbeaker/rest'
-import { getApi } from './utils.js'
+import { getApi, find, offsetPaginate } from './utils.js'
 
 export const createUsername = (email: string) => email.replace('@', '.')
 
 export async function getUser(user: { email: string, username: string, id: string }): Promise<SimpleUserSchema | undefined> {
   const api = getApi()
 
-  let gitlabUser: SimpleUserSchema | undefined
-
-  // test finding by extern_uid by searching with email
-  const usersByEmail = await api.Users.all({ search: user.email })
-  gitlabUser = usersByEmail.find(gitlabUser => gitlabUser?.externUid === user.id)
-  if (gitlabUser) return gitlabUser
-
-  // if not found, test finding by extern_uid by searching with username
-  const usersByUsername = await api.Users.all({ username: user.username })
-  gitlabUser = usersByUsername.find(gitlabUser => gitlabUser?.externUid === user.id)
-  if (gitlabUser) return gitlabUser
-
-  // if not found, test finding by email or username
-  const allUsers = [...usersByEmail, ...usersByUsername]
-  return allUsers.find(gitlabUser => gitlabUser.email === user.email)
-    || allUsers.find(gitlabUser => gitlabUser.username === user.username)
+  return find(
+    offsetPaginate(opts => api.Users.all({ ...opts, asAdmin: true })),
+    gitlabUser =>
+      gitlabUser?.externUid === user.id
+      || gitlabUser.email === user.email
+      || gitlabUser.username === user.username,
+  )
 }
 
 export async function upsertUser(user: UserObject): Promise<SimpleUserSchema> {
@@ -57,7 +48,11 @@ export async function upsertUser(user: UserObject): Promise<SimpleUserSchema> {
         console.log(`Gitlab plugin: Updating user: ${user.email}`)
         console.log(incorrectProps)
       }
-      await api.Users.edit(existingUser.id, userDefinitionBase)
+      try {
+        await api.Users.edit(existingUser.id, userDefinitionBase)
+      } catch (err) {
+        console.error(`Gitlab plugin: Failed to update user: ${user.email} for ${err}`)
+      }
     }
     return existingUser
   }

plugins/gitlab/src/utils.ts

@@ -1,5 +1,5 @@
 import { Gitlab } from '@gitbeaker/rest'
-import type { Gitlab as IGitlab } from '@gitbeaker/core'
+import type { Gitlab as IGitlab, BaseRequestOptions, PaginationRequestOptions, OffsetPagination } from '@gitbeaker/core'
 import { GitbeakerRequestError } from '@gitbeaker/requester-utils'
 import config from './config.js'
 
@@ -13,8 +13,12 @@ export async function getGroupRootId(throwIfNotFound?: boolean): Promise<number
   const gitlabApi = getApi()
   const projectRootDir = config().projectsRootDir
   if (groupRootId) return groupRootId
-  const groupRootSearch = await gitlabApi.Groups.search(projectRootDir)
-  const searchId = (groupRootSearch.find(grp => grp.full_path === projectRootDir))?.id
+  const groupRoot = await find(offsetPaginate(opts => gitlabApi.Groups.all({
+    search: projectRootDir,
+    orderBy: 'id',
+    ...opts,
+  })), grp => grp.full_path === projectRootDir)
+  const searchId = groupRoot?.id
   if (typeof searchId === 'undefined') {
     if (throwIfNotFound) {
       throw new Error(`Gitlab inaccessible, impossible de trouver le groupe ${projectRootDir}`)
@@ -35,19 +39,23 @@ async function createGroupRoot(): Promise<number> {
     throw new Error('No projectRootDir available')
   }
 
-  let parentGroup = (await gitlabApi.Groups.search(rootGroupPath))
-    .find(grp => grp.full_path === rootGroupPath)
-    ?? await gitlabApi.Groups.create(rootGroupPath, rootGroupPath)
+  let parentGroup = await find(offsetPaginate(opts => gitlabApi.Groups.all({
+    search: rootGroupPath,
+    orderBy: 'id',
+    ...opts,
+  })), grp => grp.full_path === rootGroupPath) ?? await gitlabApi.Groups.create(rootGroupPath, rootGroupPath)
 
   if (parentGroup.full_path === projectRootDir) {
     return parentGroup.id
   }
 
   for (const path of projectRootDirArray) {
     const futureFullPath = `${parentGroup.full_path}/${path}`
-    parentGroup = (await gitlabApi.Groups.search(futureFullPath))
-      .find(grp => grp.full_path === futureFullPath)
-      ?? await gitlabApi.Groups.create(path, path, { parentId: parentGroup.id, visibility: 'internal' })
+    parentGroup = await find(offsetPaginate(opts => gitlabApi.Groups.all({
+      search: futureFullPath,
+      orderBy: 'id',
+      ...opts,
+    })), grp => grp.full_path === futureFullPath) ?? await gitlabApi.Groups.create(path, path, { parentId: parentGroup.id, visibility: 'internal' })
 
     if (parentGroup.full_path === projectRootDir) {
       return parentGroup.id
@@ -57,17 +65,11 @@ async function createGroupRoot(): Promise<number> {
 }
 
 export async function getOrCreateGroupRoot(): Promise<number> {
-  let rootId = await getGroupRootId(false)
-  if (typeof rootId === 'undefined') {
-    rootId = await createGroupRoot()
-  }
-  return rootId
+  return await getGroupRootId(false) ?? createGroupRoot()
 }
 
 export function getApi(): IGitlab {
-  if (!api) {
-    api = new Gitlab({ token: config().token, host: config().internalUrl })
-  }
+  api ??= new Gitlab({ token: config().token, host: config().internalUrl })
   return api
 }
 
@@ -89,3 +91,38 @@ export function cleanGitlabError<T>(error: T): T {
   }
   return error
 }
+
+export async function* offsetPaginate<T>(
+  request: (options: PaginationRequestOptions<'offset'> & BaseRequestOptions<true>) => Promise<{ data: T[], paginationInfo: OffsetPagination }>,
+): AsyncGenerator<T> {
+  let page: number | null = 1
+  while (page !== null) {
+    const { data, paginationInfo } = await request({ page, showExpanded: true, pagination: 'offset' })
+    for (const item of data) {
+      yield item
+    }
+    page = paginationInfo.next
+  }
+}
+
+export async function getAll<T>(
+  iterable: AsyncIterable<T>,
+): Promise<T[]> {
+  const items: T[] = []
+  for await (const item of iterable) {
+    items.push(item)
+  }
+  return items
+}
+
+export async function find<T>(
+  iterable: AsyncIterable<T>,
+  predicate: (item: T) => boolean,
+): Promise<T | undefined> {
+  for await (const item of iterable) {
+    if (predicate(item)) {
+      return item
+    }
+  }
+  return undefined
+}

plugins/sonarqube/src/functions.ts

@@ -119,13 +119,13 @@ export const upsertProject: StepCall<Project> = async (payload) => {
 
       // Remove excess repositories
       ...sonarRepositories
-        .filter(sonarRepository => !project.repositories.find(repo => repo.internalRepoName === sonarRepository.repository))
+        .filter(sonarRepository => !project.repositories.some(repo => repo.internalRepoName === sonarRepository.repository))
         .map(sonarRepository => deleteDsoRepository(sonarRepository.key)),
 
       // Create or configure needed repos
       ...project.repositories.map(async (repository) => {
         const projectKey = generateProjectKey(projectSlug, repository.internalRepoName)
-        if (!sonarRepositories.find(sonarRepository => sonarRepository.repository === repository.internalRepoName)) {
+        if (!sonarRepositories.some(sonarRepository => sonarRepository.repository === repository.internalRepoName)) {
           await createDsoRepository(projectSlug, repository.internalRepoName)
         }
         await ensureRepositoryConfiguration(projectKey, username, keycloakGroupPath)
@@ -166,6 +166,7 @@ export const setVariables: StepCall<Project> = async (payload) => {
       ...project.repositories.map(async (repo) => {
         const projectKey = generateProjectKey(projectSlug, repo.internalRepoName)
         const repoId = await payload.apis.gitlab.getProjectId(repo.internalRepoName)
+        if (!repoId) return
         const listVars = await gitlabApi.getGitlabRepoVariables(repoId)
         return [
           await gitlabApi.setGitlabRepoVariable(repoId, listVars, {
@@ -193,9 +194,9 @@ export const setVariables: StepCall<Project> = async (payload) => {
             environment_scope: '*',
           }),
         ]
-      }).flat(),
+      }),
       // Sonar vars saving in CI (group)
-      await gitlabApi.setGitlabGroupVariable(listGroupVars, {
+      gitlabApi.setGitlabGroupVariable(listGroupVars, {
         key: 'SONAR_TOKEN',
         masked: true,
         protected: false,