diff --git a/.changeset/move-misc-to-sdk.md b/.changeset/move-misc-to-sdk.md new file mode 100644 index 00000000..2dcdcdb4 --- /dev/null +++ b/.changeset/move-misc-to-sdk.md @@ -0,0 +1,12 @@ +--- +'@transcend-io/sdk': minor +'@transcend-io/cli': patch +--- + +feat(sdk): move assessment templates, ensureAllDataSubjectsExist, and syncCodePackages to SDK + +- Move `fetchAllAssessmentTemplates`, `AssessmentTemplate` interface, and `ASSESSMENT_TEMPLATES` GQL to `sdk/src/assessments/` +- Move `ensureAllDataSubjectsExist` to `sdk/src/data-inventory/` with new `EnsureDataSubjectsInput` interface (replaces CLI's `TranscendInput` dependency) +- Move `createCodePackage`, `updateCodePackages`, `syncCodePackages` to `sdk/src/code-intelligence/` with `CodePackageInput` interface +- Delete dead-code `syncCookies.ts` from CLI (already duplicated in SDK, all callers already import from SDK) +- All moved functions follow the `(client, options)` signature convention with optional `logger` diff --git a/packages/cli/src/commands/inventory/scan-packages/impl.ts b/packages/cli/src/commands/inventory/scan-packages/impl.ts index 253f7163..e0373ddd 100644 --- a/packages/cli/src/commands/inventory/scan-packages/impl.ts +++ b/packages/cli/src/commands/inventory/scan-packages/impl.ts @@ -1,13 +1,12 @@ import { execSync } from 'child_process'; -import { buildTranscendGraphQLClient } from '@transcend-io/sdk'; +import { buildTranscendGraphQLClient, syncCodePackages } from '@transcend-io/sdk'; import colors from 'colors'; import { ADMIN_DASH } from '../../../constants.js'; import type { LocalContext } from '../../../context.js'; import { doneInputValidation } from '../../../lib/cli/done-input-validation.js'; import { findCodePackagesInFolder } from '../../../lib/code-scanning/index.js'; -import { syncCodePackages } from '../../../lib/graphql/index.js'; import { logger } from '../../../logger.js'; const REPO_ERROR = @@ -60,7 +59,7 @@ export async function scanPackages( }); // Report scan to Transcend - await syncCodePackages(client, results); + await syncCodePackages(client, results, { logger }); const newUrl = new URL(ADMIN_DASH); newUrl.pathname = '/code-scanning/code-packages'; diff --git a/packages/cli/src/lib/graphql/ensureAllDataSubjectsExist.ts b/packages/cli/src/lib/graphql/ensureAllDataSubjectsExist.ts deleted file mode 100644 index 240d9789..00000000 --- a/packages/cli/src/lib/graphql/ensureAllDataSubjectsExist.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { createDataSubject, fetchAllDataSubjects, type DataSubject } from '@transcend-io/sdk'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { keyBy, flatten, uniq, difference } from 'lodash-es'; - -import { TranscendInput } from '../../codecs.js'; -import { logger } from '../../logger.js'; - -/** - * Fetch all of the data subjects in the organization - * - * @param input - Input to fetch - * @param client - GraphQL client - * @param fetchAll - When true, always fetch all subjects - * @returns The list of data subjects - */ -export async function ensureAllDataSubjectsExist( - { - 'data-silos': dataSilos = [], - 'data-subjects': dataSubjects = [], - 'processing-activities': processingActivities = [], - enrichers = [], - }: TranscendInput, - client: GraphQLClient, - fetchAll = false, -): Promise<{ [type in string]: DataSubject }> { - const expectedDataSubjects = uniq([ - ...flatten(dataSilos.map((silo) => silo['data-subjects'] || []) || []), - ...flatten(processingActivities.map(({ dataSubjectTypes }) => dataSubjectTypes ?? []) ?? []), - ...flatten(enrichers.map((enricher) => enricher['data-subjects'] || []) || []), - ...dataSubjects.map((subject) => subject.type), - ]); - if (expectedDataSubjects.length === 0 && !fetchAll) { - return {}; - } - - const internalSubjects = await fetchAllDataSubjects(client, { logger }); - const dataSubjectByName = keyBy(internalSubjects, 'type'); - - const missingDataSubjects = difference( - expectedDataSubjects, - internalSubjects.map(({ type }) => type), - ); - - if (missingDataSubjects.length > 0) { - logger.info(colors.magenta(`Creating ${missingDataSubjects.length} new data subjects...`)); - for (const dataSubjectType of missingDataSubjects) { - logger.info(colors.magenta(`Creating data subject ${dataSubjectType}...`)); - const created = await createDataSubject(client, { input: dataSubjectType, logger }); - logger.info(colors.green(`Created data subject ${dataSubjectType}!`)); - dataSubjectByName[dataSubjectType] = created; - } - } - - return dataSubjectByName; -} diff --git a/packages/cli/src/lib/graphql/gqls/index.ts b/packages/cli/src/lib/graphql/gqls/index.ts index 91d99c4f..48f75a93 100644 --- a/packages/cli/src/lib/graphql/gqls/index.ts +++ b/packages/cli/src/lib/graphql/gqls/index.ts @@ -1,3 +1,2 @@ export * from './entry.js'; export * from './request.js'; -export * from './assessmentTemplate.js'; diff --git a/packages/cli/src/lib/graphql/index.ts b/packages/cli/src/lib/graphql/index.ts index 6baff12c..44569196 100644 --- a/packages/cli/src/lib/graphql/index.ts +++ b/packages/cli/src/lib/graphql/index.ts @@ -1,10 +1,6 @@ -export * from './ensureAllDataSubjectsExist.js'; -export * from './fetchAllAssessmentTemplates.js'; export * from './fetchAllRequests.js'; export * from './fetchRequestDataSiloActiveCount.js'; export * from './gqls/index.js'; export * from './pullTranscendConfiguration.js'; -export * from './syncCodePackages.js'; export * from './syncConfigurationToTranscend.js'; -export * from './syncCookies.js'; export * from './syncDataSilos.js'; diff --git a/packages/cli/src/lib/graphql/pullTranscendConfiguration.ts b/packages/cli/src/lib/graphql/pullTranscendConfiguration.ts index 55a72994..2749ca1d 100644 --- a/packages/cli/src/lib/graphql/pullTranscendConfiguration.ts +++ b/packages/cli/src/lib/graphql/pullTranscendConfiguration.ts @@ -13,6 +13,7 @@ import { fetchAllAgentFunctions, fetchAllAgents, fetchAllAssessments, + fetchAllAssessmentTemplates, fetchAllAttributes, fetchAllBusinessEntities, fetchAllCookies, @@ -91,7 +92,6 @@ import { } from '../../codecs.js'; import { TranscendPullResource } from '../../enums.js'; import { logger } from '../../logger.js'; -import { fetchAllAssessmentTemplates } from './fetchAllAssessmentTemplates.js'; export const DEFAULT_TRANSCEND_PULL_RESOURCES = [ TranscendPullResource.DataSilos, @@ -334,7 +334,7 @@ export async function pullTranscendConfiguration( : [], // Fetch assessmentTemplates resources.includes(TranscendPullResource.AssessmentTemplates) - ? fetchAllAssessmentTemplates(client) + ? fetchAllAssessmentTemplates(client, { logger }) : [], // Fetch purpose and preferences resources.includes(TranscendPullResource.Purposes) diff --git a/packages/cli/src/lib/graphql/syncCodePackages.ts b/packages/cli/src/lib/graphql/syncCodePackages.ts deleted file mode 100644 index c3b9d8df..00000000 --- a/packages/cli/src/lib/graphql/syncCodePackages.ts +++ /dev/null @@ -1,271 +0,0 @@ -import { CodePackageType } from '@transcend-io/privacy-types'; -import { - CREATE_CODE_PACKAGE, - UPDATE_CODE_PACKAGES, - fetchAllCodePackages, - type CodePackage, - makeGraphQLRequest, - syncRepositories, - syncSoftwareDevelopmentKits, -} from '@transcend-io/sdk'; -import { map, mapSeries } from '@transcend-io/utils'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk, uniq, keyBy, uniqBy } from 'lodash-es'; - -import { CodePackageInput, RepositoryInput } from '../../codecs.js'; -import { logger } from '../../logger.js'; - -const CHUNK_SIZE = 100; - -const LOOKUP_SPLIT_KEY = '%%%%'; - -/** - * Create a new code package - * - * @param client - GraphQL client - * @param input - Code package input - * @returns Code package ID - */ -export async function createCodePackage( - client: GraphQLClient, - input: { - /** Name of package */ - name: string; - /** Description of package */ - description?: string; - /** Type of package */ - type: CodePackageType; - /** Relative path to package */ - relativePath: string; - /** Repository ID */ - repositoryId?: string; - /** Name of repository */ - repositoryName?: string; - /** IDs of SDKs */ - softwareDevelopmentKitIds?: string[]; - /** IDs of owners */ - ownerIds?: string[]; - /** Emails of owners */ - ownerEmails?: string[]; - /** IDs of teams */ - teamIds?: string[]; - /** Names of teams */ - teamNames?: string[]; - }, -): Promise { - const { - createCodePackage: { codePackage }, - } = await makeGraphQLRequest<{ - /** createCodePackage mutation */ - createCodePackage: { - /** Code package */ - codePackage: CodePackage; - }; - }>(client, CREATE_CODE_PACKAGE, { - variables: { input }, - logger, - }); - logger.info(colors.green(`Successfully created code package "${input.name}"!`)); - return codePackage; -} - -/** - * Update an existing code package - * - * @param client - GraphQL client - * @param inputs - Code package input - * @returns Code packages that were updated - */ -export async function updateCodePackages( - client: GraphQLClient, - inputs: { - /** ID of code package */ - id: string; - /** Name of package */ - name: string; - /** Description of package */ - description?: string; - /** Type of package */ - type: CodePackageType; - /** Relative path to package */ - relativePath: string; - /** Repository ID */ - repositoryId?: string; - /** Name of repository */ - repositoryName?: string; - /** IDs of SDKs */ - softwareDevelopmentKitIds?: string[]; - /** IDs of owners */ - ownerIds?: string[]; - /** Emails of owners */ - ownerEmails?: string[]; - /** IDs of teams */ - teamIds?: string[]; - /** Names of teams */ - teamNames?: string[]; - }[], -): Promise { - const { - updateCodePackages: { codePackages }, - } = await makeGraphQLRequest<{ - /** updateCodePackages mutation */ - updateCodePackages: { - /** Code packages */ - codePackages: CodePackage[]; - }; - }>(client, UPDATE_CODE_PACKAGES, { - variables: { - input: { - codePackages: inputs, - }, - }, - logger, - }); - logger.info(colors.green(`Successfully updated ${inputs.length} code packages!`)); - return codePackages; -} - -/** - * Uploads silo discovery results for Transcend to classify - * - * @param client - GraphQL Client - * @param codePackages - Packages to upload - * @param concurrency - How many concurrent requests to make - * @returns True if successful, false if any updates failed, or an error occurs - */ -export async function syncCodePackages( - client: GraphQLClient, - codePackages: CodePackageInput[], - concurrency = 20, -): Promise { - let encounteredError = false; - const [existingCodePackages, { softwareDevelopmentKits: existingSoftwareDevelopmentKits }] = - await Promise.all([ - // fetch all code packages - fetchAllCodePackages(client, { logger }), - // make sure all SDKs exist - syncSoftwareDevelopmentKits( - client, - uniqBy( - codePackages - .map(({ type, softwareDevelopmentKits = [] }) => - softwareDevelopmentKits.map(({ name }) => ({ - name, - codePackageType: type, - })), - ) - .flat(), - ({ name, codePackageType }) => `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, - ), - { logger, concurrency }, - ), - // make sure all Repositories exist - syncRepositories( - client, - uniqBy(codePackages, 'repositoryName').map( - ({ repositoryName }) => - ({ - name: repositoryName, - url: `https://github.com/${repositoryName}`, - }) as RepositoryInput, - ), - { logger }, - ), - ]); - - const softwareDevelopmentKitLookup = keyBy( - existingSoftwareDevelopmentKits, - ({ name, codePackageType }) => `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, - ); - const codePackagesLookup = keyBy( - existingCodePackages, - ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}`, - ); - - // Determine which codePackages are new vs existing - const mapCodePackagesToExisting = codePackages.map((codePackageInput) => [ - codePackageInput, - codePackagesLookup[`${codePackageInput.name}${LOOKUP_SPLIT_KEY}${codePackageInput.type}`]?.id, - ]); - - // Create the new codePackages - const newCodePackages = mapCodePackagesToExisting - .filter(([, existing]) => !existing) - .map(([codePackageInput]) => codePackageInput as CodePackageInput); - try { - logger.info(colors.magenta(`Creating "${newCodePackages.length}" new code packages...`)); - await map( - newCodePackages, - async ({ softwareDevelopmentKits, ...codePackage }) => { - await createCodePackage(client, { - ...codePackage, - ...(softwareDevelopmentKits - ? { - softwareDevelopmentKitIds: uniq( - softwareDevelopmentKits.map(({ name }) => { - const sdk = - softwareDevelopmentKitLookup[`${name}${LOOKUP_SPLIT_KEY}${codePackage.type}`]; - if (!sdk) { - throw new Error(`Failed to find SDK with name: "${name}"`); - } - return sdk.id; - }), - ), - } - : {}), - }); - }, - { - concurrency, - }, - ); - logger.info(colors.green(`Successfully synced ${newCodePackages.length} code packages!`)); - } catch (err) { - encounteredError = true; - logger.error(colors.red(`Failed to create code packages! - ${err.message}`)); - } - - // Update existing codePackages - const existingCodePackageInputs = mapCodePackagesToExisting.filter( - (x): x is [CodePackageInput, string] => !!x[1], - ); - logger.info(colors.magenta(`Updating "${existingCodePackageInputs.length}" code packages...`)); - const chunks = chunk(existingCodePackageInputs, CHUNK_SIZE); - - await mapSeries(chunks, async (chunk) => { - try { - await updateCodePackages( - client, - chunk.map( - // eslint-disable-next-line @typescript-eslint/no-unused-vars - ([{ softwareDevelopmentKits, repositoryName, ...input }, id]) => ({ - ...input, - ...(softwareDevelopmentKits - ? { - softwareDevelopmentKitIds: uniq( - softwareDevelopmentKits.map(({ name }) => { - const sdk = - softwareDevelopmentKitLookup[`${name}${LOOKUP_SPLIT_KEY}${input.type}`]; - if (!sdk) { - throw new Error(`Failed to find SDK with name: "${name}"`); - } - return sdk.id; - }), - ), - } - : {}), - id, - }), - ), - ); - logger.info(colors.green(`Successfully updated "${chunk.length}" code packages!`)); - } catch (err) { - encounteredError = true; - logger.error(colors.red(`Failed to update code packages! - ${err.message}`)); - } - }); - - logger.info(colors.green(`Synced "${codePackages.length}" code packages!`)); - return !encounteredError; -} diff --git a/packages/cli/src/lib/graphql/syncConfigurationToTranscend.ts b/packages/cli/src/lib/graphql/syncConfigurationToTranscend.ts index 1eb2bc49..0f38772e 100644 --- a/packages/cli/src/lib/graphql/syncConfigurationToTranscend.ts +++ b/packages/cli/src/lib/graphql/syncConfigurationToTranscend.ts @@ -1,4 +1,5 @@ import { + ensureAllDataSubjectsExist, fetchAllActions, fetchAllAttributes, fetchAllDataSubjects, @@ -41,7 +42,6 @@ import { GraphQLClient } from 'graphql-request'; /* eslint-disable max-lines */ import { TranscendInput } from '../../codecs.js'; import { logger } from '../../logger.js'; -import { ensureAllDataSubjectsExist } from './ensureAllDataSubjectsExist.js'; import { syncDataSilos } from './syncDataSilos.js'; const CONCURRENCY = 10; @@ -120,7 +120,19 @@ export async function syncConfigurationToTranscend( : ({} as { [k in string]: Identifier }), // Grab all data subjects in the organization dataSilos || dataSubjects || enrichers || processingActivities - ? ensureAllDataSubjectsExist(input, client) + ? ensureAllDataSubjectsExist(client, { + input: { + dataSiloDataSubjects: (dataSilos ?? []).map((silo) => silo['data-subjects'] ?? []), + processingActivityDataSubjects: (processingActivities ?? []).map( + ({ dataSubjectTypes }) => dataSubjectTypes ?? [], + ), + enricherDataSubjects: (enrichers ?? []).map( + (enricher) => enricher['data-subjects'] ?? [], + ), + dataSubjectTypes: (dataSubjects ?? []).map((subject) => subject.type), + }, + logger, + }) : {}, // Grab API keys dataSilos && diff --git a/packages/cli/src/lib/graphql/syncCookies.ts b/packages/cli/src/lib/graphql/syncCookies.ts deleted file mode 100644 index 29cd5ee8..00000000 --- a/packages/cli/src/lib/graphql/syncCookies.ts +++ /dev/null @@ -1,99 +0,0 @@ -// import { keyBy } from 'lodash-es'; -import { - fetchConsentManagerId, - makeGraphQLRequest, - UPDATE_OR_CREATE_COOKIES, -} from '@transcend-io/sdk'; -import { mapSeries } from '@transcend-io/utils'; -import colors from 'colors'; -import { GraphQLClient } from 'graphql-request'; -import { chunk } from 'lodash-es'; - -import { CookieInput } from '../../codecs.js'; -import { logger } from '../../logger.js'; - -const MAX_PAGE_SIZE = 100; - -/** - * Update or create cookies that already existed - * - * @param client - GraphQL client - * @param cookieInputs - List of cookie input - */ -export async function updateOrCreateCookies( - client: GraphQLClient, - cookieInputs: CookieInput[], -): Promise { - const airgapBundleId = await fetchConsentManagerId(client, { logger }); - - // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // const purposes = await fetchAllPurposes(client); - // const purposeNameToId = keyBy(purposes, 'name'); - - await mapSeries(chunk(cookieInputs, MAX_PAGE_SIZE), async (page) => { - await makeGraphQLRequest(client, UPDATE_OR_CREATE_COOKIES, { - variables: { - airgapBundleId, - cookies: page.map((cookie) => ({ - name: cookie.name, - trackingPurposes: - cookie.trackingPurposes && cookie.trackingPurposes.length > 0 - ? cookie.trackingPurposes - : undefined, - // TODO: https://transcend.height.app/T-19841 - add with custom purposes - // purposeIds: cookie.trackingPurposes - // ? cookie.trackingPurposes - // .filter((purpose) => purpose !== 'Unknown') - // .map((purpose) => purposeNameToId[purpose].id) - // : undefined, - description: cookie.description, - service: cookie.service, - status: cookie.status, - attributes: cookie.attributes, - isRegex: cookie.isRegex, - // TODO: https://transcend.height.app/T-23718 - // owners, - // teams, - })), - }, - logger, - }); - }); -} - -/** - * Sync the set of cookies from the YML interface into the product - * - * @param client - GraphQL client - * @param cookies - Cookies to sync - * @returns True upon success, false upon failure - */ -export async function syncCookies(client: GraphQLClient, cookies: CookieInput[]): Promise { - let encounteredError = false; - logger.info(colors.magenta(`Syncing "${cookies.length}" cookies...`)); - - // Ensure no duplicates are being uploaded - const notUnique = cookies.filter( - (cookie) => - cookies.filter((cook) => cookie.name === cook.name && cookie.isRegex === cook.isRegex) - .length > 1, - ); - if (notUnique.length > 0) { - throw new Error( - `Failed to upload cookies as there were non-unique entries found: ${notUnique - .map(({ name }) => name) - .join(',')}`, - ); - } - - try { - logger.info(colors.magenta(`Upserting "${cookies.length}" new cookies...`)); - await updateOrCreateCookies(client, cookies); - logger.info(colors.green(`Successfully synced ${cookies.length} cookies!`)); - } catch (err) { - encounteredError = true; - logger.error(colors.red(`Failed to create cookies! - ${err.message}`)); - } - - return !encounteredError; -} diff --git a/packages/cli/src/lib/graphql/fetchAllAssessmentTemplates.ts b/packages/sdk/src/assessments/fetchAllAssessmentTemplates.ts similarity index 78% rename from packages/cli/src/lib/graphql/fetchAllAssessmentTemplates.ts rename to packages/sdk/src/assessments/fetchAllAssessmentTemplates.ts index 7762afef..6edf6620 100644 --- a/packages/cli/src/lib/graphql/fetchAllAssessmentTemplates.ts +++ b/packages/sdk/src/assessments/fetchAllAssessmentTemplates.ts @@ -2,16 +2,12 @@ import { AssessmentFormTemplateSource, AssessmentFormTemplateStatus, } from '@transcend-io/privacy-types'; -import { - makeGraphQLRequest, - type AssessmentSection, - type RetentionSchedule, - type UserPreview, -} from '@transcend-io/sdk'; +import type { Logger } from '@transcend-io/utils'; import { GraphQLClient } from 'graphql-request'; -import { logger } from '../../logger.js'; -import { ASSESSMENT_TEMPLATES } from './gqls/index.js'; +import { makeGraphQLRequest, NOOP_LOGGER } from '../api/makeGraphQLRequest.js'; +import type { AssessmentSection, RetentionSchedule, UserPreview } from './fetchAllAssessments.js'; +import { ASSESSMENT_TEMPLATES } from './gqls/assessmentTemplate.js'; /** * Represents an assessment template with various properties and metadata. @@ -29,7 +25,7 @@ export interface AssessmentTemplate { description: string; /** The current status of the assessment template */ status: AssessmentFormTemplateStatus; - /** The source fo the form template */ + /** The source of the form template */ source: AssessmentFormTemplateSource; /** ID of parent template */ parentId: string; @@ -53,11 +49,19 @@ const PAGE_SIZE = 20; * Fetch all assessment templates in the organization * * @param client - GraphQL client + * @param options - Options * @returns All assessment templates in the organization */ export async function fetchAllAssessmentTemplates( client: GraphQLClient, + options: { + /** Filter criteria */ + filterBy?: Record; + /** Logger instance */ + logger?: Logger; + } = {}, ): Promise { + const { filterBy, logger = NOOP_LOGGER } = options; const assessmentTemplates: AssessmentTemplate[] = []; let offset = 0; @@ -72,7 +76,7 @@ export async function fetchAllAssessmentTemplates( nodes: AssessmentTemplate[]; }; }>(client, ASSESSMENT_TEMPLATES, { - variables: { first: PAGE_SIZE, offset }, + variables: { first: PAGE_SIZE, offset, filterBy }, logger, }); assessmentTemplates.push(...nodes); diff --git a/packages/cli/src/lib/graphql/gqls/assessmentTemplate.ts b/packages/sdk/src/assessments/gqls/assessmentTemplate.ts similarity index 95% rename from packages/cli/src/lib/graphql/gqls/assessmentTemplate.ts rename to packages/sdk/src/assessments/gqls/assessmentTemplate.ts index d44fd528..a672dfdd 100644 --- a/packages/cli/src/lib/graphql/gqls/assessmentTemplate.ts +++ b/packages/sdk/src/assessments/gqls/assessmentTemplate.ts @@ -1,6 +1,7 @@ -import { ASSESSMENT_SECTION_FIELDS } from '@transcend-io/sdk'; import { gql } from 'graphql-request'; +import { ASSESSMENT_SECTION_FIELDS } from './assessment.js'; + // TODO: https://transcend.height.app/T-27909 - enable optimizations // isExportCsv: true // useMaster: false diff --git a/packages/sdk/src/assessments/index.ts b/packages/sdk/src/assessments/index.ts index 6a1504e6..5760f5cb 100644 --- a/packages/sdk/src/assessments/index.ts +++ b/packages/sdk/src/assessments/index.ts @@ -1,7 +1,9 @@ export * from './fetchAllActionItemCollections.js'; export * from './fetchAllActionItems.js'; export * from './fetchAllAssessments.js'; +export * from './fetchAllAssessmentTemplates.js'; export * from './gqls/assessment.js'; +export * from './gqls/assessmentTemplate.js'; export * from './parseAssessmentDisplayLogic.js'; export * from './parseAssessmentRiskLogic.js'; export * from './syncActionItemCollections.js'; diff --git a/packages/sdk/src/code-intelligence/index.ts b/packages/sdk/src/code-intelligence/index.ts index 063516df..fabb3364 100644 --- a/packages/sdk/src/code-intelligence/index.ts +++ b/packages/sdk/src/code-intelligence/index.ts @@ -2,5 +2,6 @@ export * from './fetchAllCodePackages.js'; export * from './fetchAllRepositories.js'; export * from './fetchAllSoftwareDevelopmentKits.js'; export * from './gqls/codePackage.js'; +export * from './syncCodePackages.js'; export * from './syncRepositories.js'; export * from './syncSoftwareDevelopmentKits.js'; diff --git a/packages/sdk/src/code-intelligence/syncCodePackages.ts b/packages/sdk/src/code-intelligence/syncCodePackages.ts new file mode 100644 index 00000000..9db53373 --- /dev/null +++ b/packages/sdk/src/code-intelligence/syncCodePackages.ts @@ -0,0 +1,305 @@ +import { CodePackageType } from '@transcend-io/privacy-types'; +import { map, mapSeries, type Logger } from '@transcend-io/utils'; +import { GraphQLClient } from 'graphql-request'; +import { chunk, uniq, keyBy, uniqBy } from 'lodash-es'; + +import { makeGraphQLRequest, NOOP_LOGGER } from '../api/makeGraphQLRequest.js'; +import { fetchAllCodePackages, type CodePackage } from './fetchAllCodePackages.js'; +import { CREATE_CODE_PACKAGE, UPDATE_CODE_PACKAGES } from './gqls/codePackage.js'; +import { syncRepositories, type RepositoryInput } from './syncRepositories.js'; +import { syncSoftwareDevelopmentKits } from './syncSoftwareDevelopmentKits.js'; + +/** + * Input for a code package to sync + */ +export interface CodePackageInput { + /** The name of the package */ + name: string; + /** Type of code package */ + type: CodePackageType; + /** Relative path to code package within the repository */ + relativePath: string; + /** Name of repository that the code packages are being uploaded to */ + repositoryName: string; + /** Description of the code package */ + description?: string; + /** Software development kits used by the package */ + softwareDevelopmentKits?: { + /** Name of the SDK */ + name: string; + }[]; + /** Names of the teams that manage the code package */ + teamNames?: string[]; + /** Emails of the owners that manage the code package */ + ownerEmails?: string[]; +} + +const CHUNK_SIZE = 100; + +const LOOKUP_SPLIT_KEY = '%%%%'; + +/** + * Create a new code package + * + * @param client - GraphQL client + * @param options - Options + * @returns Created code package + */ +export async function createCodePackage( + client: GraphQLClient, + options: { + /** Code package input */ + input: { + /** Name of package */ + name: string; + /** Description of package */ + description?: string; + /** Type of package */ + type: CodePackageType; + /** Relative path to package */ + relativePath: string; + /** Repository ID */ + repositoryId?: string; + /** Name of repository */ + repositoryName?: string; + /** IDs of SDKs */ + softwareDevelopmentKitIds?: string[]; + /** IDs of owners */ + ownerIds?: string[]; + /** Emails of owners */ + ownerEmails?: string[]; + /** IDs of teams */ + teamIds?: string[]; + /** Names of teams */ + teamNames?: string[]; + }; + /** Logger instance */ + logger?: Logger; + }, +): Promise { + const { input, logger = NOOP_LOGGER } = options; + const { + createCodePackage: { codePackage }, + } = await makeGraphQLRequest<{ + /** createCodePackage mutation */ + createCodePackage: { + /** Code package */ + codePackage: CodePackage; + }; + }>(client, CREATE_CODE_PACKAGE, { + variables: { input }, + logger, + }); + logger.info(`Successfully created code package "${input.name}"!`); + return codePackage; +} + +/** + * Update existing code packages + * + * @param client - GraphQL client + * @param options - Options + * @returns Code packages that were updated + */ +export async function updateCodePackages( + client: GraphQLClient, + options: { + /** Code package inputs to update */ + input: { + /** ID of code package */ + id: string; + /** Name of package */ + name: string; + /** Description of package */ + description?: string; + /** Type of package */ + type: CodePackageType; + /** Relative path to package */ + relativePath: string; + /** Repository ID */ + repositoryId?: string; + /** Name of repository */ + repositoryName?: string; + /** IDs of SDKs */ + softwareDevelopmentKitIds?: string[]; + /** IDs of owners */ + ownerIds?: string[]; + /** Emails of owners */ + ownerEmails?: string[]; + /** IDs of teams */ + teamIds?: string[]; + /** Names of teams */ + teamNames?: string[]; + }[]; + /** Logger instance */ + logger?: Logger; + }, +): Promise { + const { input: inputs, logger = NOOP_LOGGER } = options; + const { + updateCodePackages: { codePackages }, + } = await makeGraphQLRequest<{ + /** updateCodePackages mutation */ + updateCodePackages: { + /** Code packages */ + codePackages: CodePackage[]; + }; + }>(client, UPDATE_CODE_PACKAGES, { + variables: { + input: { + codePackages: inputs, + }, + }, + logger, + }); + logger.info(`Successfully updated ${inputs.length} code packages!`); + return codePackages; +} + +/** + * Sync code packages: creates missing packages and updates existing ones. + * Also ensures required repositories and software development kits exist. + * + * @param client - GraphQL client + * @param codePackages - Code packages to sync + * @param options - Options + * @returns True if successful, false if any updates failed + */ +export async function syncCodePackages( + client: GraphQLClient, + codePackages: CodePackageInput[], + options: { + /** Concurrency for create operations */ + concurrency?: number; + /** Logger instance */ + logger?: Logger; + } = {}, +): Promise { + const { concurrency = 20, logger = NOOP_LOGGER } = options; + let encounteredError = false; + + const [existingCodePackages, { softwareDevelopmentKits: existingSoftwareDevelopmentKits }] = + await Promise.all([ + fetchAllCodePackages(client, { logger }), + syncSoftwareDevelopmentKits( + client, + uniqBy( + codePackages + .map(({ type, softwareDevelopmentKits = [] }) => + softwareDevelopmentKits.map(({ name }) => ({ + name, + codePackageType: type, + })), + ) + .flat(), + ({ name, codePackageType }) => `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, + ), + { logger, concurrency }, + ), + syncRepositories( + client, + uniqBy(codePackages, 'repositoryName').map( + ({ repositoryName }) => + ({ + name: repositoryName, + url: `https://github.com/${repositoryName}`, + }) as RepositoryInput, + ), + { logger }, + ), + ]); + + const softwareDevelopmentKitLookup = keyBy( + existingSoftwareDevelopmentKits, + ({ name, codePackageType }) => `${name}${LOOKUP_SPLIT_KEY}${codePackageType}`, + ); + const codePackagesLookup = keyBy( + existingCodePackages, + ({ name, type }) => `${name}${LOOKUP_SPLIT_KEY}${type}`, + ); + + const mapCodePackagesToExisting = codePackages.map((codePackageInput) => [ + codePackageInput, + codePackagesLookup[`${codePackageInput.name}${LOOKUP_SPLIT_KEY}${codePackageInput.type}`]?.id, + ]); + + // Create new code packages + const newCodePackages = mapCodePackagesToExisting + .filter(([, existing]) => !existing) + .map(([codePackageInput]) => codePackageInput as CodePackageInput); + try { + logger.info(`Creating "${newCodePackages.length}" new code packages...`); + await map( + newCodePackages, + async ({ softwareDevelopmentKits, ...codePackage }) => { + await createCodePackage(client, { + input: { + ...codePackage, + ...(softwareDevelopmentKits + ? { + softwareDevelopmentKitIds: uniq( + softwareDevelopmentKits.map(({ name }) => { + const sdk = + softwareDevelopmentKitLookup[ + `${name}${LOOKUP_SPLIT_KEY}${codePackage.type}` + ]; + if (!sdk) { + throw new Error(`Failed to find SDK with name: "${name}"`); + } + return sdk.id; + }), + ), + } + : {}), + }, + logger, + }); + }, + { concurrency }, + ); + logger.info(`Successfully synced ${newCodePackages.length} code packages!`); + } catch (err) { + encounteredError = true; + logger.error(`Failed to create code packages! - ${(err as Error).message}`); + } + + // Update existing code packages + const existingCodePackageInputs = mapCodePackagesToExisting.filter( + (x): x is [CodePackageInput, string] => !!x[1], + ); + logger.info(`Updating "${existingCodePackageInputs.length}" code packages...`); + const chunks = chunk(existingCodePackageInputs, CHUNK_SIZE); + + await mapSeries(chunks, async (chk) => { + try { + await updateCodePackages(client, { + input: chk.map(([{ softwareDevelopmentKits, repositoryName, ...input }, id]) => ({ + ...input, + ...(softwareDevelopmentKits + ? { + softwareDevelopmentKitIds: uniq( + softwareDevelopmentKits.map(({ name }) => { + const sdk = + softwareDevelopmentKitLookup[`${name}${LOOKUP_SPLIT_KEY}${input.type}`]; + if (!sdk) { + throw new Error(`Failed to find SDK with name: "${name}"`); + } + return sdk.id; + }), + ), + } + : {}), + id, + })), + logger, + }); + logger.info(`Successfully updated "${chk.length}" code packages!`); + } catch (err) { + encounteredError = true; + logger.error(`Failed to update code packages! - ${(err as Error).message}`); + } + }); + + logger.info(`Synced "${codePackages.length}" code packages!`); + return !encounteredError; +} diff --git a/packages/sdk/src/data-inventory/ensureAllDataSubjectsExist.ts b/packages/sdk/src/data-inventory/ensureAllDataSubjectsExist.ts new file mode 100644 index 00000000..f1e6f58f --- /dev/null +++ b/packages/sdk/src/data-inventory/ensureAllDataSubjectsExist.ts @@ -0,0 +1,80 @@ +import type { Logger } from '@transcend-io/utils'; +import { GraphQLClient } from 'graphql-request'; +import { keyBy, difference, uniq } from 'lodash-es'; + +import { NOOP_LOGGER } from '../api/makeGraphQLRequest.js'; +import { + fetchAllDataSubjects, + createDataSubject, + type DataSubject, +} from '../dsr-automation/fetchDataSubjects.js'; + +/** + * SDK-friendly input describing which data subject types are expected. + * Each array field contains arrays of data subject type strings from different sources. + */ +export interface EnsureDataSubjectsInput { + /** Data subject type arrays from data silos (one array per silo) */ + dataSiloDataSubjects?: string[][]; + /** Data subject type arrays from processing activities (one array per activity) */ + processingActivityDataSubjects?: string[][]; + /** Data subject type arrays from enrichers (one array per enricher) */ + enricherDataSubjects?: string[][]; + /** Data subject types defined directly */ + dataSubjectTypes?: string[]; +} + +/** + * Ensure all referenced data subjects exist, creating any missing ones. + * + * @param client - GraphQL client + * @param options - Options + * @returns Map from data subject type to DataSubject + */ +export async function ensureAllDataSubjectsExist( + client: GraphQLClient, + options: { + /** Input describing expected data subject types */ + input: EnsureDataSubjectsInput; + /** When true, always fetch all subjects even if none are expected */ + fetchAll?: boolean; + /** Logger instance */ + logger?: Logger; + }, +): Promise> { + const { input, fetchAll = false, logger = NOOP_LOGGER } = options; + + const expectedDataSubjects = uniq([ + ...(input.dataSiloDataSubjects ?? []).flat(), + ...(input.processingActivityDataSubjects ?? []).flat(), + ...(input.enricherDataSubjects ?? []).flat(), + ...(input.dataSubjectTypes ?? []), + ]); + + if (expectedDataSubjects.length === 0 && !fetchAll) { + return {}; + } + + const internalSubjects = await fetchAllDataSubjects(client, { logger }); + const dataSubjectByName = keyBy(internalSubjects, 'type'); + + const missingDataSubjects = difference( + expectedDataSubjects, + internalSubjects.map(({ type }) => type), + ); + + if (missingDataSubjects.length > 0) { + logger.info(`Creating ${missingDataSubjects.length} new data subjects...`); + for (const dataSubjectType of missingDataSubjects) { + logger.info(`Creating data subject ${dataSubjectType}...`); + const created = await createDataSubject(client, { + input: dataSubjectType, + logger, + }); + logger.info(`Created data subject ${dataSubjectType}!`); + dataSubjectByName[dataSubjectType] = created; + } + } + + return dataSubjectByName; +} diff --git a/packages/sdk/src/data-inventory/index.ts b/packages/sdk/src/data-inventory/index.ts index 633d8811..1bc61985 100644 --- a/packages/sdk/src/data-inventory/index.ts +++ b/packages/sdk/src/data-inventory/index.ts @@ -1,3 +1,4 @@ +export * from './ensureAllDataSubjectsExist.js'; export * from './fetchAllBusinessEntities.js'; export * from './fetchAllDataCategories.js'; export * from './fetchAllDataPoints.js';