From d597c2ab29c7997572facdb8e9a0eda7f2d61a73 Mon Sep 17 00:00:00 2001 From: Cornelius Suermann Date: Mon, 22 Jan 2024 12:29:28 +0100 Subject: [PATCH] feat: Extend collection of usage metrics during license renewal (no-changelog) (#8369) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Iván Ovejero --- packages/cli/src/License.ts | 15 +--- .../cli/src/controllers/auth.controller.ts | 2 +- .../src/controllers/invitation.controller.ts | 2 +- .../cli/src/controllers/users.controller.ts | 2 +- .../repositories/usageMetrics.repository.ts | 75 ++++++++++++++++++ .../databases/repositories/user.repository.ts | 2 +- .../cli/src/services/usageMetrics.service.ts | 27 +++++++ .../test/integration/shared/db/credentials.ts | 25 ++++++ .../usageMetrics.repository.test.ts | 78 +++++++++++++++++++ 9 files changed, 212 insertions(+), 16 deletions(-) create mode 100644 packages/cli/src/databases/repositories/usageMetrics.repository.ts create mode 100644 packages/cli/src/services/usageMetrics.service.ts create mode 100644 packages/cli/test/integration/usageMetrics.repository.test.ts diff --git a/packages/cli/src/License.ts b/packages/cli/src/License.ts index 287edbea37..ce678939c8 100644 --- a/packages/cli/src/License.ts +++ b/packages/cli/src/License.ts @@ -12,12 +12,12 @@ import { UNLIMITED_LICENSE_QUOTA, } from './constants'; import { SettingsRepository } from '@db/repositories/settings.repository'; -import { WorkflowRepository } from '@db/repositories/workflow.repository'; import type { BooleanLicenseFeature, N8nInstanceType, NumericLicenseFeature } from './Interfaces'; import type { RedisServicePubSubPublisher } from './services/redis/RedisServicePubSubPublisher'; import { RedisService } from './services/redis.service'; import { OrchestrationService } from '@/services/orchestration.service'; import { OnShutdown } from '@/decorators/OnShutdown'; +import { UsageMetricsService } from './services/usageMetrics.service'; type FeatureReturnType = Partial< { @@ -38,7 +38,7 @@ export class License { private readonly instanceSettings: InstanceSettings, private readonly orchestrationService: OrchestrationService, private readonly settingsRepository: SettingsRepository, - private readonly workflowRepository: WorkflowRepository, + private readonly usageMetricsService: UsageMetricsService, ) {} async init(instanceType: N8nInstanceType = 'main') { @@ -63,7 +63,7 @@ export class License { ? async (features: TFeatures) => await this.onFeatureChange(features) : async () => {}; const collectUsageMetrics = isMainInstance - ? async () => await this.collectUsageMetrics() + ? async () => await this.usageMetricsService.collectUsageMetrics() : async () => []; try { @@ -91,15 +91,6 @@ export class License { } } - async collectUsageMetrics() { - return [ - { - name: 'activeWorkflows', - value: await this.workflowRepository.count({ where: { active: true } }), - }, - ]; - } - async loadCertStr(): Promise { // if we have an ephemeral license, we don't want to load it from the database const ephemeralLicense = config.get('license.cert'); diff --git a/packages/cli/src/controllers/auth.controller.ts b/packages/cli/src/controllers/auth.controller.ts index 48d0e777e9..e0fce4c2f3 100644 --- a/packages/cli/src/controllers/auth.controller.ts +++ b/packages/cli/src/controllers/auth.controller.ts @@ -187,7 +187,7 @@ export class AuthController { } } - const users = await this.userRepository.findManybyIds([inviterId, inviteeId]); + const users = await this.userRepository.findManyByIds([inviterId, inviteeId]); if (users.length !== 2) { this.logger.debug( diff --git a/packages/cli/src/controllers/invitation.controller.ts b/packages/cli/src/controllers/invitation.controller.ts index 192de142f1..864754e1f4 100644 --- a/packages/cli/src/controllers/invitation.controller.ts +++ b/packages/cli/src/controllers/invitation.controller.ts @@ -136,7 +136,7 @@ export class InvitationController { const validPassword = this.passwordUtility.validate(password); - const users = await this.userRepository.findManybyIds([inviterId, inviteeId]); + const users = await this.userRepository.findManyByIds([inviterId, inviteeId]); if (users.length !== 2) { this.logger.debug( diff --git a/packages/cli/src/controllers/users.controller.ts b/packages/cli/src/controllers/users.controller.ts index 0918c5a362..4709a22d14 100644 --- a/packages/cli/src/controllers/users.controller.ts +++ b/packages/cli/src/controllers/users.controller.ts @@ -172,7 +172,7 @@ export class UsersController { const userIds = transferId ? [transferId, idToDelete] : [idToDelete]; - const users = await this.userRepository.findManybyIds(userIds); + const users = await this.userRepository.findManyByIds(userIds); if (!users.length || (transferId && users.length !== 2)) { throw new NotFoundError( diff --git a/packages/cli/src/databases/repositories/usageMetrics.repository.ts b/packages/cli/src/databases/repositories/usageMetrics.repository.ts new file mode 100644 index 0000000000..c7cd36ed9b --- /dev/null +++ b/packages/cli/src/databases/repositories/usageMetrics.repository.ts @@ -0,0 +1,75 @@ +import config from '@/config'; +import { Service } from 'typedi'; +import { DataSource, Repository, Entity } from 'typeorm'; + +@Entity() +export class UsageMetrics {} + +@Service() +export class UsageMetricsRepository extends Repository { + constructor(dataSource: DataSource) { + super(UsageMetrics, dataSource.manager); + } + + toTableName(name: string) { + const tablePrefix = config.getEnv('database.tablePrefix'); + + let tableName = + config.getEnv('database.type') === 'mysqldb' + ? `\`${tablePrefix}${name}\`` + : `"${tablePrefix}${name}"`; + + const pgSchema = config.getEnv('database.postgresdb.schema'); + + if (pgSchema !== 'public') tableName = [pgSchema, tablePrefix + name].join('.'); + + return tableName; + } + + async getLicenseRenewalMetrics() { + type Row = { + enabled_user_count: string | number; + active_workflow_count: string | number; + total_workflow_count: string | number; + total_credentials_count: string | number; + production_executions_count: string | number; + manual_executions_count: string | number; + }; + + const userTable = this.toTableName('user'); + const workflowTable = this.toTableName('workflow_entity'); + const credentialTable = this.toTableName('credentials_entity'); + const workflowStatsTable = this.toTableName('workflow_statistics'); + + const [ + { + enabled_user_count: enabledUsers, + active_workflow_count: activeWorkflows, + total_workflow_count: totalWorkflows, + total_credentials_count: totalCredentials, + production_executions_count: productionExecutions, + manual_executions_count: manualExecutions, + }, + ] = (await this.query(` + SELECT + (SELECT COUNT(*) FROM ${userTable} WHERE disabled = false) AS enabled_user_count, + (SELECT COUNT(*) FROM ${workflowTable} WHERE active = true) AS active_workflow_count, + (SELECT COUNT(*) FROM ${workflowTable}) AS total_workflow_count, + (SELECT COUNT(*) FROM ${credentialTable}) AS total_credentials_count, + (SELECT SUM(count) FROM ${workflowStatsTable} WHERE name IN ('production_success', 'production_error')) AS production_executions_count, + (SELECT SUM(count) FROM ${workflowStatsTable} WHERE name IN ('manual_success', 'manual_error')) AS manual_executions_count; + `)) as Row[]; + + const toNumber = (value: string | number) => + typeof value === 'number' ? value : parseInt(value, 10); + + return { + enabledUsers: toNumber(enabledUsers), + activeWorkflows: toNumber(activeWorkflows), + totalWorkflows: toNumber(totalWorkflows), + totalCredentials: toNumber(totalCredentials), + productionExecutions: toNumber(productionExecutions), + manualExecutions: toNumber(manualExecutions), + }; + } +} diff --git a/packages/cli/src/databases/repositories/user.repository.ts b/packages/cli/src/databases/repositories/user.repository.ts index c21684d513..88e46c96a9 100644 --- a/packages/cli/src/databases/repositories/user.repository.ts +++ b/packages/cli/src/databases/repositories/user.repository.ts @@ -10,7 +10,7 @@ export class UserRepository extends Repository { super(User, dataSource.manager); } - async findManybyIds(userIds: string[]) { + async findManyByIds(userIds: string[]) { return await this.find({ where: { id: In(userIds) }, relations: ['globalRole'], diff --git a/packages/cli/src/services/usageMetrics.service.ts b/packages/cli/src/services/usageMetrics.service.ts new file mode 100644 index 0000000000..4f8b83923b --- /dev/null +++ b/packages/cli/src/services/usageMetrics.service.ts @@ -0,0 +1,27 @@ +import { UsageMetricsRepository } from '@/databases/repositories/usageMetrics.repository'; +import { Service } from 'typedi'; + +@Service() +export class UsageMetricsService { + constructor(private readonly usageMetricsRepository: UsageMetricsRepository) {} + + async collectUsageMetrics() { + const { + activeWorkflows, + totalWorkflows, + enabledUsers, + totalCredentials, + productionExecutions, + manualExecutions, + } = await this.usageMetricsRepository.getLicenseRenewalMetrics(); + + return [ + { name: 'activeWorkflows', value: activeWorkflows }, + { name: 'totalWorkflows', value: totalWorkflows }, + { name: 'enabledUsers', value: enabledUsers }, + { name: 'totalCredentials', value: totalCredentials }, + { name: 'productionExecutions', value: productionExecutions }, + { name: 'manualExecutions', value: manualExecutions }, + ]; + } +} diff --git a/packages/cli/test/integration/shared/db/credentials.ts b/packages/cli/test/integration/shared/db/credentials.ts index cc6c520869..f92507ce8b 100644 --- a/packages/cli/test/integration/shared/db/credentials.ts +++ b/packages/cli/test/integration/shared/db/credentials.ts @@ -18,6 +18,31 @@ async function encryptCredentialData(credential: CredentialsEntity) { return coreCredential.getDataToSave() as ICredentialsDb; } +const emptyAttributes = { + name: 'test', + type: 'test', + data: '', + nodesAccess: [], +}; + +export async function createManyCredentials( + amount: number, + attributes: Partial = emptyAttributes, +) { + return await Promise.all( + Array(amount) + .fill(0) + .map(async () => await createCredentials(attributes)), + ); +} + +export async function createCredentials(attributes: Partial = emptyAttributes) { + const credentialsRepository = Container.get(CredentialsRepository); + const entity = credentialsRepository.create(attributes); + + return await credentialsRepository.save(entity); +} + /** * Save a credential to the test DB, sharing it with a user. */ diff --git a/packages/cli/test/integration/usageMetrics.repository.test.ts b/packages/cli/test/integration/usageMetrics.repository.test.ts new file mode 100644 index 0000000000..602536f12a --- /dev/null +++ b/packages/cli/test/integration/usageMetrics.repository.test.ts @@ -0,0 +1,78 @@ +import { UsageMetricsRepository } from '@/databases/repositories/usageMetrics.repository'; +import { createAdmin, createMember, createOwner, createUser } from './shared/db/users'; +import * as testDb from './shared/testDb'; +import Container from 'typedi'; +import { CredentialsRepository } from '@/databases/repositories/credentials.repository'; +import { createManyWorkflows } from './shared/db/workflows'; +import { createManyCredentials } from './shared/db/credentials'; +import { WorkflowStatisticsRepository } from '@/databases/repositories/workflowStatistics.repository'; +import { StatisticsNames } from '@/databases/entities/WorkflowStatistics'; +import { WorkflowRepository } from '@/databases/repositories/workflow.repository'; + +describe('UsageMetricsRepository', () => { + let usageMetricsRepository: UsageMetricsRepository; + let credentialsRepository: CredentialsRepository; + let workflowStatisticsRepository: WorkflowStatisticsRepository; + let workflowRepository: WorkflowRepository; + + beforeAll(async () => { + await testDb.init(); + + usageMetricsRepository = Container.get(UsageMetricsRepository); + credentialsRepository = Container.get(CredentialsRepository); + workflowStatisticsRepository = Container.get(WorkflowStatisticsRepository); + workflowRepository = Container.get(WorkflowRepository); + + await testDb.truncate(['User', 'Credentials', 'Workflow', 'Execution', 'WorkflowStatistics']); + }); + + afterAll(async () => { + await testDb.terminate(); + }); + + describe('getLicenseRenewalMetrics()', () => { + test('should return license renewal metrics', async () => { + const [firstWorkflow, secondWorkflow] = await createManyWorkflows(2, { active: false }); + + await Promise.all([ + createOwner(), + createAdmin(), + createMember(), + createMember(), + createUser({ disabled: true }), + createManyCredentials(2), + createManyWorkflows(3, { active: true }), + ]); + + await Promise.all([ + workflowStatisticsRepository.insertWorkflowStatistics( + StatisticsNames.productionSuccess, + firstWorkflow.id, + ), + workflowStatisticsRepository.insertWorkflowStatistics( + StatisticsNames.productionError, + firstWorkflow.id, + ), + workflowStatisticsRepository.insertWorkflowStatistics( + StatisticsNames.manualSuccess, + secondWorkflow.id, + ), + workflowStatisticsRepository.insertWorkflowStatistics( + StatisticsNames.manualError, + secondWorkflow.id, + ), + ]); + + const metrics = await usageMetricsRepository.getLicenseRenewalMetrics(); + + expect(metrics).toStrictEqual({ + enabledUsers: 4, + totalCredentials: 2, + totalWorkflows: 5, + activeWorkflows: 3, + productionExecutions: 2, + manualExecutions: 2, + }); + }); + }); +});