2024-12-20 10:45:04 -08:00
|
|
|
import type { PushMessage } from '@n8n/api-types';
|
2025-01-06 01:21:24 -08:00
|
|
|
import { Container } from '@n8n/di';
|
2023-06-22 15:38:12 -07:00
|
|
|
import { Request } from 'express';
|
2024-12-23 04:46:13 -08:00
|
|
|
import { Logger } from 'n8n-core';
|
2023-06-22 15:38:12 -07:00
|
|
|
import { v4 as uuid } from 'uuid';
|
2024-09-12 09:07:18 -07:00
|
|
|
|
|
|
|
import { ActiveWorkflowManager } from '@/active-workflow-manager';
|
2023-06-22 15:38:12 -07:00
|
|
|
import config from '@/config';
|
2024-09-12 09:07:18 -07:00
|
|
|
import { LICENSE_FEATURES, LICENSE_QUOTAS, UNLIMITED_LICENSE_QUOTA, inE2ETests } from '@/constants';
|
|
|
|
import { AuthUserRepository } from '@/databases/repositories/auth-user.repository';
|
2024-08-27 08:24:20 -07:00
|
|
|
import { SettingsRepository } from '@/databases/repositories/settings.repository';
|
|
|
|
import { UserRepository } from '@/databases/repositories/user.repository';
|
2024-02-28 08:02:18 -08:00
|
|
|
import { Patch, Post, RestController } from '@/decorators';
|
2024-09-12 09:07:18 -07:00
|
|
|
import { MessageEventBus } from '@/eventbus/message-event-bus/message-event-bus';
|
2024-09-13 04:02:00 -07:00
|
|
|
import type { BooleanLicenseFeature, NumericLicenseFeature } from '@/interfaces';
|
2024-09-17 05:10:22 -07:00
|
|
|
import type { FeatureReturnType } from '@/license';
|
2024-09-12 09:07:18 -07:00
|
|
|
import { License } from '@/license';
|
2024-08-22 02:10:37 -07:00
|
|
|
import { MfaService } from '@/mfa/mfa.service';
|
2023-11-28 07:47:28 -08:00
|
|
|
import { Push } from '@/push';
|
perf(core): Improve caching service (#8213)
Story: https://linear.app/n8n/issue/PAY-1188
- Implement Redis hashes on the caching service, based on Micha's work
in #7747, adapted from `node-cache-manager-ioredis-yet`. Optimize
workflow ownership lookups and manual webhook lookups with Redis hashes.
- Simplify the caching service by removing all currently unused methods
and options: `enable`, `disable`, `getCache`, `keys`, `keyValues`,
`refreshFunctionEach`, `refreshFunctionMany`, `refreshTtl`, etc.
- Remove the flag `N8N_CACHE_ENABLED`. Currently some features on
`master` are broken with caching disabled, and test webhooks now rely
entirely on caching, for multi-main setup support. We originally
introduced this flag to protect against excessive memory usage, but
total cache usage is low enough that we decided to drop this setting.
Apparently this flag was also never documented.
- Overall caching service refactor: use generics, reduce branching, add
discriminants for cache kinds for better type safety, type caching
events, improve readability, remove outdated docs, etc. Also refactor
and expand caching service tests.
Follow-up to: https://github.com/n8n-io/n8n/pull/8176
---------
Co-authored-by: Michael Auerswald <michael.auerswald@gmail.com>
2024-01-05 02:52:44 -08:00
|
|
|
import { CacheService } from '@/services/cache/cache.service';
|
2023-12-11 09:23:42 -08:00
|
|
|
import { PasswordUtility } from '@/services/password.utility';
|
2023-06-22 15:38:12 -07:00
|
|
|
|
|
|
|
if (!inE2ETests) {
|
2024-05-03 06:24:27 -07:00
|
|
|
Container.get(Logger).error('E2E endpoints only allowed during E2E tests');
|
2023-06-22 15:38:12 -07:00
|
|
|
process.exit(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
const tablesToTruncate = [
|
|
|
|
'auth_identity',
|
|
|
|
'auth_provider_sync_history',
|
|
|
|
'credentials_entity',
|
2024-05-17 01:53:15 -07:00
|
|
|
'event_destinations',
|
2023-06-22 15:38:12 -07:00
|
|
|
'execution_entity',
|
|
|
|
'installed_nodes',
|
2024-05-17 01:53:15 -07:00
|
|
|
'installed_packages',
|
|
|
|
'project',
|
|
|
|
'project_relation',
|
|
|
|
'settings',
|
|
|
|
'shared_credentials',
|
|
|
|
'shared_workflow',
|
|
|
|
'tag_entity',
|
2023-06-22 15:38:12 -07:00
|
|
|
'user',
|
|
|
|
'variables',
|
2024-05-17 01:53:15 -07:00
|
|
|
'webhook_entity',
|
|
|
|
'workflow_entity',
|
|
|
|
'workflow_statistics',
|
|
|
|
'workflows_tags',
|
2023-06-22 15:38:12 -07:00
|
|
|
];
|
|
|
|
|
2024-12-26 07:09:42 -08:00
|
|
|
type UserSetupPayload = {
|
|
|
|
email: string;
|
|
|
|
password: string;
|
|
|
|
firstName: string;
|
|
|
|
lastName: string;
|
|
|
|
mfaEnabled?: boolean;
|
|
|
|
mfaSecret?: string;
|
|
|
|
mfaRecoveryCodes?: string[];
|
|
|
|
};
|
|
|
|
|
2023-06-22 15:38:12 -07:00
|
|
|
type ResetRequest = Request<
|
|
|
|
{},
|
|
|
|
{},
|
|
|
|
{
|
|
|
|
owner: UserSetupPayload;
|
|
|
|
members: UserSetupPayload[];
|
2023-12-06 05:31:06 -08:00
|
|
|
admin: UserSetupPayload;
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
|
|
|
>;
|
|
|
|
|
2024-12-20 10:45:04 -08:00
|
|
|
type PushRequest = Request<
|
2023-11-28 07:47:28 -08:00
|
|
|
{},
|
|
|
|
{},
|
|
|
|
{
|
2024-04-03 04:43:14 -07:00
|
|
|
pushRef: string;
|
2024-12-20 10:45:04 -08:00
|
|
|
} & PushMessage
|
2023-11-28 07:47:28 -08:00
|
|
|
>;
|
|
|
|
|
2023-06-22 15:38:12 -07:00
|
|
|
@RestController('/e2e')
|
|
|
|
export class E2EController {
|
2023-07-12 05:11:46 -07:00
|
|
|
private enabledFeatures: Record<BooleanLicenseFeature, boolean> = {
|
2023-06-22 15:38:12 -07:00
|
|
|
[LICENSE_FEATURES.SHARING]: false,
|
|
|
|
[LICENSE_FEATURES.LDAP]: false,
|
|
|
|
[LICENSE_FEATURES.SAML]: false,
|
|
|
|
[LICENSE_FEATURES.LOG_STREAMING]: false,
|
|
|
|
[LICENSE_FEATURES.ADVANCED_EXECUTION_FILTERS]: false,
|
|
|
|
[LICENSE_FEATURES.SOURCE_CONTROL]: false,
|
|
|
|
[LICENSE_FEATURES.VARIABLES]: false,
|
|
|
|
[LICENSE_FEATURES.API_DISABLED]: false,
|
2023-08-25 01:33:46 -07:00
|
|
|
[LICENSE_FEATURES.EXTERNAL_SECRETS]: false,
|
2023-08-16 01:05:03 -07:00
|
|
|
[LICENSE_FEATURES.SHOW_NON_PROD_BANNER]: false,
|
2023-08-04 03:27:06 -07:00
|
|
|
[LICENSE_FEATURES.WORKFLOW_HISTORY]: false,
|
2023-08-09 07:38:17 -07:00
|
|
|
[LICENSE_FEATURES.DEBUG_IN_EDITOR]: false,
|
2023-10-05 06:25:17 -07:00
|
|
|
[LICENSE_FEATURES.BINARY_DATA_S3]: false,
|
2023-10-30 08:22:32 -07:00
|
|
|
[LICENSE_FEATURES.MULTIPLE_MAIN_INSTANCES]: false,
|
2023-11-10 14:48:31 -08:00
|
|
|
[LICENSE_FEATURES.WORKER_VIEW]: false,
|
2023-11-28 05:16:47 -08:00
|
|
|
[LICENSE_FEATURES.ADVANCED_PERMISSIONS]: false,
|
2024-05-17 01:53:15 -07:00
|
|
|
[LICENSE_FEATURES.PROJECT_ROLE_ADMIN]: false,
|
|
|
|
[LICENSE_FEATURES.PROJECT_ROLE_EDITOR]: false,
|
|
|
|
[LICENSE_FEATURES.PROJECT_ROLE_VIEWER]: false,
|
2024-08-14 05:59:11 -07:00
|
|
|
[LICENSE_FEATURES.AI_ASSISTANT]: false,
|
2024-08-14 00:44:19 -07:00
|
|
|
[LICENSE_FEATURES.COMMUNITY_NODES_CUSTOM_REGISTRY]: false,
|
2024-10-09 08:24:33 -07:00
|
|
|
[LICENSE_FEATURES.ASK_AI]: false,
|
2024-12-27 07:23:20 -08:00
|
|
|
[LICENSE_FEATURES.AI_CREDITS]: false,
|
2024-05-17 01:53:15 -07:00
|
|
|
};
|
|
|
|
|
2025-01-09 09:08:49 -08:00
|
|
|
private static readonly numericFeaturesDefaults: Record<NumericLicenseFeature, number> = {
|
2024-05-17 01:53:15 -07:00
|
|
|
[LICENSE_QUOTAS.TRIGGER_LIMIT]: -1,
|
|
|
|
[LICENSE_QUOTAS.VARIABLES_LIMIT]: -1,
|
|
|
|
[LICENSE_QUOTAS.USERS_LIMIT]: -1,
|
|
|
|
[LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT]: -1,
|
|
|
|
[LICENSE_QUOTAS.TEAM_PROJECT_LIMIT]: 0,
|
2024-12-27 07:23:20 -08:00
|
|
|
[LICENSE_QUOTAS.AI_CREDITS]: 0,
|
2023-06-22 15:38:12 -07:00
|
|
|
};
|
|
|
|
|
2025-01-09 09:08:49 -08:00
|
|
|
private numericFeatures: Record<NumericLicenseFeature, number> = {
|
|
|
|
[LICENSE_QUOTAS.TRIGGER_LIMIT]:
|
|
|
|
E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.TRIGGER_LIMIT],
|
|
|
|
[LICENSE_QUOTAS.VARIABLES_LIMIT]:
|
|
|
|
E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.VARIABLES_LIMIT],
|
|
|
|
[LICENSE_QUOTAS.USERS_LIMIT]: E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.USERS_LIMIT],
|
|
|
|
[LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT]:
|
|
|
|
E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.WORKFLOW_HISTORY_PRUNE_LIMIT],
|
|
|
|
[LICENSE_QUOTAS.TEAM_PROJECT_LIMIT]:
|
|
|
|
E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.TEAM_PROJECT_LIMIT],
|
|
|
|
[LICENSE_QUOTAS.AI_CREDITS]: E2EController.numericFeaturesDefaults[LICENSE_QUOTAS.AI_CREDITS],
|
|
|
|
};
|
|
|
|
|
2023-06-22 15:38:12 -07:00
|
|
|
constructor(
|
|
|
|
license: License,
|
2023-12-27 02:50:43 -08:00
|
|
|
private readonly settingsRepo: SettingsRepository,
|
2024-05-06 08:54:05 -07:00
|
|
|
private readonly workflowRunner: ActiveWorkflowManager,
|
2023-12-27 02:50:43 -08:00
|
|
|
private readonly mfaService: MfaService,
|
|
|
|
private readonly cacheService: CacheService,
|
|
|
|
private readonly push: Push,
|
2023-12-11 09:23:42 -08:00
|
|
|
private readonly passwordUtility: PasswordUtility,
|
2024-01-26 03:21:15 -08:00
|
|
|
private readonly eventBus: MessageEventBus,
|
2024-05-17 01:53:15 -07:00
|
|
|
private readonly userRepository: UserRepository,
|
2024-05-31 00:40:19 -07:00
|
|
|
private readonly authUserRepository: AuthUserRepository,
|
2023-06-22 15:38:12 -07:00
|
|
|
) {
|
2023-07-12 05:11:46 -07:00
|
|
|
license.isFeatureEnabled = (feature: BooleanLicenseFeature) =>
|
2023-06-22 15:38:12 -07:00
|
|
|
this.enabledFeatures[feature] ?? false;
|
2024-09-17 05:10:22 -07:00
|
|
|
|
|
|
|
// Ugly hack to satisfy biome parser
|
|
|
|
const getFeatureValue = <T extends keyof FeatureReturnType>(
|
|
|
|
feature: T,
|
|
|
|
): FeatureReturnType[T] => {
|
|
|
|
if (feature in this.numericFeatures) {
|
|
|
|
return this.numericFeatures[feature as NumericLicenseFeature] as FeatureReturnType[T];
|
|
|
|
} else {
|
|
|
|
return UNLIMITED_LICENSE_QUOTA as FeatureReturnType[T];
|
|
|
|
}
|
|
|
|
};
|
|
|
|
license.getFeatureValue = getFeatureValue;
|
2024-07-18 06:34:39 -07:00
|
|
|
|
|
|
|
license.getPlanName = () => 'Enterprise';
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
|
|
|
|
2024-02-28 08:02:18 -08:00
|
|
|
@Post('/reset', { skipAuth: true })
|
2023-06-22 15:38:12 -07:00
|
|
|
async reset(req: ResetRequest) {
|
|
|
|
this.resetFeatures();
|
|
|
|
await this.resetLogStreaming();
|
2023-07-05 11:01:24 -07:00
|
|
|
await this.removeActiveWorkflows();
|
2023-06-22 15:38:12 -07:00
|
|
|
await this.truncateAll();
|
2023-12-06 05:31:06 -08:00
|
|
|
await this.resetCache();
|
|
|
|
await this.setupUserManagement(req.body.owner, req.body.members, req.body.admin);
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
|
|
|
|
2024-02-28 08:02:18 -08:00
|
|
|
@Post('/push', { skipAuth: true })
|
2024-12-20 10:45:04 -08:00
|
|
|
async pushSend(req: PushRequest) {
|
|
|
|
const { pushRef: _, ...pushMsg } = req.body;
|
|
|
|
this.push.broadcast(pushMsg);
|
2023-11-28 07:47:28 -08:00
|
|
|
}
|
|
|
|
|
2024-02-28 08:02:18 -08:00
|
|
|
@Patch('/feature', { skipAuth: true })
|
2023-07-12 05:11:46 -07:00
|
|
|
setFeature(req: Request<{}, {}, { feature: BooleanLicenseFeature; enabled: boolean }>) {
|
2023-06-22 15:38:12 -07:00
|
|
|
const { enabled, feature } = req.body;
|
|
|
|
this.enabledFeatures[feature] = enabled;
|
|
|
|
}
|
|
|
|
|
2024-05-17 01:53:15 -07:00
|
|
|
@Patch('/quota', { skipAuth: true })
|
|
|
|
setQuota(req: Request<{}, {}, { feature: NumericLicenseFeature; value: number }>) {
|
|
|
|
const { value, feature } = req.body;
|
|
|
|
this.numericFeatures[feature] = value;
|
|
|
|
}
|
|
|
|
|
2024-02-28 08:02:18 -08:00
|
|
|
@Patch('/queue-mode', { skipAuth: true })
|
2023-11-10 14:48:31 -08:00
|
|
|
async setQueueMode(req: Request<{}, {}, { enabled: boolean }>) {
|
|
|
|
const { enabled } = req.body;
|
|
|
|
config.set('executions.mode', enabled ? 'queue' : 'regular');
|
|
|
|
return { success: true, message: `Queue mode set to ${config.getEnv('executions.mode')}` };
|
|
|
|
}
|
|
|
|
|
2023-06-22 15:38:12 -07:00
|
|
|
private resetFeatures() {
|
|
|
|
for (const feature of Object.keys(this.enabledFeatures)) {
|
2023-07-12 05:11:46 -07:00
|
|
|
this.enabledFeatures[feature as BooleanLicenseFeature] = false;
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
2025-01-09 09:08:49 -08:00
|
|
|
|
|
|
|
for (const feature of Object.keys(this.numericFeatures)) {
|
|
|
|
this.numericFeatures[feature as NumericLicenseFeature] =
|
|
|
|
E2EController.numericFeaturesDefaults[feature as NumericLicenseFeature];
|
|
|
|
}
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
|
|
|
|
2023-07-05 11:01:24 -07:00
|
|
|
private async removeActiveWorkflows() {
|
|
|
|
this.workflowRunner.removeAllQueuedWorkflowActivations();
|
|
|
|
await this.workflowRunner.removeAll();
|
|
|
|
}
|
|
|
|
|
2023-06-22 15:38:12 -07:00
|
|
|
private async resetLogStreaming() {
|
2024-01-26 03:21:15 -08:00
|
|
|
for (const id in this.eventBus.destinations) {
|
|
|
|
await this.eventBus.removeDestination(id, false);
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
private async truncateAll() {
|
|
|
|
for (const table of tablesToTruncate) {
|
|
|
|
try {
|
2024-01-24 04:38:57 -08:00
|
|
|
const { connection } = this.settingsRepo.manager;
|
2023-06-22 15:38:12 -07:00
|
|
|
await connection.query(
|
|
|
|
`DELETE FROM ${table}; DELETE FROM sqlite_sequence WHERE name=${table};`,
|
|
|
|
);
|
|
|
|
} catch (error) {
|
2024-05-03 06:24:27 -07:00
|
|
|
Container.get(Logger).warn('Dropping Table for E2E Reset error', {
|
|
|
|
error: error as Error,
|
|
|
|
});
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2023-12-06 05:31:06 -08:00
|
|
|
private async setupUserManagement(
|
|
|
|
owner: UserSetupPayload,
|
|
|
|
members: UserSetupPayload[],
|
|
|
|
admin: UserSetupPayload,
|
|
|
|
) {
|
2024-05-17 01:53:15 -07:00
|
|
|
const userCreatePromises = [
|
|
|
|
this.userRepository.createUserWithProject({
|
|
|
|
id: uuid(),
|
|
|
|
...owner,
|
|
|
|
password: await this.passwordUtility.hash(owner.password),
|
|
|
|
role: 'global:owner',
|
|
|
|
}),
|
|
|
|
];
|
|
|
|
|
|
|
|
userCreatePromises.push(
|
|
|
|
this.userRepository.createUserWithProject({
|
|
|
|
id: uuid(),
|
|
|
|
...admin,
|
|
|
|
password: await this.passwordUtility.hash(admin.password),
|
|
|
|
role: 'global:admin',
|
|
|
|
}),
|
|
|
|
);
|
2023-08-23 19:59:16 -07:00
|
|
|
|
2023-06-22 15:38:12 -07:00
|
|
|
for (const { password, ...payload } of members) {
|
2024-05-17 01:53:15 -07:00
|
|
|
userCreatePromises.push(
|
|
|
|
this.userRepository.createUserWithProject({
|
2023-06-22 15:38:12 -07:00
|
|
|
id: uuid(),
|
|
|
|
...payload,
|
2023-12-11 09:23:42 -08:00
|
|
|
password: await this.passwordUtility.hash(password),
|
2024-01-24 04:38:57 -08:00
|
|
|
role: 'global:member',
|
2023-06-22 15:38:12 -07:00
|
|
|
}),
|
|
|
|
);
|
|
|
|
}
|
|
|
|
|
2024-05-31 00:40:19 -07:00
|
|
|
const [newOwner] = await Promise.all(userCreatePromises);
|
|
|
|
|
|
|
|
if (owner?.mfaSecret && owner.mfaRecoveryCodes?.length) {
|
|
|
|
const { encryptedRecoveryCodes, encryptedSecret } =
|
|
|
|
this.mfaService.encryptSecretAndRecoveryCodes(owner.mfaSecret, owner.mfaRecoveryCodes);
|
|
|
|
|
|
|
|
await this.authUserRepository.update(newOwner.user.id, {
|
|
|
|
mfaSecret: encryptedSecret,
|
|
|
|
mfaRecoveryCodes: encryptedRecoveryCodes,
|
|
|
|
});
|
|
|
|
}
|
2023-06-22 15:38:12 -07:00
|
|
|
|
|
|
|
await this.settingsRepo.update(
|
|
|
|
{ key: 'userManagement.isInstanceOwnerSetUp' },
|
|
|
|
{ value: 'true' },
|
|
|
|
);
|
|
|
|
|
|
|
|
config.set('userManagement.isInstanceOwnerSetUp', true);
|
|
|
|
}
|
2023-12-06 05:31:06 -08:00
|
|
|
|
|
|
|
private async resetCache() {
|
|
|
|
await this.cacheService.reset();
|
|
|
|
}
|
2023-06-22 15:38:12 -07:00
|
|
|
}
|