n8n/packages/cli/src/Db.ts
Jan Oberhauser c85faff4f1
feat: Make it possible to dynamically load community nodes (#2849)
*  Make it possible to dynamically load node packages

*  Fix comment

*  Make possible to dynamically install nodes from npm

* Created migration for sqlite regarding community nodes

* Saving to db whenever a package with nodes is installed

* Created endpoint to fetch installed packages

* WIP - uninstall package with nodes

* Fix lint issues

* Updating nodes via API

* Lint and improvement fixes

* Created community node helpers and removed packages taht do not contain nodes

* Check for package updates when fetching installed packages

* Blocked access to non-owner and preventing incorrect install of packages

* Added auto healing process

* Unit tests for helpers

* Finishing tests for helpers

* Improved unit tests, refactored more helpers and created integration tests for GET

* Implemented detection of missing packages on init and added warning to frontend settings

* Add check for banned packages and fix broken tests

* Create migrations for other db systems

* Updated with latest changes from master

* Fixed conflict errors

* Improved unit tests, refactored more helpers and created integration tests for GET

* Implemented detection of missing packages on init and added warning to frontend settings

* 🔥 Removing access check for the Settings sidebar item

*  Added inital community nodes settings screen

* Added executionMode flag to settings

*  Implemented N8N-callout component

* 💄Updating Callout component template propery names

* 💄 Updating Callout component styling.

* 💄Updating Callout component sizing and colors.

* ✔️ Updating Callout component test snapshots after styling changes

*  Updating the `ActionBox` component so it supports callouts and conditional button rendering

* 💄 Removing duplicate callout theme validation in the `ActionBox` component. Adding a selection control for it in the storybook.

*  Added warning message if instance is in the queue mode. Updated colors based on the new design.

*  Added a custom permission support to router

* 🔨 Implemented UM detection as a custom permission.

* 👌Updating route permission logic.

*  Implemented installed community packages list in the settings view

* 👌 Updating settings routes rules and community nodes setting view.

* Allow installation of packages that failed to load

* 👌 Updating `ActionBox`, `CommuntyPackageCard` components and settings loading logic.

* 👌 Fixing community nodes loading state and sidebar icon spacing.

*  Implemented loading skeletons for community package cards

* 👌 Handling errrors while loading installed package list. Updating spacing.

* 👌 Updating community nodes error messages.

* Added disable flag

* 🐛 Fixing a community nodes update detection bug when there are missing packages. (#3497)

*  Added front-end support for community nodes feature flag

*  Implemented community package installation modal dialog

* 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state.

*  Implemented community packages install flow

* Standardize error codes (#3501)

* Standardize error: 400 for request issues such as invalid package name and 500 for installation problems

* Fix http status code for when package is not found

*  Implemented community package installation modal dialog

* 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state.

*  Implemented community packages install flow

*  Updated error handling based on the response codes

*  Implemented community package installation modal dialog

*  Implemented community package uninstall flow.

*  Finished update confirm modal UI

* 💄 Replaced community nodes tooltip image with the one exported from figma.

*  Implemented community package update process

*  Updating community nodes list after successful package update

* 🔒 Updating public API setting route to use new access rules. Updating express app definition in community nodes tests

*  Implemented community package installation modal dialog

* 💄 Community nodes installation modal updates: Moved links to constants and used them in translations, disabling inputs in loading state.

*  Implemented community packages install flow

*  Updated error handling based on the response codes

* Change output for installation request

* Improve payload for update requests

* 👌 Updating community nodes install modal UI

* 👌 Updating community nodes confirm modal logic

* 👌 Refactoring community nodes confirm modal dialog

* 👌 Separating community nodes components loading states

* 💄 Updating community nodes install modal spacing.

* Fix behavior for installing already installed packages

* 💡 Commenting community nodes install process

* 🔥 Removing leftover commits of deleted Vue mutations

*  Updated node list to identify community nodes and handle node name clash

*  Implemented missing community node dialog.

* 💄 Updating n8n-tabs component to support tooltips

*  Updating node details with community node details.

* 🔨 Using back-end response when updating community packages

* 👌 Updating tabs component and refactoring community nodes store mutations

* 👌 Adding community node flag to node type descriptions and using it to identify community nodes

* 👌 Hiding unnecessary elements from missing node details panel.

* 👌 Updating missing node type descriptions for custom and community nodes

* 👌 Updating community node package name detection logic

* 👌 Removing communityNode flag from node description

*  Adding `force` flag to credentials fetching (#3527)

*  Adding `force` flag to credentials fetching which can be used to skip check when loading credentials

*  Forcing credentials loading when opening nodeView

* 👌 Minor updates to community nodes details panel

* tests for post endpoint

* duplicate comments

* Add Patch and Delete enpoints tests

* 🔒 Using `pageCategory`prop to assemble the list of settings routes instead of hard-coded array (#3562)

* 📈 Added front-end telemetry events for community nodes

* 📈 Updating community nodes telemetry events

* 💄 Updating community nodes settings UI elements based on product/design review

* 💄 Updating node view & node details view for community nodes based on product/design feedback

* 💄 Fixing community node text capitalisation

*  Adding community node install error message under the package name input field

* Fixed and improved tests

* Fix lint issue

* feat: Migrated to npm release of riot-tmpl fork.

* 📈 Updating community nodes telemetry events based on the product review

* 💄 Updating community nodes UI based on the design feedback

* 🔀 Merging recent node draggable panels changes

* Implement self healing process

* Improve error messages for package name requirement and disk space

* 💄 Removing front-end error message override since appropriate response is available from the back-end

* Fix lint issues

* Fix installed node name

* 💄 Removed additional node name parsing

* 📈 Updating community nodes telemetry events

* Fix postgres migration for cascading nodes when package is removed

* Remove postman mock for banned packages

* 📈 Adding missing telemetry event for community node documentation click

* 🐛 Fixing community nodes UI bugs reported during the bug bash

* Fix issue with uninstalling packages not reflecting UI

* 🐛 Fixing a missing node type bug when trying to run a workflow.

* Improve error detection for installing packages

* 💄 Updating community nodes components styling and wording based on the product feedback

* Implement telemetry be events

* Add author name and email to packages

* Fix telemetry be events for community packages

* 📈 Updating front-end telemetry events with community nodes author data

* 💄 Updating credentials documentation link logic to handle community nodes credentials

* 🐛 Fixing draggable panels logic

* Fix duplicate wrong import

* 💄 Hiding community nodes credentials documentation links when they don't contain an absolute URL

* Fix issue with detection of missing packages

* 💄 Adding the `Docs` tab to community nodes

* 💄 Adding a failed loading indicator to community nodes list

* Prevent n8n from crashing on startup

* Refactor and improve code quality

*  Remove not needed depenedency

Co-authored-by: Omar Ajoue <krynble@gmail.com>
Co-authored-by: Milorad Filipović <milorad@n8n.io>
Co-authored-by: Milorad FIlipović <miloradfilipovic19@gmail.com>
Co-authored-by: agobrech <ael.gobrecht@gmail.com>
Co-authored-by: Alex Grozav <alex@grozav.com>
2022-07-20 16:24:03 +02:00

204 lines
6.9 KiB
TypeScript

/* eslint-disable import/no-mutable-exports */
/* eslint-disable import/no-cycle */
/* eslint-disable @typescript-eslint/no-unsafe-assignment */
/* eslint-disable @typescript-eslint/restrict-template-expressions */
/* eslint-disable no-case-declarations */
/* eslint-disable @typescript-eslint/naming-convention */
import { UserSettings } from 'n8n-core';
import {
Connection,
ConnectionOptions,
createConnection,
EntityManager,
EntityTarget,
getRepository,
LoggerOptions,
Repository,
} from 'typeorm';
import { TlsOptions } from 'tls';
import path from 'path';
// eslint-disable-next-line import/no-cycle
import { DatabaseType, GenericHelpers, IDatabaseCollections } from '.';
import config from '../config';
// eslint-disable-next-line import/no-cycle
import { entities } from './databases/entities';
import { postgresMigrations } from './databases/migrations/postgresdb';
import { mysqlMigrations } from './databases/migrations/mysqldb';
import { sqliteMigrations } from './databases/migrations/sqlite';
export let isInitialized = false;
export const collections = {} as IDatabaseCollections;
let connection: Connection;
export async function transaction<T>(fn: (entityManager: EntityManager) => Promise<T>): Promise<T> {
return connection.transaction(fn);
}
export function linkRepository<Entity>(entityClass: EntityTarget<Entity>): Repository<Entity> {
return getRepository(entityClass, connection.name);
}
export async function init(
testConnectionOptions?: ConnectionOptions,
): Promise<IDatabaseCollections> {
if (isInitialized) return collections;
const dbType = (await GenericHelpers.getConfigValue('database.type')) as DatabaseType;
const n8nFolder = UserSettings.getUserN8nFolderPath();
let connectionOptions: ConnectionOptions;
const entityPrefix = config.getEnv('database.tablePrefix');
if (testConnectionOptions) {
connectionOptions = testConnectionOptions;
} else {
switch (dbType) {
case 'postgresdb':
const sslCa = (await GenericHelpers.getConfigValue('database.postgresdb.ssl.ca')) as string;
const sslCert = (await GenericHelpers.getConfigValue(
'database.postgresdb.ssl.cert',
)) as string;
const sslKey = (await GenericHelpers.getConfigValue(
'database.postgresdb.ssl.key',
)) as string;
const sslRejectUnauthorized = (await GenericHelpers.getConfigValue(
'database.postgresdb.ssl.rejectUnauthorized',
)) as boolean;
let ssl: TlsOptions | undefined;
if (sslCa !== '' || sslCert !== '' || sslKey !== '' || !sslRejectUnauthorized) {
ssl = {
ca: sslCa || undefined,
cert: sslCert || undefined,
key: sslKey || undefined,
rejectUnauthorized: sslRejectUnauthorized,
};
}
connectionOptions = {
type: 'postgres',
entityPrefix,
database: (await GenericHelpers.getConfigValue('database.postgresdb.database')) as string,
host: (await GenericHelpers.getConfigValue('database.postgresdb.host')) as string,
password: (await GenericHelpers.getConfigValue('database.postgresdb.password')) as string,
port: (await GenericHelpers.getConfigValue('database.postgresdb.port')) as number,
username: (await GenericHelpers.getConfigValue('database.postgresdb.user')) as string,
schema: config.getEnv('database.postgresdb.schema'),
migrations: postgresMigrations,
migrationsRun: true,
migrationsTableName: `${entityPrefix}migrations`,
ssl,
};
break;
case 'mariadb':
case 'mysqldb':
connectionOptions = {
type: dbType === 'mysqldb' ? 'mysql' : 'mariadb',
database: (await GenericHelpers.getConfigValue('database.mysqldb.database')) as string,
entityPrefix,
host: (await GenericHelpers.getConfigValue('database.mysqldb.host')) as string,
password: (await GenericHelpers.getConfigValue('database.mysqldb.password')) as string,
port: (await GenericHelpers.getConfigValue('database.mysqldb.port')) as number,
username: (await GenericHelpers.getConfigValue('database.mysqldb.user')) as string,
migrations: mysqlMigrations,
migrationsRun: true,
migrationsTableName: `${entityPrefix}migrations`,
timezone: 'Z', // set UTC as default
};
break;
case 'sqlite':
connectionOptions = {
type: 'sqlite',
database: path.join(n8nFolder, 'database.sqlite'),
entityPrefix,
migrations: sqliteMigrations,
migrationsRun: false, // migrations for sqlite will be ran manually for now; see below
migrationsTableName: `${entityPrefix}migrations`,
};
break;
default:
throw new Error(`The database "${dbType}" is currently not supported!`);
}
}
let loggingOption: LoggerOptions = (await GenericHelpers.getConfigValue(
'database.logging.enabled',
)) as boolean;
if (loggingOption) {
const optionsString = (
(await GenericHelpers.getConfigValue('database.logging.options')) as string
).replace(/\s+/g, '');
if (optionsString === 'all') {
loggingOption = optionsString;
} else {
loggingOption = optionsString.split(',') as LoggerOptions;
}
}
Object.assign(connectionOptions, {
entities: Object.values(entities),
synchronize: false,
logging: loggingOption,
maxQueryExecutionTime: (await GenericHelpers.getConfigValue(
'database.logging.maxQueryExecutionTime',
)) as string,
});
connection = await createConnection(connectionOptions);
if (!testConnectionOptions && dbType === 'sqlite') {
// This specific migration changes database metadata.
// A field is now nullable. We need to reconnect so that
// n8n knows it has changed. Happens only on sqlite.
let migrations = [];
try {
migrations = await connection.query(
`SELECT id FROM ${entityPrefix}migrations where name = "MakeStoppedAtNullable1607431743769"`,
);
} catch (error) {
// Migration table does not exist yet - it will be created after migrations run for the first time.
}
// If you remove this call, remember to turn back on the
// setting to run migrations automatically above.
await connection.runMigrations({
transaction: 'none',
});
// eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
if (migrations.length === 0) {
await connection.close();
connection = await createConnection(connectionOptions);
}
}
collections.Credentials = linkRepository(entities.CredentialsEntity);
collections.Execution = linkRepository(entities.ExecutionEntity);
collections.Workflow = linkRepository(entities.WorkflowEntity);
collections.Webhook = linkRepository(entities.WebhookEntity);
collections.Tag = linkRepository(entities.TagEntity);
collections.Role = linkRepository(entities.Role);
collections.User = linkRepository(entities.User);
collections.SharedCredentials = linkRepository(entities.SharedCredentials);
collections.SharedWorkflow = linkRepository(entities.SharedWorkflow);
collections.Settings = linkRepository(entities.Settings);
collections.InstalledPackages = linkRepository(entities.InstalledPackages);
collections.InstalledNodes = linkRepository(entities.InstalledNodes);
isInitialized = true;
return collections;
}