build: Migrate from lerna to turborepo (#3796)

* feat: Added turborepo setup.

* feat: Updated nodes and credentials loading.

* feat: Removed remaining lerna references.

* fix: Updated npm run bootstrap command.

* feat: Added CI step for installing latest npm.

* chore: Removed lerna config.

* feat: Added gulp to global ci packages.

* fix: Set node as moduleResolution target in workflow. Fixed CI.

* fix: Added turborepo installation to the n8n-custom docker image.

* fix: Updated copied docker files for n8n-custom.

* fix: Added git as dependency in n8n-custom dockerfile.

* fix: Changed npm install command in n8n-custom.

* 📦 Update `package-lock.json`

Co-authored-by: Iván Ovejero <ivov.src@gmail.com>

.github/workflows/test-workflows.yml

@@ -45,8 +45,8 @@ jobs:
         name: npm install and build
         run: |
           cd n8n
+          npm install -g npm@latest
           npm install
-          npm run bootstrap
           npm run build --if-present
         env:
           CI: true

.github/workflows/tests.yml

@@ -21,8 +21,8 @@ jobs:
         node-version: ${{ matrix.node-version }}
     - name: npm install, build, and test
       run: |
+        npm install -g npm@latest
         npm install
-        npm run bootstrap
         npm run build --if-present
         npm test
         npm run lint

.gitignore

@@ -4,7 +4,6 @@ node_modules
 tmp
 dist
 npm-debug.log*
-lerna-debug.log
 yarn.lock
 google-generated-credentials.json
 _START_PACKAGE
@@ -15,3 +14,4 @@ _START_PACKAGE
 .idea
 nodelinter.config.json
 packages/*/package-lock.json
+packages/*/.turbo

CONTRIBUTING.md

@@ -33,7 +33,7 @@ The most important directories:
  - [/packages/cli](/packages/cli) - CLI code to run front- & backend
  - [/packages/core](/packages/core) - Core code which handles workflow
                                       execution, active webhooks and
-                                      workflows. **Contact n8n before 
+                                      workflows. **Contact n8n before
 									  starting on any changes here**
  - [/packages/design-system](/packages/design-system) - Vue frontend components
  - [/packages/editor-ui](/packages/editor-ui) - Vue frontend workflow editor
@@ -72,19 +72,13 @@ Windows:
 npm install -g windows-build-tools
 ```
 
-#### lerna
+#### npm workspaces
 
 n8n is split up in different modules which are all in a single mono repository.
-To facilitate those modules management, [lerna](https://lerna.js.org) gets
-used. It automatically sets up file-links between modules which depend on each
+To facilitate the module management, [npm workspaces](https://docs.npmjs.com/cli/v7/using-npm/workspaces) are
+used. This automatically sets up file-links between modules which depend on each
 other.
 
-So for the setup to work correctly lerna has to be installed globally like this:
-
-```
-npm install -g lerna
-```
-
 ### Actual n8n setup
 
 > **IMPORTANT**: All the steps below have to get executed at least once to get the development setup up and running!
@@ -94,27 +88,27 @@ checked out and set up:
 
 1. [Fork](https://guides.github.com/activities/forking/#fork) the n8n repository
 
-1. Clone your forked repository
+2. Clone your forked repository
 	```
 	git clone https://github.com/<your_github_username>/n8n.git
 	```
 
-1. Add the original n8n repository as `upstream` to your forked repository
+3. Add the original n8n repository as `upstream` to your forked repository
     ```
 	git remote add upstream https://github.com/n8n-io/n8n.git
 	```
 
-1. Go into repository folder
+4. Go into repository folder
 	```
 	cd n8n
 	```
 
-1. Install all dependencies of all modules and link them together:
+5. Install all dependencies of all modules and link them together:
 	```
-	lerna bootstrap --hoist
+	npm install
 	```
 
-1. Build all the code:
+6. Build all the code:
 	```
 	npm run build
 	```

docker/images/n8n-custom/Dockerfile

@@ -5,12 +5,12 @@ FROM node:16-alpine as builder
 USER root
 
 # Install all needed dependencies
-RUN apk --update add --virtual build-dependencies python3 build-base ca-certificates && \
-	npm_config_user=root npm install -g lerna run-script-os
+RUN apk --update add --virtual build-dependencies python3 build-base ca-certificates git && \
+	npm_config_user=root npm install -g npm@latest run-script-os turbo
 
 WORKDIR /data
 
-COPY lerna.json .
+COPY turbo.json .
 COPY package.json .
 COPY package-lock.json .
 COPY packages/cli/ ./packages/cli/
@@ -22,8 +22,7 @@ COPY packages/workflow/ ./packages/workflow/
 RUN rm -rf node_modules packages/*/node_modules packages/*/dist
 
 RUN npm config set legacy-peer-deps true
-RUN npm install --production --loglevel notice
-RUN lerna bootstrap --hoist -- --production
+RUN npm install --loglevel notice
 RUN npm run build
 
 
@@ -37,7 +36,7 @@ RUN apk add --update graphicsmagick tzdata tini su-exec git
 WORKDIR /data
 
 # Install all needed dependencies
-RUN npm_config_user=root npm install -g full-icu
+RUN npm_config_user=root npm install -g npm@latest full-icu
 
 # Install fonts
 RUN apk --no-cache add --virtual fonts msttcorefonts-installer fontconfig && \

docker/images/n8n-debian/Dockerfile

@@ -11,7 +11,7 @@ RUN \
 # Set a custom user to not have n8n run as root
 USER root
 
-RUN npm_config_user=root npm install -g full-icu n8n@${N8N_VERSION}
+RUN npm_config_user=root npm install -g npm@latest full-icu n8n@${N8N_VERSION}
 
 ENV NODE_ICU_DATA /usr/local/lib/node_modules/full-icu
 

docker/images/n8n-rhel7/Dockerfile

@@ -16,7 +16,7 @@ RUN \
 # Set a custom user to not have n8n run as root
 USER root
 
-RUN npm_config_user=root npm install -g n8n@${N8N_VERSION}
+RUN npm_config_user=root npm install -g npm@latest n8n@${N8N_VERSION}
 
 WORKDIR /data
 

docker/images/n8n/Dockerfile

@@ -14,7 +14,7 @@ USER root
 # it needs to build it correctly.
 RUN apk --update add --virtual build-dependencies python3 build-base ca-certificates && \
 	npm config set python "$(which python3)" && \
-	npm_config_user=root npm install -g full-icu n8n@${N8N_VERSION} && \
+	npm_config_user=root npm install -g npm@latest full-icu n8n@${N8N_VERSION} && \
 	apk del build-dependencies \
 	&& rm -rf /root /tmp/* /var/cache/apk/* && mkdir /root;
 

lerna.json

@@ -1,6 +0,0 @@
-{
-    "packages": [
-        "packages/*"
-    ],
-    "version": "independent"
-}

package.json

@@ -4,26 +4,28 @@
   "private": true,
   "homepage": "https://n8n.io",
   "scripts": {
-    "bootstrap": "lerna bootstrap --hoist --no-ci",
-    "build": "lerna exec npm run build",
-    "dev": "lerna exec npm run dev --parallel",
-    "clean:dist": "lerna exec -- rimraf ./dist",
-    "format": "lerna exec npm run format",
-    "lint": "lerna exec npm run lint",
-    "lintfix": "lerna exec npm run lintfix",
+    "build": "turbo run build",
+    "dev": "turbo run dev --parallel",
+    "clean:dist": "npm exec -ws -- rimraf ./dist",
+    "format": "turbo run format",
+    "lint": "turbo run lint",
+    "lintfix": "turbo run lintfix",
     "optimize-svg": "find ./packages -name '*.svg' ! -name 'pipedrive.svg' -print0 | xargs -0 -P16 -L20 npx svgo",
     "start": "run-script-os",
     "start:default": "cd packages/cli/bin && ./n8n",
     "start:windows": "cd packages/cli/bin && n8n",
-    "test": "lerna run test",
-    "watch": "lerna run --parallel watch",
+    "test": "turbo run test",
+    "watch": "turbo run watch",
     "webhook": "./packages/cli/bin/n8n webhook",
     "worker": "./packages/cli/bin/n8n worker"
   },
   "devDependencies": {
-    "lerna": "^3.13.1",
     "rimraf": "^3.0.2",
-    "run-script-os": "^1.0.7"
+    "run-script-os": "^1.0.7",
+    "turbo": "1.2.15"
   },
-  "postcss": {}
+  "postcss": {},
+  "workspaces": [
+    "packages/*"
+  ]
 }

packages/cli/src/LoadNodesAndCredentials.ts

@@ -95,6 +95,9 @@ class LoadNodesAndCredentialsClass {
 			// In case "n8n" package is the root and the packages are
 			// in the "node_modules" folder underneath it.
 			path.join(__dirname, '..', '..', 'node_modules', 'n8n-workflow'),
+			// In case "n8n" package is installed using npm/yarn workspaces
+			// the node_modules folder is in the root of the workspace.
+			path.join(__dirname, '..', '..', '..', '..', 'node_modules', 'n8n-workflow'),
 		];
 		for (const checkPath of checkPaths) {
 			try {

packages/editor-ui/package.json

@@ -94,7 +94,7 @@
     "uuid": "^8.3.2",
     "vue": "~2.6.11",
     "vue-agile": "^2.0.0",
-    "vue-cli-plugin-webpack-bundle-analyzer": "^2.0.0",
+    "vue-cli-plugin-webpack-bundle-analyzer": "^4.0.0",
     "vue-json-pretty": "1.7.1",
     "vue-prism-editor": "^0.3.0",
     "vue-router": "^3.0.6",

packages/workflow/tsconfig.json

@@ -19,6 +19,7 @@
 		"declaration": true,
 		"outDir": "./dist/",
 		"target": "es2019",
+		"moduleResolution": "node",
 		"sourceMap": true,
 		"useUnknownInCatchVariables": false,
 	},

turbo.json

@@ -0,0 +1,16 @@
+{
+	"$schema": "https://turborepo.org/schema.json",
+	"pipeline": {
+		"build": {
+			"dependsOn": ["^build"]
+		},
+		"format": {},
+		"lint": {},
+		"lintfix": {},
+		"test": {},
+		"watch": {},
+		"dev": {
+			"cache": false
+		}
+	}
+}