Remote Write 1.1: e2e benchmarks (#13102)

* Remote Write e2e benchmarks

Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>

* Prometheus ports automatically assigned

Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>

* make dashboard editable + more modular to different job label values

Signed-off-by: Callum Styan <callumstyan@gmail.com>

* Dashboard improvements

* memory stats
* diffs look at counter increases

Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>

* run script: absolute path for config templates

Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>

* grafana dashboard improvements

* show actual values of metrics
* add memory stats and diff

Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>

* dashboard changes

Signed-off-by: Callum Styan <callumstyan@gmail.com>

---------

Signed-off-by: Nicolás Pazos <npazosmendez@gmail.com>
Signed-off-by: Callum Styan <callumstyan@gmail.com>
Co-authored-by: Callum Styan <callumstyan@gmail.com>

scripts/remotewrite11-bench/.gitignore

@@ -0,0 +1,3 @@
+tsdb/
+prometheus
+prometheus.old

scripts/remotewrite11-bench/README.md

@@ -0,0 +1,46 @@
+# e2e Remote Write benchmark
+
+This benchmark's purpose is to compare different versions of the remote write
+protocol. It runs multiple pairs of sender=>receiver prometheus instances and
+has the senders scrape an entire port-forwarded remote kubernetes namepsace.
+
+
+## Run
+
+1. Set envvars to port forward pods:
+
+    ```
+    export CONTEXT=my-k8-context
+    export NAMESPACE=my-namespace
+
+    ```
+    If desired, tweak the INSTANCES variable in the `run.sh` script.
+
+2. Run
+
+```
+./run.sh
+```
+
+## Profiles
+
+```
+go tool pprof -seconds=240 http://localhost:9095/debug/pprof/profile
+go tool pprof -seconds=240 http://localhost:9094/debug/pprof/profile
+```
+
+## Stats
+
+# Grafana instance with provisiones datasource and dashboard
+```
+docker run --network host -v ${PWD}/local_grafana/:/etc/grafana/provisioning  --env GF_AUTH_ANONYMOUS_ENABLED=true --env GF_AUTH_ANONYMOUS_ORG_ROLE=Admin  --env GF_AUTH_BASIC_ENABLED=false --env ORG_ID=123 grafana/grafana:latest
+```
+
+
+```
+http://localhost:9095/graph?g0.expr=sum%20by%20(job)%20(process_cpu_seconds_total%7Bjob%3D~%22(sender%7Creceiver).%2B%22%2C%7D)&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=15m
+
+http://localhost:9095/graph?g0.expr=sum%20by%20(job)%20(prometheus_remote_storage_bytes_total%7Bjob%3D~%22(sender%7Creceiver).%2B%22%2C%7D)&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=15m
+
+http://localhost:9095/graph?g0.expr=sum%20by%20(job)%20(go_memstats_alloc_bytes_total%7Bjob%3D~%22(sender%7Creceiver).%2B%22%2C%7D)&g0.tab=0&g0.stacked=0&g0.show_exemplars=0&g0.range_input=15m
+```

scripts/remotewrite11-bench/local_grafana/dashboards/dashboards.yaml

@@ -0,0 +1,24 @@
+apiVersion: 1
+
+providers:
+  # <string> an unique provider name. Required
+  - name: 'a unique provider name'
+    # <int> Org id. Default to 1
+    orgId: 1
+    # <string> name of the dashboard folder.
+    folder: ''
+    # <string> folder UID. will be automatically generated if not specified
+    folderUid: ''
+    # <string> provider type. Default to 'file'
+    type: file
+    # <bool> disable dashboard deletion
+    disableDeletion: false
+    # <int> how often Grafana will scan for changed dashboards
+    updateIntervalSeconds: 10
+    # <bool> allow updating provisioned dashboards from the UI
+    allowUiUpdates: true
+    options:
+      # <string, required> path to dashboard files on disk. Required when using the 'file' type
+      path: /etc/grafana/provisioning/dashboards
+      # <bool> use folder names from filesystem to create folders in Grafana
+      foldersFromFilesStructure: true

scripts/remotewrite11-bench/local_grafana/datasources/grafana_datasource.yaml

@@ -0,0 +1,16 @@
+# Configuration file version
+apiVersion: 1
+
+deleteDatasources:
+  - name: Prometheus
+    orgId: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    orgId: 1
+
+    url: http://localhost:9093
+    isDefault: true
+    version: 1
+    editable: true

scripts/remotewrite11-bench/receiver-template.yml

@@ -0,0 +1,3 @@
+global:
+  external_labels:
+    role: ${RECEIVER_NAME}

scripts/remotewrite11-bench/run.sh

@@ -0,0 +1,90 @@
+#!/bin/sh
+set -e
+trap "trap - SIGTERM && kill -- -$$" SIGINT SIGTERM EXIT
+
+# CONFIGURABLES
+# ~~~~~~~~~~~~~
+declare -a INSTANCES
+# (sender,receiver) pairs to run: (sender_name; sender_flags; receiver_name; receiver_flags)
+INSTANCES+=('sender-v1;;receiver-v1;')
+INSTANCES+=('sender-v11;--enable-feature rw-1-1-sender;receiver-v11;--enable-feature rw-1-1-receiver')
+
+
+# ~~~~~~~~~~~~~
+
+# append two ports to all instances
+PROM_PORT=9090
+for i in "${!INSTANCES[@]}"; do
+  INSTANCES[$i]="${INSTANCES[$i]};$PROM_PORT"
+  PROM_PORT=$((PROM_PORT+1))
+  INSTANCES[$i]="${INSTANCES[$i]};$PROM_PORT"
+  PROM_PORT=$((PROM_PORT+1))
+done
+
+# Check if all required variables are set
+: "${CONTEXT:?}"
+: "${NAMESPACE:?}"
+
+BASE_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
+
+TEMP_DIR=$(mktemp -d)
+echo "Working on dir: $TEMP_DIR"
+SCRAPE_CONFIGS=""
+declare -a COMMANDS
+PORT=40000
+# Get all pods from namespace, create scrape configs and port forward them
+while read pod ; do
+  COMMANDS+=("kubectl --context $CONTEXT -n $NAMESPACE port-forward $pod $PORT:http-metrics")
+  SCRAPE_CONFIGS="${SCRAPE_CONFIGS}
+  - job_name: '$pod'
+    static_configs:
+      - targets: ['localhost:$PORT']"
+
+  PORT=$((PORT+1))
+done < <(kubectl --context $CONTEXT -n $NAMESPACE get pods | awk '{print $1}' | tail +2 )
+
+
+# Run all prometheus instances and add them to the scrape configs
+for instance in "${INSTANCES[@]}"
+do
+  IFS=";" read -r -a arr <<< "${instance}"
+
+  sender="${arr[0]}"
+  sender_flags="${arr[1]}"
+  receiver="${arr[2]}"
+  receiver_flags="${arr[3]}"
+  sender_port="${arr[4]}"
+  receiver_port="${arr[5]}"
+
+  SCRAPE_CONFIGS="${SCRAPE_CONFIGS}
+  - job_name: '$sender'
+    static_configs:
+      - targets: ['localhost:$sender_port']
+  - job_name: '$receiver'
+    static_configs:
+      - targets: ['localhost:$receiver_port']"
+
+  COMMANDS+=("./prometheus --config.file=$TEMP_DIR/$sender.yml --web.listen-address=0.0.0.0:$sender_port --storage.tsdb.path=$TEMP_DIR/tsdb/$sender/data/ $sender_flags 2>&1 | awk '{print \"[$sender]\",\$0}'")
+  COMMANDS+=("./prometheus --config.file=$TEMP_DIR/$receiver.yml --web.listen-address=0.0.0.0:$receiver_port --storage.tsdb.path=$TEMP_DIR/tsdb/$receiver/data/ $receiver_flags --web.enable-remote-write-receiver 2>&1 | awk '{print \"[$receiver]\",\$0}'")
+done
+
+# Create the yml configs for the prometheus instances
+for instance in "${INSTANCES[@]}"
+do
+  IFS=";" read -r -a arr <<< "${instance}"
+  export SENDER_NAME="${arr[0]}"
+  export RECEIVER_NAME="${arr[2]}"
+  export REMOTE_WRITE_PORT="${arr[5]}"
+  export SCRAPE_CONFIGS="$SCRAPE_CONFIGS"
+  envsubst < "$BASE_DIR/receiver-template.yml" > "$TEMP_DIR/$RECEIVER_NAME.yml"
+  envsubst < "$BASE_DIR/sender-template.yml" > "$TEMP_DIR/$SENDER_NAME.yml"
+done
+
+# Actually run all commands
+for cmd in "${COMMANDS[@]}"
+do
+  eval $cmd &
+done
+
+echo Running...
+read -r -d '' _ </dev/tty

scripts/remotewrite11-bench/sender-template.yml

@@ -0,0 +1,13 @@
+global:
+  scrape_interval: 15s
+  external_labels:
+    role: ${SENDER_NAME}
+
+remote_write:
+    - url: 'http://localhost:${REMOTE_WRITE_PORT}/api/v1/write'
+      name: ${RECEIVER_NAME}
+      metadata_config:
+        send: false
+
+scrape_configs:
+${SCRAPE_CONFIGS}