diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml old mode 100644 new mode 100755 index cb2e97ba..2c99cacc --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -180,25 +180,6 @@ services: - testing networks: - VE-network - -# vachan-demos: -# image: kavitha3797/vachan-demos:v2-0-0-alpha-11 -# expose: -# - 8002 -# command: uvicorn main:app --host 0.0.0.0 --port 8002 -# restart: always -# environment: -# - VACHAN_DOMAIN=${VACHAN_DOMAIN:-api.vachanengine.org} -# - VACHAN_LOGGING_LEVEL=INFO -# volumes: -# - logs-vol:/app/logs -# depends_on: -# - vachan-api -# profiles: -# - local-run -# - deployment -# networks: -# - VE-network vachan-cms-rest: image: vachanengine/vachan-cms-rest:v2.0.7 @@ -344,7 +325,6 @@ services: networks: - VE-network - vachan-ai: image: jayasankarkk/vachan-ai:0.0.0-alpha.6 healthcheck: @@ -407,7 +387,6 @@ services: - VE-network container_name: vachan-ai - worker: image: jayasankarkk/vachan-ai:0.0.0-alpha.6 healthcheck: @@ -444,7 +423,6 @@ services: - VE-network container_name: redis-worker - # Web Server web-server-local: image: nginx:latest @@ -496,7 +474,6 @@ services: networks: - VE-network - ofelia-scheduler: image: mcuadros/ofelia:v0.3.7 depends_on: @@ -578,6 +555,44 @@ services: networks: - VE-network + # SigNoz services (copied from signoz/deploy/docker-compose.yaml, networks section removed) + clickhouse: + image: clickhouse/clickhouse-server:24.1.2-alpine + container_name: signoz-clickhouse + volumes: + - ./signoz/data/clickhouse:/var/lib/clickhouse + - ./signoz/clickhouse-config.xml:/etc/clickhouse-server/config.xml:ro + - ./signoz/clickhouse-config.xml:/etc/clickhouse-server/config.xml + - ./signoz/clickhouse-users.xml:/etc/clickhouse-server/users.xml + - ./signoz/custom-function.xml:/etc/clickhouse-server/custom-function.xml + - ./signoz/user_scripts:/var/lib/clickhouse/user_scripts/ + ports: + - "9000:9000" # Adjust if needed to avoid conflicts + + alertmanager: + image: signoz/alertmanager:${ALERTMANAGER_TAG:-0.23.5} # Adjust version if needed + container_name: signoz-alertmanager + volumes: + - ./signoz/alertmanager.yml:/prometheus/alertmanager.yml:ro # Adjust path if needed + command: + - '--config.file=/prometheus/alertmanager.yml' + - '--storage.path=/data' + ports: + - "9093:9093" # Adjust if needed + + query-service: + image: signoz/query-service:${DOCKER_TAG:-0.39.0} # Adjust version if needed + command: + - "-config=/root/config/prometheus.yml" + ports: + - "8080:8080" # Adjust if needed + volumes: + - ./signoz/prometheus.yml:/root/config/prometheus.yml:ro # Adjust path if needed + - ../dashboards:/root/config/dashboards:ro # Adjust path if needed + - ./signoz/data/signoz/:/var/lib/signoz/ + depends_on: + - clickhouse + networks: VE-network: diff --git a/docker/signoz/alertmanager.yml b/docker/signoz/alertmanager.yml new file mode 100755 index 00000000..d69357f9 --- /dev/null +++ b/docker/signoz/alertmanager.yml @@ -0,0 +1,35 @@ +global: + resolve_timeout: 1m + slack_api_url: 'https://hooks.slack.com/services/xxx' + +route: + receiver: 'slack-notifications' + +receivers: +- name: 'slack-notifications' + slack_configs: + - channel: '#alerts' + send_resolved: true + icon_url: https://avatars3.githubusercontent.com/u/3380462 + title: |- + [{{ .Status | toUpper }}{{ if eq .Status "firing" }}:{{ .Alerts.Firing | len }}{{ end }}] {{ .CommonLabels.alertname }} for {{ .CommonLabels.job }} + {{- if gt (len .CommonLabels) (len .GroupLabels) -}} + {{" "}}( + {{- with .CommonLabels.Remove .GroupLabels.Names }} + {{- range $index, $label := .SortedPairs -}} + {{ if $index }}, {{ end }} + {{- $label.Name }}="{{ $label.Value -}}" + {{- end }} + {{- end -}} + ) + {{- end }} + text: >- + {{ range .Alerts -}} + *Alert:* {{ .Annotations.title }}{{ if .Labels.severity }} - `{{ .Labels.severity }}`{{ end }} + + *Description:* {{ .Annotations.description }} + + *Details:* + {{ range .Labels.SortedPairs }} • *{{ .Name }}:* `{{ .Value }}` + {{ end }} + {{ end }} \ No newline at end of file diff --git a/docker/signoz/alerts.yml b/docker/signoz/alerts.yml new file mode 100755 index 00000000..810a2075 --- /dev/null +++ b/docker/signoz/alerts.yml @@ -0,0 +1,11 @@ +groups: +- name: ExampleCPULoadGroup + rules: + - alert: HighCpuLoad + expr: system_cpu_load_average_1m > 0.1 + for: 0m + labels: + severity: warning + annotations: + summary: High CPU load + description: "CPU load is > 0.1\n VALUE = {{ $value }}\n LABELS = {{ $labels }}" diff --git a/docker/signoz/clickhouse-config.xml b/docker/signoz/clickhouse-config.xml new file mode 100755 index 00000000..f8213b65 --- /dev/null +++ b/docker/signoz/clickhouse-config.xml @@ -0,0 +1,1140 @@ + + + + + + information + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + + 1000M + 10 + + + + + + + + + + + + + + + + + + 8123 + + + 9000 + + + 9004 + + + 9005 + + + + + + + + + + + + 9009 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + 4096 + + + 3 + + + + + false + + + /path/to/ssl_cert_file + /path/to/ssl_key_file + + + false + + + /path/to/ssl_ca_cert_file + + + none + + + 0 + + + -1 + -1 + + + false + + + + + + + + + + + none + true + true + sslv2,sslv3 + true + + + + true + true + sslv2,sslv3 + true + + + + RejectCertificateHandler + + + + + + + + + 100 + + + 0 + + + + 10000 + + + + + + 0.9 + + + 4194304 + + + 0 + + + + + + 8589934592 + + + 5368709120 + + + + 1000 + + + 134217728 + + + 10000 + + + /var/lib/clickhouse/ + + + /var/lib/clickhouse/tmp/ + + + + ` + + + + + + /var/lib/clickhouse/user_files/ + + + + + + + + + + + + + users.xml + + + + /var/lib/clickhouse/access/ + + + + + + + default + + + + + + + + + + + + default + + + + + + + + + true + + + false + + ' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + apt install --no-install-recommends -f ./clickhouse-jdbc-bridge_$PKG_VER-1_all.deb + clickhouse-jdbc-bridge & + + * [CentOS/RHEL] + export MVN_URL=https://repo1.maven.org/maven2/ru/yandex/clickhouse/clickhouse-jdbc-bridge + export PKG_VER=$(curl -sL $MVN_URL/maven-metadata.xml | grep '' | sed -e 's|.*>\(.*\)<.*|\1|') + wget https://github.com/ClickHouse/clickhouse-jdbc-bridge/releases/download/v$PKG_VER/clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + yum localinstall -y clickhouse-jdbc-bridge-$PKG_VER-1.noarch.rpm + clickhouse-jdbc-bridge & + + Please refer to https://github.com/ClickHouse/clickhouse-jdbc-bridge#usage for more information. + ]]> + + + + + + + + + + + + + + + + + 3600 + + + + 3600 + + + 60 + + + + + + + + + + + + + system + query_log
+ + toYYYYMM(event_date) + + + + + + 7500 +
+ + + + system + trace_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + system + query_thread_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + query_views_log
+ toYYYYMM(event_date) + 7500 +
+ + + + system + part_log
+ toYYYYMM(event_date) + 7500 +
+ + + + + + system + metric_log
+ 7500 + 1000 +
+ + + + system + asynchronous_metric_log
+ + 7000 +
+ + + + + + engine MergeTree + partition by toYYYYMM(finish_date) + order by (finish_date, finish_time_us, trace_id) + + system + opentelemetry_span_log
+ 7500 +
+ + + + + system + crash_log
+ + + 1000 +
+ + + + + + + system + processors_profile_log
+ + toYYYYMM(event_date) + 7500 +
+ + + + + + + + + *_dictionary.xml + + + *function.xml + /var/lib/clickhouse/user_scripts/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + /clickhouse/task_queue/ddl + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + click_cost + any + + 0 + 3600 + + + 86400 + 60 + + + + max + + 0 + 60 + + + 3600 + 300 + + + 86400 + 3600 + + + + + + /var/lib/clickhouse/format_schemas/ + + + + + hide encrypt/decrypt arguments + ((?:aes_)?(?:encrypt|decrypt)(?:_mysql)?)\s*\(\s*(?:'(?:\\'|.)+'|.*?)\s*\) + + \1(???) + + + + + + + + + + false + + false + + + https://6f33034cfe684dd7a3ab9875e57b1c8d@o388870.ingest.sentry.io/5226277 + + + + + + + + + + + 268435456 + true + +
diff --git a/docker/signoz/clickhouse-users.xml b/docker/signoz/clickhouse-users.xml new file mode 100755 index 00000000..f1856207 --- /dev/null +++ b/docker/signoz/clickhouse-users.xml @@ -0,0 +1,123 @@ + + + + + + + + + + 10000000000 + + + random + + + + + 1 + + + + + + + + + + + + + ::/0 + + + + default + + + default + + + + + + + + + + + + + + 3600 + + + 0 + 0 + 0 + 0 + 0 + + + + diff --git a/docker/signoz/custom-function.xml b/docker/signoz/custom-function.xml new file mode 100755 index 00000000..b2b3f91a --- /dev/null +++ b/docker/signoz/custom-function.xml @@ -0,0 +1,21 @@ + + + executable + histogramQuantile + Float64 + + Array(Float64) + buckets + + + Array(Float64) + counts + + + Float64 + quantile + + CSV + ./histogramQuantile + + diff --git a/docker/signoz/data/alertmanager/.gitkeep b/docker/signoz/data/alertmanager/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/alertmanager/nflog b/docker/signoz/data/alertmanager/nflog new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/alertmanager/silences b/docker/signoz/data/alertmanager/silences new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/clickhouse-2/.gitkeep b/docker/signoz/data/clickhouse-2/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/clickhouse-3/.gitkeep b/docker/signoz/data/clickhouse-3/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/clickhouse/.gitkeep b/docker/signoz/data/clickhouse/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/signoz/.gitkeep b/docker/signoz/data/signoz/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/signoz/signoz.db b/docker/signoz/data/signoz/signoz.db new file mode 100755 index 00000000..aa6eb524 Binary files /dev/null and b/docker/signoz/data/signoz/signoz.db differ diff --git a/docker/signoz/data/zookeeper-1/.gitkeep b/docker/signoz/data/zookeeper-1/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/zookeeper-1/data/myid b/docker/signoz/data/zookeeper-1/data/myid new file mode 100755 index 00000000..d00491fd --- /dev/null +++ b/docker/signoz/data/zookeeper-1/data/myid @@ -0,0 +1 @@ +1 diff --git a/docker/signoz/data/zookeeper-1/data/version-2/snapshot.10b2 b/docker/signoz/data/zookeeper-1/data/version-2/snapshot.10b2 new file mode 100755 index 00000000..22d52b76 Binary files /dev/null and b/docker/signoz/data/zookeeper-1/data/version-2/snapshot.10b2 differ diff --git a/docker/signoz/data/zookeeper-1/data/version-2/snapshot.10dd b/docker/signoz/data/zookeeper-1/data/version-2/snapshot.10dd new file mode 100755 index 00000000..3c737132 Binary files /dev/null and b/docker/signoz/data/zookeeper-1/data/version-2/snapshot.10dd differ diff --git a/docker/signoz/data/zookeeper-1/data/version-2/snapshot.112b b/docker/signoz/data/zookeeper-1/data/version-2/snapshot.112b new file mode 100755 index 00000000..9dc80f5c Binary files /dev/null and b/docker/signoz/data/zookeeper-1/data/version-2/snapshot.112b differ diff --git a/docker/signoz/data/zookeeper-2/.gitkeep b/docker/signoz/data/zookeeper-2/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/data/zookeeper-3/.gitkeep b/docker/signoz/data/zookeeper-3/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker/signoz/prometheus.yml b/docker/signoz/prometheus.yml new file mode 100755 index 00000000..6a796ea1 --- /dev/null +++ b/docker/signoz/prometheus.yml @@ -0,0 +1,25 @@ +# my global config +global: + scrape_interval: 5s # Set the scrape interval to every 15 seconds. Default is every 1 minute. + evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute. + # scrape_timeout is set to the global default (10s). + +# Alertmanager configuration +alerting: + alertmanagers: + - static_configs: + - targets: + - alertmanager:9093 + +# Load rules once and periodically evaluate them according to the global 'evaluation_interval'. +rule_files: + # - "first_rules.yml" + # - "second_rules.yml" + - 'alerts.yml' + +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: [] + +remote_read: + - url: tcp://clickhouse:9000/?database=signoz_metrics diff --git a/docker/signoz/user_scripts/histogramQuantile b/docker/signoz/user_scripts/histogramQuantile new file mode 100755 index 00000000..3b77a7b2 Binary files /dev/null and b/docker/signoz/user_scripts/histogramQuantile differ diff --git a/docker/signoz/user_scripts/histogramQuantile.go b/docker/signoz/user_scripts/histogramQuantile.go new file mode 100755 index 00000000..9540a774 --- /dev/null +++ b/docker/signoz/user_scripts/histogramQuantile.go @@ -0,0 +1,237 @@ +package main + +import ( + "bufio" + "fmt" + "math" + "os" + "sort" + "strconv" + "strings" +) + +// NOTE: executable must be built with target OS and architecture set to linux/amd64 +// env GOOS=linux GOARCH=amd64 go build -o histogramQuantile histogramQuantile.go + +// The following code is adapted from the following source: +// https://github.com/prometheus/prometheus/blob/main/promql/quantile.go + +type bucket struct { + upperBound float64 + count float64 +} + +// buckets implements sort.Interface. +type buckets []bucket + +func (b buckets) Len() int { return len(b) } +func (b buckets) Swap(i, j int) { b[i], b[j] = b[j], b[i] } +func (b buckets) Less(i, j int) bool { return b[i].upperBound < b[j].upperBound } + +// bucketQuantile calculates the quantile 'q' based on the given buckets. The +// buckets will be sorted by upperBound by this function (i.e. no sorting +// needed before calling this function). The quantile value is interpolated +// assuming a linear distribution within a bucket. However, if the quantile +// falls into the highest bucket, the upper bound of the 2nd highest bucket is +// returned. A natural lower bound of 0 is assumed if the upper bound of the +// lowest bucket is greater 0. In that case, interpolation in the lowest bucket +// happens linearly between 0 and the upper bound of the lowest bucket. +// However, if the lowest bucket has an upper bound less or equal 0, this upper +// bound is returned if the quantile falls into the lowest bucket. +// +// There are a number of special cases (once we have a way to report errors +// happening during evaluations of AST functions, we should report those +// explicitly): +// +// If 'buckets' has 0 observations, NaN is returned. +// +// If 'buckets' has fewer than 2 elements, NaN is returned. +// +// If the highest bucket is not +Inf, NaN is returned. +// +// If q==NaN, NaN is returned. +// +// If q<0, -Inf is returned. +// +// If q>1, +Inf is returned. +func bucketQuantile(q float64, buckets buckets) float64 { + if math.IsNaN(q) { + return math.NaN() + } + if q < 0 { + return math.Inf(-1) + } + if q > 1 { + return math.Inf(+1) + } + sort.Sort(buckets) + if !math.IsInf(buckets[len(buckets)-1].upperBound, +1) { + return math.NaN() + } + + buckets = coalesceBuckets(buckets) + ensureMonotonic(buckets) + + if len(buckets) < 2 { + return math.NaN() + } + observations := buckets[len(buckets)-1].count + if observations == 0 { + return math.NaN() + } + rank := q * observations + b := sort.Search(len(buckets)-1, func(i int) bool { return buckets[i].count >= rank }) + + if b == len(buckets)-1 { + return buckets[len(buckets)-2].upperBound + } + if b == 0 && buckets[0].upperBound <= 0 { + return buckets[0].upperBound + } + var ( + bucketStart float64 + bucketEnd = buckets[b].upperBound + count = buckets[b].count + ) + if b > 0 { + bucketStart = buckets[b-1].upperBound + count -= buckets[b-1].count + rank -= buckets[b-1].count + } + return bucketStart + (bucketEnd-bucketStart)*(rank/count) +} + +// coalesceBuckets merges buckets with the same upper bound. +// +// The input buckets must be sorted. +func coalesceBuckets(buckets buckets) buckets { + last := buckets[0] + i := 0 + for _, b := range buckets[1:] { + if b.upperBound == last.upperBound { + last.count += b.count + } else { + buckets[i] = last + last = b + i++ + } + } + buckets[i] = last + return buckets[:i+1] +} + +// The assumption that bucket counts increase monotonically with increasing +// upperBound may be violated during: +// +// * Recording rule evaluation of histogram_quantile, especially when rate() +// has been applied to the underlying bucket timeseries. +// * Evaluation of histogram_quantile computed over federated bucket +// timeseries, especially when rate() has been applied. +// +// This is because scraped data is not made available to rule evaluation or +// federation atomically, so some buckets are computed with data from the +// most recent scrapes, but the other buckets are missing data from the most +// recent scrape. +// +// Monotonicity is usually guaranteed because if a bucket with upper bound +// u1 has count c1, then any bucket with a higher upper bound u > u1 must +// have counted all c1 observations and perhaps more, so that c >= c1. +// +// Randomly interspersed partial sampling breaks that guarantee, and rate() +// exacerbates it. Specifically, suppose bucket le=1000 has a count of 10 from +// 4 samples but the bucket with le=2000 has a count of 7 from 3 samples. The +// monotonicity is broken. It is exacerbated by rate() because under normal +// operation, cumulative counting of buckets will cause the bucket counts to +// diverge such that small differences from missing samples are not a problem. +// rate() removes this divergence.) +// +// bucketQuantile depends on that monotonicity to do a binary search for the +// bucket with the φ-quantile count, so breaking the monotonicity +// guarantee causes bucketQuantile() to return undefined (nonsense) results. +// +// As a somewhat hacky solution until ingestion is atomic per scrape, we +// calculate the "envelope" of the histogram buckets, essentially removing +// any decreases in the count between successive buckets. + +func ensureMonotonic(buckets buckets) { + max := buckets[0].count + for i := 1; i < len(buckets); i++ { + switch { + case buckets[i].count > max: + max = buckets[i].count + case buckets[i].count < max: + buckets[i].count = max + } + } +} + +// End of copied code. + +func readLines() []string { + r := bufio.NewReader(os.Stdin) + bytes := []byte{} + lines := []string{} + for { + line, isPrefix, err := r.ReadLine() + if err != nil { + break + } + bytes = append(bytes, line...) + if !isPrefix { + str := strings.TrimSpace(string(bytes)) + if len(str) > 0 { + lines = append(lines, str) + bytes = []byte{} + } + } + } + if len(bytes) > 0 { + lines = append(lines, string(bytes)) + } + return lines +} + +func main() { + lines := readLines() + for _, text := range lines { + // Example input + // "[1, 2, 4, 8, 16]", "[1, 5, 8, 10, 14]", 0.9" + // bounds - counts - quantile + parts := strings.Split(text, "\",") + + var bucketNumbers []float64 + // Strip the ends with square brackets + text = parts[0][2 : len(parts[0])-1] + // Parse the bucket bounds + for _, num := range strings.Split(text, ",") { + num = strings.TrimSpace(num) + number, err := strconv.ParseFloat(num, 64) + if err == nil { + bucketNumbers = append(bucketNumbers, number) + } + } + + var bucketCounts []float64 + // Strip the ends with square brackets + text = parts[1][2 : len(parts[1])-1] + // Parse the bucket counts + for _, num := range strings.Split(text, ",") { + num = strings.TrimSpace(num) + number, err := strconv.ParseFloat(num, 64) + if err == nil { + bucketCounts = append(bucketCounts, number) + } + } + + // Parse the quantile + q, err := strconv.ParseFloat(parts[2], 64) + var b buckets + + if err == nil { + for i := 0; i < len(bucketNumbers); i++ { + b = append(b, bucket{upperBound: bucketNumbers[i], count: bucketCounts[i]}) + } + } + fmt.Println(bucketQuantile(q, b)) + } +}