From 19f2c774ffd1e5fec08e305cc450b98eb87505f7 Mon Sep 17 00:00:00 2001 From: Jinsong Wang Date: Tue, 2 Apr 2024 10:45:09 -0700 Subject: [PATCH 01/13] Collect metrics from traceloop instrument --- ai/build.gradle | 2 + ai/config/config.yaml | 2 + .../java/com/instana/dc/ai/LLMDcUtil.java | 2 + .../com/instana/dc/ai/impl/llm/LLMDc.java | 215 ++++++++---------- .../ai/impl/llm/MetricsCollectorService.java | 186 +++++++++++++++ 5 files changed, 290 insertions(+), 117 deletions(-) create mode 100644 ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java diff --git a/ai/build.gradle b/ai/build.gradle index 799a582..d071e06 100644 --- a/ai/build.gradle +++ b/ai/build.gradle @@ -25,6 +25,8 @@ dependencies { implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-yaml:2.16.0-rc1") implementation("com.google.protobuf:protobuf-java:3.23.4") implementation("com.google.protobuf:protobuf-java-util:3.23.4") + implementation("com.linecorp.armeria:armeria:1.27.3") + implementation("com.linecorp.armeria:armeria-grpc:1.27.3") implementation(files("libs/otel-dc-0.9.7.jar")) testImplementation("org.junit.jupiter:junit-jupiter-api:5.8.1") diff --git a/ai/config/config.yaml b/ai/config/config.yaml index ce5ca49..2c5dd46 100644 --- a/ai/config/config.yaml +++ b/ai/config/config.yaml @@ -7,3 +7,5 @@ server.scheme: https instances: - otel.backend.url: http://localhost:4317 otel.service.name: WatsonxDC + price.prompt.tokens.per.kilo: 0.3 + price.complete.tokens.per.kilo: 0.3 diff --git a/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java b/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java index f5db0d5..d248020 100644 --- a/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java +++ b/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java @@ -19,6 +19,8 @@ public class LLMDcUtil { public static final String DEFAULT_INSTRUMENTATION_SCOPE_VER = "1.0.0"; public static final String SERVICE_NAME = "service.name"; public static final String SERVICE_INSTANCE_ID = "service.instance.id"; + public final static String PRICE_PROMPT_TOKES_PER_KILO = "price.prompt.tokens.per.kilo"; + public final static String PRICE_COMPLETE_TOKES_PER_KILO = "price.complete.tokens.per.kilo"; /* Configurations for Metrics: */ diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index 61cf1d7..b17fa28 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -6,51 +6,48 @@ import com.instana.dc.ai.AbstractLLMDc; import com.instana.dc.ai.DataCollector.CustomDcConfig; +import com.instana.dc.ai.impl.llm.MetricsCollectorService.OtelMetric; import java.util.logging.Logger; import java.util.*; -import java.io.IOException; -import io.opentelemetry.proto.common.v1.KeyValue; -import io.opentelemetry.proto.metrics.v1.ResourceMetrics; -import io.opentelemetry.proto.metrics.v1.MetricsData; -import io.opentelemetry.proto.metrics.v1.NumberDataPoint; -import io.opentelemetry.proto.metrics.v1.ScopeMetrics; -import io.opentelemetry.proto.metrics.v1.Sum; -import io.opentelemetry.proto.metrics.v1.Metric; - -import com.sun.net.httpserver.HttpServer; -import com.sun.net.httpserver.HttpHandler; -import com.sun.net.httpserver.HttpExchange; - -import java.io.InputStream; -import java.io.OutputStream; -import java.net.InetSocketAddress; +import com.linecorp.armeria.common.HttpData; +import com.linecorp.armeria.common.HttpResponse; +import com.linecorp.armeria.common.HttpStatus; +import com.linecorp.armeria.common.MediaType; +import com.linecorp.armeria.server.Server; +import com.linecorp.armeria.server.grpc.GrpcService; +import com.linecorp.armeria.server.healthcheck.HealthCheckService; //import static com.instana.agent.sensorsdk.semconv.SemanticAttributes.*; import static com.instana.dc.ai.LLMDcUtil.*; +@SuppressWarnings("null") public class LLMDc extends AbstractLLMDc { private static final Logger logger = Logger.getLogger(LLMDc.class.getName()); public static final String SENSOR_NAME = "com.instana.plugin.watsonx"; - private static List payloadList = new ArrayList<>(); private HashMap modelAggrMap = new HashMap<>(); + private MetricsCollectorService metricsCollector = new MetricsCollectorService(); + private Double pricePromptTokens = 0.0; + private Double priceCompleteTokens = 0.0; /** * The poll rate in the configuration, in seconds. In other words, the number of * seconds between calls to Watsonx. - */ - + */ + private class ModelAggregation { private final String modelId; private final String userId; - private int totalTokens; - private double totalCost; + private int totalPromptTokens; + private int totalCompleteTokens; private int totalDuration; private int maxDuration; private int totalReqCount; - + private int lastTotalPromptTokens; + private int lastTotalCompleteTokens; + public ModelAggregation(String modelId, String userId) { this.modelId = modelId; this.userId = userId; @@ -61,17 +58,28 @@ public String getModelId() { public String getUserId() { return userId; } - public void addTotalTokens(int tokens) { - totalTokens += tokens; + public void addTotalPromptTokens(int currTokens) { + int deltaPromptTokens = 0; + if(currTokens > lastTotalPromptTokens && lastTotalPromptTokens != 0) { + deltaPromptTokens = currTokens - lastTotalPromptTokens; + } + lastTotalPromptTokens = currTokens; + totalPromptTokens += deltaPromptTokens; } - public int getTotalTokens() { - return totalTokens; + + public int getTotalPromptTokens() { + return totalPromptTokens; } - public void addTotalCost(double cost) { - totalCost += cost; + public void addTotalCompleteTokens(int currTokens) { + int deltaCompleteTokens = 0; + if(currTokens > lastTotalCompleteTokens && lastTotalCompleteTokens != 0) { + deltaCompleteTokens = currTokens - lastTotalCompleteTokens; + } + lastTotalCompleteTokens = currTokens; + totalCompleteTokens += deltaCompleteTokens; } - public double getTotalCost() { - return totalCost; + public int getTotalCompleteTokens() { + return totalCompleteTokens; } public void addTotalDuration(int duration) { totalDuration += duration; @@ -91,48 +99,46 @@ public int getReqCount() { return totalReqCount; } public void resetMetrics() { - totalTokens = 0; - totalCost = 0; + totalPromptTokens = 0; + totalCompleteTokens = 0; totalDuration = 0; maxDuration = 0; totalReqCount = 0; } } - static class PostHandler implements HttpHandler { - @Override - public void handle(HttpExchange exchange) throws IOException { - if ("POST".equals(exchange.getRequestMethod())) { - InputStream requestBody = exchange.getRequestBody(); - byte[] payload = requestBody.readAllBytes(); - payloadList.add(payload); - - String response = "OK"; - exchange.sendResponseHeaders(200, response.length()); - OutputStream os = exchange.getResponseBody(); - os.write(response.getBytes()); - // os.close(); - } else { - exchange.sendResponseHeaders(405, -1); - } - } - } - public LLMDc(Map properties, CustomDcConfig cdcConfig) throws Exception { super(properties, cdcConfig); + pricePromptTokens = (Double) properties.getOrDefault(PRICE_PROMPT_TOKES_PER_KILO, 0.03); + priceCompleteTokens = (Double) properties.getOrDefault(PRICE_COMPLETE_TOKES_PER_KILO, 0.03); } @Override public void initOnce() throws ClassNotFoundException { - try { - HttpServer server = HttpServer.create(new InetSocketAddress(8000), 0); - server.createContext("/v1/metrics", new PostHandler()); - server.setExecutor(null); - server.start(); - } catch (Exception e) { - e.printStackTrace(); - } - logger.info("Server is running on port 8000"); + var server = Server.builder() + .http(8000) + .service( + GrpcService.builder() + .addService(metricsCollector) + .build()) + .service( + "/", + (ctx, req) -> { + var requests = metricsCollector.getMetrics(); + if (requests != null) { + return HttpResponse.of( + HttpStatus.OK, MediaType.JSON, HttpData.wrap("OK".getBytes())); + } else { + return HttpResponse.of( + HttpStatus.BAD_REQUEST, MediaType.JSON, + HttpData.wrap("Bad Request".getBytes())); + } + }) + .service("/health", HealthCheckService.of()) + .build(); + + server.start().join(); + Runtime.getRuntime().addShutdownHook(new Thread(() -> server.stop().join())); } @Override @@ -149,60 +155,30 @@ public void collectData() { aggr.resetMetrics(); } - for (byte[] payload : payloadList) { + List otelMetrics = metricsCollector.getMetrics(); + metricsCollector.clearMetrics(); + for (OtelMetric metric : otelMetrics) { try { - MetricsData md = MetricsData.parseFrom(payload); - - List rmList = md.getResourceMetricsList(); - for ( ResourceMetrics rm : rmList ) { - List smList = rm.getScopeMetricsList(); - for ( ScopeMetrics sm : smList ) { - List mList = sm.getMetricsList(); - for ( Metric m : mList ) { - String platform = ""; - String modelId = ""; - String userId = "llmUser"; - long asInt = 0; - - String name = m.getName(); - Sum s = m.getSum(); - List ndpList = s.getDataPointsList(); - for ( NumberDataPoint ndp : ndpList ) { - List kvList = ndp.getAttributesList(); - for ( KeyValue kv : kvList ) { - String key = kv.getKey(); - String val = kv.getValue().getStringValue(); - if (key.compareTo("model_id") == 0) { - modelId = val; - } else if (key.compareTo("llm_platform") == 0) { - platform = val; - } - } - asInt = ndp.getAsInt(); - } - - ModelAggregation modelAggr = modelAggrMap.get(modelId); - if(modelAggr == null) { - modelAggr = new ModelAggregation(modelId, userId); - modelAggrMap.put(modelId, modelAggr); - } - - if (name.compareTo("llm.usage.total_tokens") == 0) { - modelAggr.addTotalTokens((int)asInt); - } else if (name.compareTo("llm.request.count") == 0) { - modelAggr.addReqCount((int)asInt); - } else if (name.compareTo("llm.response.duration") == 0) { - long ms = (long)asInt/1000000; - modelAggr.addTotalDuration((int)ms); - } - } - } - } - } catch (com.google.protobuf.InvalidProtocolBufferException e) { + String modelId = metric.getModelId(); + long promptTokens = metric.getPromtTokens(); + long completeTokens = metric.getCompleteTokens(); + double duration = metric.getDuration(); + + ModelAggregation modelAggr = modelAggrMap.get(modelId); + if (modelAggr == null) { + modelAggr = new ModelAggregation(modelId, "llmUser"); + modelAggrMap.put(modelId, modelAggr); + } + + modelAggr.addTotalPromptTokens((int)(promptTokens)); + modelAggr.addTotalCompleteTokens((int)(completeTokens)); + modelAggr.addReqCount(1); + modelAggr.addTotalDuration((int)(duration*1000)); + + } catch (Exception e) { e.printStackTrace(); } } - payloadList.clear(); getRawMetric(LLM_STATUS_NAME).setValue(1); for(Map.Entry entry : modelAggrMap.entrySet()){ @@ -210,11 +186,16 @@ public void collectData() { int requestCount = aggr.getReqCount(); int totalDuration = aggr.getTotalDuration(); int maxDuration = aggr.getMaxDuration(); - int totalTokens = aggr.getTotalTokens(); + int totalPromptTokens = aggr.getTotalPromptTokens(); + int totalCompleteTokens = aggr.getTotalCompleteTokens(); int avgDuration = totalDuration/(requestCount==0?1:requestCount); - double intervalTokens = (double)totalTokens/LLM_POLL_INTERVAL; - double intervalCost = intervalTokens * 0.01; + double intervalPromptTokens = (double)totalPromptTokens/LLM_POLL_INTERVAL; + double intervalCompleteTokens = (double)totalCompleteTokens/LLM_POLL_INTERVAL; + double intervalTotalTokens = intervalPromptTokens + intervalCompleteTokens; + double intervalPromptCost = (intervalPromptTokens/1000) * pricePromptTokens; + double intervalCompleteCost = (intervalCompleteTokens/1000) * priceCompleteTokens; + double intervalTotalCost = intervalPromptCost + intervalCompleteCost; double intervalRequests = (double)requestCount/LLM_POLL_INTERVAL; aggr.resetMetrics(); @@ -223,14 +204,14 @@ public void collectData() { attributes.put("user_id", aggr.getUserId()); getRawMetric(LLM_DURATION_NAME).setValue(avgDuration, attributes); getRawMetric(LLM_DURATION_MAX_NAME).setValue(maxDuration, attributes); - getRawMetric(LLM_COST_NAME).setValue(intervalCost, attributes); - getRawMetric(LLM_TOKEN_NAME).setValue(intervalTokens, attributes); + getRawMetric(LLM_COST_NAME).setValue(intervalTotalCost, attributes); + getRawMetric(LLM_TOKEN_NAME).setValue(intervalTotalTokens, attributes); getRawMetric(LLM_REQ_COUNT_NAME).setValue(intervalRequests, attributes); logger.info("LLM_DURATION_NAME: " + avgDuration); logger.info("LLM_DURATION_MAX_NAME: " + maxDuration); - logger.info("LLM_COST_NAME: " + intervalCost); - logger.info("LLM_TOKEN_NAME: " + intervalTokens); + logger.info("LLM_COST_NAME: " + intervalTotalCost); + logger.info("LLM_TOKEN_NAME: " + intervalTotalTokens); logger.info("LLM_REQ_COUNT_NAME: " + intervalRequests); } } diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java new file mode 100644 index 0000000..117cfc8 --- /dev/null +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java @@ -0,0 +1,186 @@ +package com.instana.dc.ai.impl.llm; + +import com.google.common.collect.ImmutableList; +import io.grpc.stub.StreamObserver; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceRequest; +import io.opentelemetry.proto.collector.metrics.v1.ExportMetricsServiceResponse; +import io.opentelemetry.proto.collector.metrics.v1.MetricsServiceGrpc; +import io.opentelemetry.proto.common.v1.InstrumentationScope; +import io.opentelemetry.proto.common.v1.KeyValue; +import io.opentelemetry.proto.metrics.v1.Metric; +import io.opentelemetry.proto.metrics.v1.ResourceMetrics; +import io.opentelemetry.proto.metrics.v1.ScopeMetrics; +import io.opentelemetry.proto.metrics.v1.NumberDataPoint; +import io.opentelemetry.proto.metrics.v1.HistogramDataPoint; +import io.opentelemetry.proto.resource.v1.Resource; + +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingDeque; + + +class MetricsCollectorService extends MetricsServiceGrpc.MetricsServiceImplBase { + + public class OtelMetric { + private String modelId; + private long promptTokens; + private long completeTokens; + private double duration; + + public String getModelId() { + return modelId; + } + public long getPromtTokens() { + return promptTokens; + } + public long getCompleteTokens() { + return completeTokens; + } + public double getDuration() { + return duration; + } + public void setModelId(String modelId) { + this.modelId = modelId; + } + public void setPromptTokens(long promptTokens) { + this.promptTokens = promptTokens; + } + public void setCompleteTokens(long completeTokens) { + this.completeTokens = completeTokens; + } + public void setDuration(double duration) { + this.duration = duration; + } + } + + private final BlockingQueue exportMetrics = new LinkedBlockingDeque<>(); + + public List getMetrics() { + return ImmutableList.copyOf(exportMetrics); + } + + public void clearMetrics() { + exportMetrics.clear(); + } + + @Override + public void export( + ExportMetricsServiceRequest request, + StreamObserver responseObserver) { + + System.out.println("--------------------------------------------------------"); + + List allResourceMetrics = request.getResourceMetricsList(); + for (ResourceMetrics resourceMetrics : allResourceMetrics) { + + OtelMetric otelMetric = new OtelMetric(); + Resource resource = resourceMetrics.getResource(); + for (KeyValue reskv : resource.getAttributesList()) { + System.out.println("Received metric resouce --- attrKey: " + reskv.getKey()); + System.out.println("Received metric resouce --- attrVal: " + reskv.getValue().getStringValue()); + } + + for (ScopeMetrics scoMetrics : resourceMetrics.getScopeMetricsList()) { + InstrumentationScope instrumentationScope = scoMetrics.getScope(); + instrumentationScope.getAttributesList(); + for (KeyValue inskv : instrumentationScope.getAttributesList()) { + System.out.println("Received metric scope --- attrKey: " + inskv.getKey()); + System.out.println("Received metric scope --- attrVal: " + inskv.getValue().getStringValue()); + } + + for (Metric metric : scoMetrics.getMetricsList()) { + System.out.println("Received metric --- Name: " + metric.getName()); + System.out.println("Received metric --- Desc: " + metric.getDescription()); + System.out.println("Received metric --- Unit: " + metric.getUnit()); + System.out.println("Received metric --- Case: " + metric.getDataCase().getNumber()); + + switch(metric.getDataCase()) { + case GAUGE: + List gaugeDataPoints = metric.getGauge().getDataPointsList(); + for (NumberDataPoint dataPoint : gaugeDataPoints) { + + List kvList = dataPoint.getAttributesList(); + + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Gauge attrKey: " + kv.getKey()); + System.out.println("Received metric --- Gauge attrVal: " + kv.getValue().getStringValue()); + } + switch(dataPoint.getValueCase()) { + case AS_INT: + System.out.println("Received metric --- Gauge Int Value: " + dataPoint.getAsInt()); + break; + case AS_DOUBLE: + System.out.println("Received metric --- Gauge Double Value: " + dataPoint.getAsDouble()); + break; + default: + System.out.println("Unsupported metric Gauge ValueCase: " + dataPoint.getValueCase()); + } + } + break; + case SUM: + List sumDataPoints = metric.getSum().getDataPointsList(); + for (NumberDataPoint dataPoint : sumDataPoints) { + + List kvList = dataPoint.getAttributesList(); + + String tokenType = ""; + String modelId = ""; + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Sum attrKey: " + kv.getKey()); + System.out.println("Received metric --- Sum attrVal: " + kv.getValue().getStringValue()); + if(kv.getKey().compareTo("llm.response.model") == 0) { + modelId = kv.getValue().getStringValue(); + otelMetric.setModelId(modelId); + } else if(kv.getKey().compareTo("llm.usage.token_type") == 0) { + tokenType = kv.getValue().getStringValue(); + } + } + switch(dataPoint.getValueCase()) { + case AS_INT: + System.out.println("Received metric --- Sum Int Value: " + dataPoint.getAsInt()); + if(tokenType.compareTo("prompt") == 0) { + otelMetric.setPromptTokens(dataPoint.getAsInt()); + } else if(tokenType.compareTo("completion") == 0) { + otelMetric.setCompleteTokens(dataPoint.getAsInt()); + } + break; + case AS_DOUBLE: + System.out.println("Received metric --- Sum Double Value: " + dataPoint.getAsDouble()); + break; + default: + System.out.println("Unsupported metric Sum ValueCase: " + dataPoint.getValueCase()); + } + } + break; + case HISTOGRAM: + List histDataPoints = metric.getHistogram().getDataPointsList(); + for (HistogramDataPoint dataPoint : histDataPoints) { + + List kvList = dataPoint.getAttributesList(); + + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Histogram attrKey: " + kv.getKey()); + System.out.println("Received metric --- Histogram attrVal: " + kv.getValue().getStringValue()); + } + System.out.println("Received metric --- Histogram Double Value: " + dataPoint.getSum()); + + if(metric.getName().compareTo("llm.watsonx.completions.duration") == 0) { + otelMetric.setDuration(dataPoint.getSum()); + } + + } + break; + case SUMMARY: + default: + System.out.println("Unsupported metric DataCase: " + metric.getDataCase()); + throw new AssertionError("Unsupported metric DataCase: " + metric.getDataCase()); + } + } + } + exportMetrics.add(otelMetric); + } + + responseObserver.onNext(ExportMetricsServiceResponse.getDefaultInstance()); + responseObserver.onCompleted(); + } +} \ No newline at end of file From 1ea18d5d12cb001494f5ee18b947de3470fc3485 Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Wed, 24 Apr 2024 21:39:12 -0700 Subject: [PATCH 02/13] Fix the calculation for request count --- ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index b17fa28..4cd211e 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -35,7 +35,7 @@ public class LLMDc extends AbstractLLMDc { /** * The poll rate in the configuration, in seconds. In other words, the number of * seconds between calls to Watsonx. - */ + */ private class ModelAggregation { private final String modelId; @@ -196,7 +196,6 @@ public void collectData() { double intervalPromptCost = (intervalPromptTokens/1000) * pricePromptTokens; double intervalCompleteCost = (intervalCompleteTokens/1000) * priceCompleteTokens; double intervalTotalCost = intervalPromptCost + intervalCompleteCost; - double intervalRequests = (double)requestCount/LLM_POLL_INTERVAL; aggr.resetMetrics(); Map attributes = new HashMap<>(); @@ -206,13 +205,13 @@ public void collectData() { getRawMetric(LLM_DURATION_MAX_NAME).setValue(maxDuration, attributes); getRawMetric(LLM_COST_NAME).setValue(intervalTotalCost, attributes); getRawMetric(LLM_TOKEN_NAME).setValue(intervalTotalTokens, attributes); - getRawMetric(LLM_REQ_COUNT_NAME).setValue(intervalRequests, attributes); + getRawMetric(LLM_REQ_COUNT_NAME).setValue(requestCount, attributes); logger.info("LLM_DURATION_NAME: " + avgDuration); logger.info("LLM_DURATION_MAX_NAME: " + maxDuration); logger.info("LLM_COST_NAME: " + intervalTotalCost); logger.info("LLM_TOKEN_NAME: " + intervalTotalTokens); - logger.info("LLM_REQ_COUNT_NAME: " + intervalRequests); + logger.info("LLM_REQ_COUNT_NAME: " + requestCount); } } } From b3a92338d306edc46daad6b9f1f793a5d432e931 Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Thu, 25 Apr 2024 01:41:52 -0700 Subject: [PATCH 03/13] Refine the code and fix metrics missing problem --- .../com/instana/dc/ai/impl/llm/LLMDc.java | 14 ++- .../ai/impl/llm/MetricsCollectorService.java | 119 +++++++++--------- 2 files changed, 68 insertions(+), 65 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index 4cd211e..148bc1c 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -59,6 +59,9 @@ public String getUserId() { return userId; } public void addTotalPromptTokens(int currTokens) { + if(currTokens == 0) { + return; + } int deltaPromptTokens = 0; if(currTokens > lastTotalPromptTokens && lastTotalPromptTokens != 0) { deltaPromptTokens = currTokens - lastTotalPromptTokens; @@ -71,6 +74,9 @@ public int getTotalPromptTokens() { return totalPromptTokens; } public void addTotalCompleteTokens(int currTokens) { + if(currTokens == 0) { + return; + } int deltaCompleteTokens = 0; if(currTokens > lastTotalCompleteTokens && lastTotalCompleteTokens != 0) { deltaCompleteTokens = currTokens - lastTotalCompleteTokens; @@ -82,6 +88,9 @@ public int getTotalCompleteTokens() { return totalCompleteTokens; } public void addTotalDuration(int duration) { + if(duration == 0) { + return; + } totalDuration += duration; if(duration > maxDuration) maxDuration = duration; @@ -172,9 +181,10 @@ public void collectData() { modelAggr.addTotalPromptTokens((int)(promptTokens)); modelAggr.addTotalCompleteTokens((int)(completeTokens)); - modelAggr.addReqCount(1); modelAggr.addTotalDuration((int)(duration*1000)); - + if(promptTokens != 0) { + modelAggr.addReqCount(1); + } } catch (Exception e) { e.printStackTrace(); } diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java index 117cfc8..4484a5c 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java @@ -73,7 +73,6 @@ public void export( List allResourceMetrics = request.getResourceMetricsList(); for (ResourceMetrics resourceMetrics : allResourceMetrics) { - OtelMetric otelMetric = new OtelMetric(); Resource resource = resourceMetrics.getResource(); for (KeyValue reskv : resource.getAttributesList()) { System.out.println("Received metric resouce --- attrKey: " + reskv.getKey()); @@ -95,81 +94,76 @@ public void export( System.out.println("Received metric --- Case: " + metric.getDataCase().getNumber()); switch(metric.getDataCase()) { - case GAUGE: - List gaugeDataPoints = metric.getGauge().getDataPointsList(); - for (NumberDataPoint dataPoint : gaugeDataPoints) { + case SUM: + if(metric.getName().compareTo("llm.watsonx.completions.tokens") == 0) { + + List sumDataPoints = metric.getSum().getDataPointsList(); + for (NumberDataPoint dataPoint : sumDataPoints) { + + List kvList = dataPoint.getAttributesList(); + + String modelId = ""; + String tokenType = ""; + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Sum attrKey: " + kv.getKey()); + System.out.println("Received metric --- Sum attrVal: " + kv.getValue().getStringValue()); + if(kv.getKey().compareTo("llm.response.model") == 0) { + modelId = kv.getValue().getStringValue(); + } else if(kv.getKey().compareTo("llm.usage.token_type") == 0) { + tokenType = kv.getValue().getStringValue(); + } + } - List kvList = dataPoint.getAttributesList(); + long promptTokens = 0; + long completeTokens = 0; + if (tokenType.compareTo("prompt") == 0) { + promptTokens = dataPoint.getAsInt(); + System.out.println("Received metric --- Sum Prompt Value: " + promptTokens); + } else if (tokenType.compareTo("completion") == 0) { + completeTokens = dataPoint.getAsInt(); + System.out.println("Received metric --- Sum Complete Value: " + completeTokens); + } - for (KeyValue kv : kvList) { - System.out.println("Received metric --- Gauge attrKey: " + kv.getKey()); - System.out.println("Received metric --- Gauge attrVal: " + kv.getValue().getStringValue()); - } - switch(dataPoint.getValueCase()) { - case AS_INT: - System.out.println("Received metric --- Gauge Int Value: " + dataPoint.getAsInt()); - break; - case AS_DOUBLE: - System.out.println("Received metric --- Gauge Double Value: " + dataPoint.getAsDouble()); - break; - default: - System.out.println("Unsupported metric Gauge ValueCase: " + dataPoint.getValueCase()); - } - } - break; - case SUM: - List sumDataPoints = metric.getSum().getDataPointsList(); - for (NumberDataPoint dataPoint : sumDataPoints) { - - List kvList = dataPoint.getAttributesList(); - - String tokenType = ""; - String modelId = ""; - for (KeyValue kv : kvList) { - System.out.println("Received metric --- Sum attrKey: " + kv.getKey()); - System.out.println("Received metric --- Sum attrVal: " + kv.getValue().getStringValue()); - if(kv.getKey().compareTo("llm.response.model") == 0) { - modelId = kv.getValue().getStringValue(); + if (!modelId.isEmpty()) { + OtelMetric otelMetric = new OtelMetric(); otelMetric.setModelId(modelId); - } else if(kv.getKey().compareTo("llm.usage.token_type") == 0) { - tokenType = kv.getValue().getStringValue(); + otelMetric.setPromptTokens(promptTokens); + otelMetric.setCompleteTokens(completeTokens); + exportMetrics.add(otelMetric); } } - switch(dataPoint.getValueCase()) { - case AS_INT: - System.out.println("Received metric --- Sum Int Value: " + dataPoint.getAsInt()); - if(tokenType.compareTo("prompt") == 0) { - otelMetric.setPromptTokens(dataPoint.getAsInt()); - } else if(tokenType.compareTo("completion") == 0) { - otelMetric.setCompleteTokens(dataPoint.getAsInt()); - } - break; - case AS_DOUBLE: - System.out.println("Received metric --- Sum Double Value: " + dataPoint.getAsDouble()); - break; - default: - System.out.println("Unsupported metric Sum ValueCase: " + dataPoint.getValueCase()); - } } break; case HISTOGRAM: - List histDataPoints = metric.getHistogram().getDataPointsList(); - for (HistogramDataPoint dataPoint : histDataPoints) { + if(metric.getName().compareTo("llm.watsonx.completions.duration") == 0) { - List kvList = dataPoint.getAttributesList(); + List histDataPoints = metric.getHistogram().getDataPointsList(); + for (HistogramDataPoint dataPoint : histDataPoints) { - for (KeyValue kv : kvList) { - System.out.println("Received metric --- Histogram attrKey: " + kv.getKey()); - System.out.println("Received metric --- Histogram attrVal: " + kv.getValue().getStringValue()); - } - System.out.println("Received metric --- Histogram Double Value: " + dataPoint.getSum()); + List kvList = dataPoint.getAttributesList(); - if(metric.getName().compareTo("llm.watsonx.completions.duration") == 0) { - otelMetric.setDuration(dataPoint.getSum()); + String modelId = ""; + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Histogram attrKey: " + kv.getKey()); + System.out.println("Received metric --- Histogram attrVal: " + kv.getValue().getStringValue()); + if(kv.getKey().compareTo("llm.response.model") == 0) { + modelId = kv.getValue().getStringValue(); + } + } + + Double durationSum = dataPoint.getSum(); + System.out.println("Received metric --- Histogram Sum Value: " + durationSum); + + if(!modelId.isEmpty()) { + OtelMetric otelMetric = new OtelMetric(); + otelMetric.setModelId(modelId); + otelMetric.setDuration(durationSum); + exportMetrics.add(otelMetric); + } } - } break; + case GAUGE: case SUMMARY: default: System.out.println("Unsupported metric DataCase: " + metric.getDataCase()); @@ -177,7 +171,6 @@ public void export( } } } - exportMetrics.add(otelMetric); } responseObserver.onNext(ExportMetricsServiceResponse.getDefaultInstance()); From b790ff45f6b47fe44d1e9e9801cfe24eb04daedc Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Thu, 25 Apr 2024 10:52:07 -0700 Subject: [PATCH 04/13] Change log output for more clear --- .../com/instana/dc/ai/impl/llm/LLMDc.java | 14 +++++---- .../ai/impl/llm/MetricsCollectorService.java | 30 +++++++++---------- 2 files changed, 24 insertions(+), 20 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index 148bc1c..25f8003 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -217,11 +217,15 @@ public void collectData() { getRawMetric(LLM_TOKEN_NAME).setValue(intervalTotalTokens, attributes); getRawMetric(LLM_REQ_COUNT_NAME).setValue(requestCount, attributes); - logger.info("LLM_DURATION_NAME: " + avgDuration); - logger.info("LLM_DURATION_MAX_NAME: " + maxDuration); - logger.info("LLM_COST_NAME: " + intervalTotalCost); - logger.info("LLM_TOKEN_NAME: " + intervalTotalTokens); - logger.info("LLM_REQ_COUNT_NAME: " + requestCount); + System.out.println("-----------------------------------------"); + System.out.println("ModelId : " + attributes.get("model_id")); + System.out.println("UserId : " + attributes.get("user_id")); + System.out.println("AvgDuration : " + avgDuration); + System.out.println("MaxDuration : " + maxDuration); + System.out.println("IntervalTokens : " + intervalTotalTokens); + System.out.println("IntervalCost : " + intervalTotalCost); + System.out.println("IntervalRequest : " + requestCount); + System.out.println("-----------------------------------------"); } } } diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java index 4484a5c..3e5242d 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java @@ -75,23 +75,23 @@ public void export( Resource resource = resourceMetrics.getResource(); for (KeyValue reskv : resource.getAttributesList()) { - System.out.println("Received metric resouce --- attrKey: " + reskv.getKey()); - System.out.println("Received metric resouce --- attrVal: " + reskv.getValue().getStringValue()); + System.out.println("Received metric --- Resource attrKey: " + reskv.getKey()); + System.out.println("Received metric --- Resource attrVal: " + reskv.getValue().getStringValue()); } for (ScopeMetrics scoMetrics : resourceMetrics.getScopeMetricsList()) { InstrumentationScope instrumentationScope = scoMetrics.getScope(); instrumentationScope.getAttributesList(); for (KeyValue inskv : instrumentationScope.getAttributesList()) { - System.out.println("Received metric scope --- attrKey: " + inskv.getKey()); - System.out.println("Received metric scope --- attrVal: " + inskv.getValue().getStringValue()); + System.out.println("Received metric --- Scope attrKey: " + inskv.getKey()); + System.out.println("Received metric --- Scope attrVal: " + inskv.getValue().getStringValue()); } for (Metric metric : scoMetrics.getMetricsList()) { - System.out.println("Received metric --- Name: " + metric.getName()); - System.out.println("Received metric --- Desc: " + metric.getDescription()); - System.out.println("Received metric --- Unit: " + metric.getUnit()); - System.out.println("Received metric --- Case: " + metric.getDataCase().getNumber()); + System.out.println("Received metric --- Scope Name: " + metric.getName()); + System.out.println("Received metric --- Scope Desc: " + metric.getDescription()); + System.out.println("Received metric --- Scope Unit: " + metric.getUnit()); + System.out.println("Received metric --- Scope Case: " + metric.getDataCase().getNumber()); switch(metric.getDataCase()) { case SUM: @@ -105,8 +105,8 @@ public void export( String modelId = ""; String tokenType = ""; for (KeyValue kv : kvList) { - System.out.println("Received metric --- Sum attrKey: " + kv.getKey()); - System.out.println("Received metric --- Sum attrVal: " + kv.getValue().getStringValue()); + System.out.println("Received metric --- Tokens attrKey: " + kv.getKey()); + System.out.println("Received metric --- Tokens attrVal: " + kv.getValue().getStringValue()); if(kv.getKey().compareTo("llm.response.model") == 0) { modelId = kv.getValue().getStringValue(); } else if(kv.getKey().compareTo("llm.usage.token_type") == 0) { @@ -118,10 +118,10 @@ public void export( long completeTokens = 0; if (tokenType.compareTo("prompt") == 0) { promptTokens = dataPoint.getAsInt(); - System.out.println("Received metric --- Sum Prompt Value: " + promptTokens); + System.out.println("Received metric --- Prompt Value: " + promptTokens); } else if (tokenType.compareTo("completion") == 0) { completeTokens = dataPoint.getAsInt(); - System.out.println("Received metric --- Sum Complete Value: " + completeTokens); + System.out.println("Received metric --- Complete Value: " + completeTokens); } if (!modelId.isEmpty()) { @@ -144,15 +144,15 @@ public void export( String modelId = ""; for (KeyValue kv : kvList) { - System.out.println("Received metric --- Histogram attrKey: " + kv.getKey()); - System.out.println("Received metric --- Histogram attrVal: " + kv.getValue().getStringValue()); + System.out.println("Received metric --- Duration attrKey: " + kv.getKey()); + System.out.println("Received metric --- Duration attrVal: " + kv.getValue().getStringValue()); if(kv.getKey().compareTo("llm.response.model") == 0) { modelId = kv.getValue().getStringValue(); } } Double durationSum = dataPoint.getSum(); - System.out.println("Received metric --- Histogram Sum Value: " + durationSum); + System.out.println("Received metric --- Duration Sum Value: " + durationSum); if(!modelId.isEmpty()) { OtelMetric otelMetric = new OtelMetric(); From c07734cc302f52dbd04e9db59d8dcb08645b3cee Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Thu, 25 Apr 2024 15:26:05 -0700 Subject: [PATCH 05/13] Calculate the delta value for duration sum value --- .../com/instana/dc/ai/impl/llm/LLMDc.java | 111 ++++++++++-------- .../ai/impl/llm/MetricsCollectorService.java | 10 ++ 2 files changed, 74 insertions(+), 47 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index 25f8003..29d2575 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -40,13 +40,15 @@ public class LLMDc extends AbstractLLMDc { private class ModelAggregation { private final String modelId; private final String userId; - private int totalPromptTokens; - private int totalCompleteTokens; - private int totalDuration; + private int deltaPromptTokens; + private int deltaCompleteTokens; + private int deltaDuration; + private int deltaReqCount; private int maxDuration; - private int totalReqCount; private int lastTotalPromptTokens; private int lastTotalCompleteTokens; + private int lastTotalDuration; + private int lastTotalReqCount; public ModelAggregation(String modelId, String userId) { this.modelId = modelId; @@ -58,61 +60,76 @@ public String getModelId() { public String getUserId() { return userId; } - public void addTotalPromptTokens(int currTokens) { + public void addDeltaPromptTokens(int currTokens) { if(currTokens == 0) { return; } - int deltaPromptTokens = 0; + int diffPromptTokens = 0; if(currTokens > lastTotalPromptTokens && lastTotalPromptTokens != 0) { - deltaPromptTokens = currTokens - lastTotalPromptTokens; + diffPromptTokens = currTokens - lastTotalPromptTokens; } lastTotalPromptTokens = currTokens; - totalPromptTokens += deltaPromptTokens; + deltaPromptTokens += diffPromptTokens; } - public int getTotalPromptTokens() { - return totalPromptTokens; + public int getDeltaPromptTokens() { + return deltaPromptTokens; } - public void addTotalCompleteTokens(int currTokens) { + public void addDeltaCompleteTokens(int currTokens) { if(currTokens == 0) { return; } - int deltaCompleteTokens = 0; + int diffCompleteTokens = 0; if(currTokens > lastTotalCompleteTokens && lastTotalCompleteTokens != 0) { - deltaCompleteTokens = currTokens - lastTotalCompleteTokens; + diffCompleteTokens = currTokens - lastTotalCompleteTokens; } lastTotalCompleteTokens = currTokens; - totalCompleteTokens += deltaCompleteTokens; + deltaCompleteTokens += diffCompleteTokens; } - public int getTotalCompleteTokens() { - return totalCompleteTokens; + public int getDeltaCompleteTokens() { + return deltaCompleteTokens; } - public void addTotalDuration(int duration) { - if(duration == 0) { + public void addDeltaDuration(int currDuration) { + if(currDuration == 0) { return; } - totalDuration += duration; - if(duration > maxDuration) - maxDuration = duration; + int diffDuration = 0; + if(currDuration > lastTotalDuration && lastTotalDuration != 0) { + diffDuration = currDuration - lastTotalDuration; + } + lastTotalDuration = currDuration; + deltaDuration += diffDuration; + + if(deltaDuration > maxDuration) { + maxDuration = deltaDuration; + } } - public int getTotalDuration() { - return totalDuration; + public int getDeltaDuration() { + return deltaDuration; } public int getMaxDuration() { return maxDuration; } - public void addReqCount(int count) { - totalReqCount += count; + public void addDeltaReqCount(int currCount) { + if(currCount == 0) { + return; + } + int diffReqCount = 0; + if(currCount > lastTotalReqCount && lastTotalReqCount != 0) { + diffReqCount = currCount - lastTotalReqCount; + } + lastTotalReqCount = currCount; + deltaReqCount += diffReqCount; } - public int getReqCount() { - return totalReqCount; + public int getDeltaReqCount() { + return deltaReqCount; } public void resetMetrics() { - totalPromptTokens = 0; - totalCompleteTokens = 0; - totalDuration = 0; + deltaPromptTokens = 0; + deltaCompleteTokens = 0; + deltaDuration = 0; maxDuration = 0; - totalReqCount = 0; + deltaReqCount = 0; } } @@ -172,6 +189,7 @@ public void collectData() { long promptTokens = metric.getPromtTokens(); long completeTokens = metric.getCompleteTokens(); double duration = metric.getDuration(); + long requestCount = metric.getReqCount(); ModelAggregation modelAggr = modelAggrMap.get(modelId); if (modelAggr == null) { @@ -179,29 +197,27 @@ public void collectData() { modelAggrMap.put(modelId, modelAggr); } - modelAggr.addTotalPromptTokens((int)(promptTokens)); - modelAggr.addTotalCompleteTokens((int)(completeTokens)); - modelAggr.addTotalDuration((int)(duration*1000)); - if(promptTokens != 0) { - modelAggr.addReqCount(1); - } + modelAggr.addDeltaPromptTokens((int)(promptTokens)); + modelAggr.addDeltaCompleteTokens((int)(completeTokens)); + modelAggr.addDeltaDuration((int)(duration*1000)); + modelAggr.addDeltaReqCount((int)(requestCount)); } catch (Exception e) { e.printStackTrace(); } } - getRawMetric(LLM_STATUS_NAME).setValue(1); for(Map.Entry entry : modelAggrMap.entrySet()){ ModelAggregation aggr = entry.getValue(); - int requestCount = aggr.getReqCount(); - int totalDuration = aggr.getTotalDuration(); + int deltaRequestCount = aggr.getDeltaReqCount(); + int deltaDuration = aggr.getDeltaDuration(); + int deltaPromptTokens = aggr.getDeltaPromptTokens(); + int deltaCompleteTokens = aggr.getDeltaCompleteTokens(); int maxDuration = aggr.getMaxDuration(); - int totalPromptTokens = aggr.getTotalPromptTokens(); - int totalCompleteTokens = aggr.getTotalCompleteTokens(); - int avgDuration = totalDuration/(requestCount==0?1:requestCount); - double intervalPromptTokens = (double)totalPromptTokens/LLM_POLL_INTERVAL; - double intervalCompleteTokens = (double)totalCompleteTokens/LLM_POLL_INTERVAL; + int avgDuration = deltaDuration/(deltaRequestCount==0?1:deltaRequestCount); + double intervalReqCount = (double)deltaRequestCount/LLM_POLL_INTERVAL; + double intervalPromptTokens = (double)deltaPromptTokens/LLM_POLL_INTERVAL; + double intervalCompleteTokens = (double)deltaCompleteTokens/LLM_POLL_INTERVAL; double intervalTotalTokens = intervalPromptTokens + intervalCompleteTokens; double intervalPromptCost = (intervalPromptTokens/1000) * pricePromptTokens; double intervalCompleteCost = (intervalCompleteTokens/1000) * priceCompleteTokens; @@ -211,11 +227,12 @@ public void collectData() { Map attributes = new HashMap<>(); attributes.put("model_id", aggr.getModelId()); attributes.put("user_id", aggr.getUserId()); + getRawMetric(LLM_STATUS_NAME).setValue(1); getRawMetric(LLM_DURATION_NAME).setValue(avgDuration, attributes); getRawMetric(LLM_DURATION_MAX_NAME).setValue(maxDuration, attributes); getRawMetric(LLM_COST_NAME).setValue(intervalTotalCost, attributes); getRawMetric(LLM_TOKEN_NAME).setValue(intervalTotalTokens, attributes); - getRawMetric(LLM_REQ_COUNT_NAME).setValue(requestCount, attributes); + getRawMetric(LLM_REQ_COUNT_NAME).setValue(intervalReqCount, attributes); System.out.println("-----------------------------------------"); System.out.println("ModelId : " + attributes.get("model_id")); @@ -224,7 +241,7 @@ public void collectData() { System.out.println("MaxDuration : " + maxDuration); System.out.println("IntervalTokens : " + intervalTotalTokens); System.out.println("IntervalCost : " + intervalTotalCost); - System.out.println("IntervalRequest : " + requestCount); + System.out.println("IntervalRequest : " + intervalReqCount); System.out.println("-----------------------------------------"); } } diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java index 3e5242d..0e71f43 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java @@ -26,6 +26,7 @@ public class OtelMetric { private long promptTokens; private long completeTokens; private double duration; + private long requestCount; public String getModelId() { return modelId; @@ -39,6 +40,9 @@ public long getCompleteTokens() { public double getDuration() { return duration; } + public long getReqCount() { + return requestCount; + } public void setModelId(String modelId) { this.modelId = modelId; } @@ -51,6 +55,9 @@ public void setCompleteTokens(long completeTokens) { public void setDuration(double duration) { this.duration = duration; } + public void setReqCount(long requestCount) { + this.requestCount = requestCount; + } } private final BlockingQueue exportMetrics = new LinkedBlockingDeque<>(); @@ -152,12 +159,15 @@ public void export( } Double durationSum = dataPoint.getSum(); + long requestCount = dataPoint.getCount(); System.out.println("Received metric --- Duration Sum Value: " + durationSum); + System.out.println("Received metric --- Duration Count Value: " + requestCount); if(!modelId.isEmpty()) { OtelMetric otelMetric = new OtelMetric(); otelMetric.setModelId(modelId); otelMetric.setDuration(durationSum); + otelMetric.setReqCount(requestCount); exportMetrics.add(otelMetric); } } From 0f25a1628bbd8ee19e90cc1e0df49abe25799733 Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Mon, 29 Apr 2024 01:44:07 -0700 Subject: [PATCH 06/13] Add mutex lock for the access of metrics queue --- .../com/instana/dc/ai/impl/llm/LLMDc.java | 5 +- .../ai/impl/llm/MetricsCollectorService.java | 207 ++++++++++-------- 2 files changed, 115 insertions(+), 97 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index 29d2575..f2b9ed5 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -206,6 +206,7 @@ public void collectData() { } } + System.out.println("-----------------------------------------"); for(Map.Entry entry : modelAggrMap.entrySet()){ ModelAggregation aggr = entry.getValue(); int deltaRequestCount = aggr.getDeltaReqCount(); @@ -234,7 +235,6 @@ public void collectData() { getRawMetric(LLM_TOKEN_NAME).setValue(intervalTotalTokens, attributes); getRawMetric(LLM_REQ_COUNT_NAME).setValue(intervalReqCount, attributes); - System.out.println("-----------------------------------------"); System.out.println("ModelId : " + attributes.get("model_id")); System.out.println("UserId : " + attributes.get("user_id")); System.out.println("AvgDuration : " + avgDuration); @@ -242,7 +242,8 @@ public void collectData() { System.out.println("IntervalTokens : " + intervalTotalTokens); System.out.println("IntervalCost : " + intervalTotalCost); System.out.println("IntervalRequest : " + intervalReqCount); - System.out.println("-----------------------------------------"); + System.out.println(""); } + System.out.println("-----------------------------------------"); } } diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java index 0e71f43..ebf2d66 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java @@ -18,43 +18,53 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingDeque; - class MetricsCollectorService extends MetricsServiceGrpc.MetricsServiceImplBase { - public class OtelMetric { + private Object mutex = new Object(); + + public class OtelMetric { private String modelId; private long promptTokens; private long completeTokens; private double duration; private long requestCount; - + public String getModelId() { return modelId; } + public long getPromtTokens() { return promptTokens; } + public long getCompleteTokens() { return completeTokens; } + public double getDuration() { return duration; } + public long getReqCount() { return requestCount; } + public void setModelId(String modelId) { this.modelId = modelId; } + public void setPromptTokens(long promptTokens) { this.promptTokens = promptTokens; } + public void setCompleteTokens(long completeTokens) { this.completeTokens = completeTokens; } + public void setDuration(double duration) { this.duration = duration; } + public void setReqCount(long requestCount) { this.requestCount = requestCount; } @@ -63,7 +73,9 @@ public void setReqCount(long requestCount) { private final BlockingQueue exportMetrics = new LinkedBlockingDeque<>(); public List getMetrics() { - return ImmutableList.copyOf(exportMetrics); + synchronized (mutex) { + return ImmutableList.copyOf(exportMetrics); + } } public void clearMetrics() { @@ -74,115 +86,120 @@ public void clearMetrics() { public void export( ExportMetricsServiceRequest request, StreamObserver responseObserver) { - + System.out.println("--------------------------------------------------------"); - List allResourceMetrics = request.getResourceMetricsList(); - for (ResourceMetrics resourceMetrics : allResourceMetrics) { + synchronized (mutex) { - Resource resource = resourceMetrics.getResource(); - for (KeyValue reskv : resource.getAttributesList()) { - System.out.println("Received metric --- Resource attrKey: " + reskv.getKey()); - System.out.println("Received metric --- Resource attrVal: " + reskv.getValue().getStringValue()); - } + List allResourceMetrics = request.getResourceMetricsList(); + for (ResourceMetrics resourceMetrics : allResourceMetrics) { - for (ScopeMetrics scoMetrics : resourceMetrics.getScopeMetricsList()) { - InstrumentationScope instrumentationScope = scoMetrics.getScope(); - instrumentationScope.getAttributesList(); - for (KeyValue inskv : instrumentationScope.getAttributesList()) { - System.out.println("Received metric --- Scope attrKey: " + inskv.getKey()); - System.out.println("Received metric --- Scope attrVal: " + inskv.getValue().getStringValue()); + Resource resource = resourceMetrics.getResource(); + for (KeyValue reskv : resource.getAttributesList()) { + System.out.println("Received metric --- Resource attrKey: " + reskv.getKey()); + System.out.println("Received metric --- Resource attrVal: " + reskv.getValue().getStringValue()); } - for (Metric metric : scoMetrics.getMetricsList()) { - System.out.println("Received metric --- Scope Name: " + metric.getName()); - System.out.println("Received metric --- Scope Desc: " + metric.getDescription()); - System.out.println("Received metric --- Scope Unit: " + metric.getUnit()); - System.out.println("Received metric --- Scope Case: " + metric.getDataCase().getNumber()); - - switch(metric.getDataCase()) { - case SUM: - if(metric.getName().compareTo("llm.watsonx.completions.tokens") == 0) { - - List sumDataPoints = metric.getSum().getDataPointsList(); - for (NumberDataPoint dataPoint : sumDataPoints) { - - List kvList = dataPoint.getAttributesList(); - - String modelId = ""; - String tokenType = ""; - for (KeyValue kv : kvList) { - System.out.println("Received metric --- Tokens attrKey: " + kv.getKey()); - System.out.println("Received metric --- Tokens attrVal: " + kv.getValue().getStringValue()); - if(kv.getKey().compareTo("llm.response.model") == 0) { - modelId = kv.getValue().getStringValue(); - } else if(kv.getKey().compareTo("llm.usage.token_type") == 0) { - tokenType = kv.getValue().getStringValue(); + for (ScopeMetrics scoMetrics : resourceMetrics.getScopeMetricsList()) { + InstrumentationScope instrumentationScope = scoMetrics.getScope(); + instrumentationScope.getAttributesList(); + for (KeyValue inskv : instrumentationScope.getAttributesList()) { + System.out.println("Received metric --- Scope attrKey: " + inskv.getKey()); + System.out.println("Received metric --- Scope attrVal: " + inskv.getValue().getStringValue()); + } + + for (Metric metric : scoMetrics.getMetricsList()) { + System.out.println("Received metric --- Scope Name: " + metric.getName()); + System.out.println("Received metric --- Scope Desc: " + metric.getDescription()); + System.out.println("Received metric --- Scope Unit: " + metric.getUnit()); + System.out.println("Received metric --- Scope Case: " + metric.getDataCase().getNumber()); + + switch (metric.getDataCase()) { + case SUM: + if (metric.getName().compareTo("llm.watsonx.completions.tokens") == 0) { + + List sumDataPoints = metric.getSum().getDataPointsList(); + for (NumberDataPoint dataPoint : sumDataPoints) { + + List kvList = dataPoint.getAttributesList(); + + String modelId = ""; + String tokenType = ""; + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Tokens attrKey: " + kv.getKey()); + System.out.println("Received metric --- Tokens attrVal: " + + kv.getValue().getStringValue()); + if (kv.getKey().compareTo("llm.response.model") == 0) { + modelId = kv.getValue().getStringValue(); + } else if (kv.getKey().compareTo("llm.usage.token_type") == 0) { + tokenType = kv.getValue().getStringValue(); + } } - } - long promptTokens = 0; - long completeTokens = 0; - if (tokenType.compareTo("prompt") == 0) { - promptTokens = dataPoint.getAsInt(); - System.out.println("Received metric --- Prompt Value: " + promptTokens); - } else if (tokenType.compareTo("completion") == 0) { - completeTokens = dataPoint.getAsInt(); - System.out.println("Received metric --- Complete Value: " + completeTokens); - } + long promptTokens = 0; + long completeTokens = 0; + if (tokenType.compareTo("prompt") == 0) { + promptTokens = dataPoint.getAsInt(); + System.out.println("Received metric --- Prompt Value: " + promptTokens); + } else if (tokenType.compareTo("completion") == 0) { + completeTokens = dataPoint.getAsInt(); + System.out.println("Received metric --- Complete Value: " + completeTokens); + } - if (!modelId.isEmpty()) { - OtelMetric otelMetric = new OtelMetric(); - otelMetric.setModelId(modelId); - otelMetric.setPromptTokens(promptTokens); - otelMetric.setCompleteTokens(completeTokens); - exportMetrics.add(otelMetric); + if (!modelId.isEmpty()) { + OtelMetric otelMetric = new OtelMetric(); + otelMetric.setModelId(modelId); + otelMetric.setPromptTokens(promptTokens); + otelMetric.setCompleteTokens(completeTokens); + exportMetrics.add(otelMetric); + } } } - } - break; - case HISTOGRAM: - if(metric.getName().compareTo("llm.watsonx.completions.duration") == 0) { - - List histDataPoints = metric.getHistogram().getDataPointsList(); - for (HistogramDataPoint dataPoint : histDataPoints) { - - List kvList = dataPoint.getAttributesList(); - - String modelId = ""; - for (KeyValue kv : kvList) { - System.out.println("Received metric --- Duration attrKey: " + kv.getKey()); - System.out.println("Received metric --- Duration attrVal: " + kv.getValue().getStringValue()); - if(kv.getKey().compareTo("llm.response.model") == 0) { - modelId = kv.getValue().getStringValue(); + break; + case HISTOGRAM: + if (metric.getName().compareTo("llm.watsonx.completions.duration") == 0) { + + List histDataPoints = metric.getHistogram().getDataPointsList(); + for (HistogramDataPoint dataPoint : histDataPoints) { + + List kvList = dataPoint.getAttributesList(); + + String modelId = ""; + for (KeyValue kv : kvList) { + System.out.println("Received metric --- Duration attrKey: " + kv.getKey()); + System.out.println("Received metric --- Duration attrVal: " + + kv.getValue().getStringValue()); + if (kv.getKey().compareTo("llm.response.model") == 0) { + modelId = kv.getValue().getStringValue(); + } + } + + Double durationSum = dataPoint.getSum(); + long requestCount = dataPoint.getCount(); + System.out.println("Received metric --- Duration Sum Value: " + durationSum); + System.out.println("Received metric --- Duration Count Value: " + requestCount); + + if (!modelId.isEmpty()) { + OtelMetric otelMetric = new OtelMetric(); + otelMetric.setModelId(modelId); + otelMetric.setDuration(durationSum); + otelMetric.setReqCount(requestCount); + exportMetrics.add(otelMetric); } - } - - Double durationSum = dataPoint.getSum(); - long requestCount = dataPoint.getCount(); - System.out.println("Received metric --- Duration Sum Value: " + durationSum); - System.out.println("Received metric --- Duration Count Value: " + requestCount); - - if(!modelId.isEmpty()) { - OtelMetric otelMetric = new OtelMetric(); - otelMetric.setModelId(modelId); - otelMetric.setDuration(durationSum); - otelMetric.setReqCount(requestCount); - exportMetrics.add(otelMetric); } } - } - break; - case GAUGE: - case SUMMARY: - default: - System.out.println("Unsupported metric DataCase: " + metric.getDataCase()); - throw new AssertionError("Unsupported metric DataCase: " + metric.getDataCase()); + break; + case GAUGE: + case SUMMARY: + default: + System.out.println("Unsupported metric DataCase: " + metric.getDataCase()); + throw new AssertionError("Unsupported metric DataCase: " + metric.getDataCase()); + } } } } } - + responseObserver.onNext(ExportMetricsServiceResponse.getDefaultInstance()); responseObserver.onCompleted(); } From cee4dba21eafd4b5584a420174acc15546eba14b Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Tue, 30 Apr 2024 00:59:37 -0700 Subject: [PATCH 07/13] Add DataPoint key for metrics to avoid overlap --- .../com/instana/dc/ai/impl/llm/LLMDc.java | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index f2b9ed5..2008c91 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -209,6 +209,8 @@ public void collectData() { System.out.println("-----------------------------------------"); for(Map.Entry entry : modelAggrMap.entrySet()){ ModelAggregation aggr = entry.getValue(); + String modelId = aggr.getModelId(); + String userId = aggr.getUserId(); int deltaRequestCount = aggr.getDeltaReqCount(); int deltaDuration = aggr.getDeltaDuration(); int deltaPromptTokens = aggr.getDeltaPromptTokens(); @@ -225,24 +227,23 @@ public void collectData() { double intervalTotalCost = intervalPromptCost + intervalCompleteCost; aggr.resetMetrics(); - Map attributes = new HashMap<>(); - attributes.put("model_id", aggr.getModelId()); - attributes.put("user_id", aggr.getUserId()); - getRawMetric(LLM_STATUS_NAME).setValue(1); - getRawMetric(LLM_DURATION_NAME).setValue(avgDuration, attributes); - getRawMetric(LLM_DURATION_MAX_NAME).setValue(maxDuration, attributes); - getRawMetric(LLM_COST_NAME).setValue(intervalTotalCost, attributes); - getRawMetric(LLM_TOKEN_NAME).setValue(intervalTotalTokens, attributes); - getRawMetric(LLM_REQ_COUNT_NAME).setValue(intervalReqCount, attributes); - - System.out.println("ModelId : " + attributes.get("model_id")); - System.out.println("UserId : " + attributes.get("user_id")); + System.out.println("ModelId : " + modelId); + System.out.println("UserId : " + userId); System.out.println("AvgDuration : " + avgDuration); System.out.println("MaxDuration : " + maxDuration); System.out.println("IntervalTokens : " + intervalTotalTokens); System.out.println("IntervalCost : " + intervalTotalCost); System.out.println("IntervalRequest : " + intervalReqCount); - System.out.println(""); + + Map attributes = new HashMap<>(); + attributes.put("model_id", modelId); + attributes.put("user_id", userId); + getRawMetric(LLM_STATUS_NAME).getDataPoint(modelId).setValue(1); + getRawMetric(LLM_DURATION_NAME).getDataPoint(modelId).setValue(avgDuration, attributes); + getRawMetric(LLM_DURATION_MAX_NAME).getDataPoint(modelId).setValue(maxDuration, attributes); + getRawMetric(LLM_COST_NAME).getDataPoint(modelId).setValue(intervalTotalCost, attributes); + getRawMetric(LLM_TOKEN_NAME).getDataPoint(modelId).setValue(intervalTotalTokens, attributes); + getRawMetric(LLM_REQ_COUNT_NAME).getDataPoint(modelId).setValue(intervalReqCount, attributes); } System.out.println("-----------------------------------------"); } From 0026f62a32ce4e06c462bfc716d93ca367659643 Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Tue, 30 Apr 2024 14:01:27 -0700 Subject: [PATCH 08/13] Support the new key name of model id from instrumentation --- ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java | 2 +- .../com/instana/dc/ai/impl/llm/MetricsCollectorService.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java index 2008c91..b0b1b7d 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java @@ -238,7 +238,7 @@ public void collectData() { Map attributes = new HashMap<>(); attributes.put("model_id", modelId); attributes.put("user_id", userId); - getRawMetric(LLM_STATUS_NAME).getDataPoint(modelId).setValue(1); + getRawMetric(LLM_STATUS_NAME).setValue(1); getRawMetric(LLM_DURATION_NAME).getDataPoint(modelId).setValue(avgDuration, attributes); getRawMetric(LLM_DURATION_MAX_NAME).getDataPoint(modelId).setValue(maxDuration, attributes); getRawMetric(LLM_COST_NAME).getDataPoint(modelId).setValue(intervalTotalCost, attributes); diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java index ebf2d66..148563e 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java +++ b/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java @@ -129,7 +129,7 @@ public void export( System.out.println("Received metric --- Tokens attrKey: " + kv.getKey()); System.out.println("Received metric --- Tokens attrVal: " + kv.getValue().getStringValue()); - if (kv.getKey().compareTo("llm.response.model") == 0) { + if (kv.getKey().compareTo("llm.response.model") == 0 || kv.getKey().compareTo("gen_ai.response.model") == 0) { modelId = kv.getValue().getStringValue(); } else if (kv.getKey().compareTo("llm.usage.token_type") == 0) { tokenType = kv.getValue().getStringValue(); @@ -169,7 +169,7 @@ public void export( System.out.println("Received metric --- Duration attrKey: " + kv.getKey()); System.out.println("Received metric --- Duration attrVal: " + kv.getValue().getStringValue()); - if (kv.getKey().compareTo("llm.response.model") == 0) { + if (kv.getKey().compareTo("llm.response.model") == 0 || kv.getKey().compareTo("gen_ai.response.model") == 0) { modelId = kv.getValue().getStringValue(); } } From 595bbbc48f474b3bf2627896fd686427bb9c1c31 Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Mon, 6 May 2024 02:10:11 -0700 Subject: [PATCH 09/13] Remove platform and server address properties --- ai/config/config.yaml | 6 +-- .../java/com/instana/dc/ai/AbstractLLMDc.java | 15 +++++-- .../java/com/instana/dc/ai/DataCollector.java | 41 ------------------- .../java/com/instana/dc/ai/LLMDcUtil.java | 4 -- 4 files changed, 12 insertions(+), 54 deletions(-) diff --git a/ai/config/config.yaml b/ai/config/config.yaml index 2c5dd46..3ace1ff 100644 --- a/ai/config/config.yaml +++ b/ai/config/config.yaml @@ -1,8 +1,4 @@ -llm.platform: watsonx -llm.application: llm-dc -server.address: bam-api.res.ibm.com -server.port: 443 -server.scheme: https +llm.application: LLM_DC instances: - otel.backend.url: http://localhost:4317 diff --git a/ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java b/ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java index 1bac8d7..f9b2529 100644 --- a/ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java +++ b/ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java @@ -13,6 +13,8 @@ import io.opentelemetry.sdk.metrics.SdkMeterProvider; import io.opentelemetry.sdk.resources.Resource; +import java.net.InetAddress; +import java.net.UnknownHostException; import java.util.Map; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -40,6 +42,14 @@ public abstract class AbstractLLMDc extends AbstractDc { private final ScheduledExecutorService exec = Executors.newSingleThreadScheduledExecutor(); + public String getHostName() { + try { + return InetAddress.getLocalHost().getHostName(); + } catch (UnknownHostException e) { + return "UnknownName"; + } + } + public AbstractLLMDc(Map properties, CustomDcConfig cdcConfig) { super(new LLMRawMetricRegistry().getMap()); // pollInterval = (Integer) properties.getOrDefault(POLLING_INTERVAL, DEFAULT_POLL_INTERVAL); @@ -49,7 +59,7 @@ public AbstractLLMDc(Map properties, CustomDcConfig cdcConfig) { otelBackendUrl = (String) properties.get(OTEL_BACKEND_URL); otelUsingHttp = (Boolean) properties.getOrDefault(OTEL_BACKEND_USING_HTTP, Boolean.FALSE); serviceName = (String) properties.get(OTEL_SERVICE_NAME); - serviceInstanceId = cdcConfig.getServerAddr() + "(" + cdcConfig.getServerPort() + ")@" + serviceName; + serviceInstanceId = "LLMONITOR:" + serviceName + "@" + getHostName(); this.cdcConfig = cdcConfig; } @@ -57,9 +67,6 @@ public AbstractLLMDc(Map properties, CustomDcConfig cdcConfig) { public Resource getResourceAttributes() { Resource resource = Resource.getDefault() .merge(Resource.create(Attributes.of( - stringKey(LLM_SERVER_ADDR), cdcConfig.getServerAddr(), - stringKey(LLM_SERVER_PORT), cdcConfig.getServerPort(), - stringKey(LLM_PLATFORM_NAME), cdcConfig.getPlatform(), stringKey(SERVICE_NAME), serviceName, stringKey(SERVICE_INSTANCE_ID), serviceInstanceId ))) diff --git a/ai/src/main/java/com/instana/dc/ai/DataCollector.java b/ai/src/main/java/com/instana/dc/ai/DataCollector.java index a34710f..0891777 100644 --- a/ai/src/main/java/com/instana/dc/ai/DataCollector.java +++ b/ai/src/main/java/com/instana/dc/ai/DataCollector.java @@ -80,51 +80,11 @@ private void startCollect() { } static public class CustomDcConfig { - @JsonProperty("llm.platform") - private String platform; @JsonProperty("llm.application") private String application; - @JsonProperty("server.address") - private String serverAddress; - @JsonProperty("server.port") - private String port; - @JsonProperty("server.scheme") - private String scheme; private final List> instances = new ArrayList<>(); - public void setPlatform(String platform) { - this.platform = platform; - } - - public String getPlatform() { - return platform; - } - - public void setServerAddr(String serverAddress) { - this.serverAddress = serverAddress; - } - - public String getServerAddr() { - return serverAddress; - } - - public void setServerPort(String port) { - this.port = port; - } - - public String getServerPort() { - return port; - } - - public void setScheme(String scheme) { - this.scheme = scheme; - } - - public String getScheme() { - return scheme; - } - public void setApplication(String application) { this.application = application; } @@ -137,5 +97,4 @@ public List> getInstances() { return instances; } } - } diff --git a/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java b/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java index d248020..dae1d22 100644 --- a/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java +++ b/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java @@ -11,10 +11,6 @@ public class LLMDcUtil { /* Configurations for the Data Collector: */ - public static final String LLM_SERVER_ADDR = "server.address"; - public static final String LLM_SERVER_PORT = "server.port"; - public static final String LLM_SERVER_SCHEME = "server.scheme"; - public static final String LLM_PLATFORM_NAME = "llm.platform"; public static final String DEFAULT_INSTRUMENTATION_SCOPE = "instana.sensor-sdk.dc.llm"; public static final String DEFAULT_INSTRUMENTATION_SCOPE_VER = "1.0.0"; public static final String SERVICE_NAME = "service.name"; From ce900c1d83756828a8fb8c79de411ae158cba10b Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Mon, 6 May 2024 02:48:09 -0700 Subject: [PATCH 10/13] Update version and README for installation without build --- ai/CHANGELOG.md | 2 +- ai/DEVELOP.md | 60 +++++++++++++++++++++++++++++++++++++++ ai/README.md | 66 +++++++++++++++---------------------------- ai/build.gradle | 2 +- ai/config/config.yaml | 6 ++-- 5 files changed, 88 insertions(+), 48 deletions(-) create mode 100644 ai/DEVELOP.md diff --git a/ai/CHANGELOG.md b/ai/CHANGELOG.md index 5fe38fa..5fe930e 100644 --- a/ai/CHANGELOG.md +++ b/ai/CHANGELOG.md @@ -1,4 +1,4 @@ Changelog ========== -## Version 0.0.1 +## Version 1.0.0 diff --git a/ai/DEVELOP.md b/ai/DEVELOP.md new file mode 100644 index 0000000..e9a9408 --- /dev/null +++ b/ai/DEVELOP.md @@ -0,0 +1,60 @@ +# ODCAI (OTel based Data Collector for AI) + +**[Semantic Convention](docs/semconv)** | +**[Support](docs/support/README.md)** | +**[Changelog](CHANGELOG.md)** | +**[Contributing](CONTRIBUTING.md)** | +**[License](LICENSE)** + +--- +ODCAI (OTel based Data Collector for AI) is the tool or template to generate OpenTelemetry metrics for various +LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. + +## Requirements + +- Java 11+ + +## Build & Installation + +1) Make sure Java SDK 11+ is installed. +```bash +java -version +``` + +2) Get the source code from `github.ibm.com`. +```bash +git clone https://github.ibm.com/instana-llmetry/otel-dc.git +cd otel-dc/ai +``` + +3) Build with Gradle +```bash +./gradlew clean build +``` +*Note: gradle 7.4 will be installed if you do not have it.* + +## Run ODCAI + +1) Make sure code is built with Java SDK 11+. + +2) Refine configuration file (config/config.yaml) according to your own LLM application. + +3) Start up your OTLP backend which accept OTLP connections. Right now we support following protocols: +- otlp/grpc + +1) Option-1: Run the Data Collector with gradle +```bash +./gradlew run +``` +1) Option-2: Extract deployment package generated by gradle: +```bash +tar vxf otel-dc-ai-*.tar +cd otel-dc-ai-* +``` + +Then, user can create the proper configuration files: + - config/config.yaml + - config/logging.properties + +Run the Data Collector with following command according to your current OS: + - bin/odcai diff --git a/ai/README.md b/ai/README.md index ada8d4a..fecd1ec 100644 --- a/ai/README.md +++ b/ai/README.md @@ -1,64 +1,44 @@ # ODCAI (OTel based Data Collector for AI) -**[Semantic Convention](docs/semconv)** | -**[Support](docs/support/README.md)** | -**[Changelog](CHANGELOG.md)** | -**[Contributing](CONTRIBUTING.md)** | -**[License](LICENSE)** - ---- -ODCAI (OTel based Data Collector for AI) is the tool or template to generate OpenTelemetry metrics for various -LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. +ODCAI (OTel based Data Collector for AI) is the tool or template to generate OpenTelemetry metrics for various LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. ## Requirements - Java 11+ -## Build & Installation - -1) Make sure Java SDK 11+ is installed. +Ensure that Java SDK 11+ is installed. ```bash java -version ``` -2) Get the source code from `github.ibm.com`. -```bash -git clone https://github.ibm.com/instana-llmetry/otel-dc.git -cd otel-dc/ai -``` +## Installation -3) Build with Gradle +1) Download the installation package: ```bash -./gradlew clean build +curl -O https://github.com/instana/otel-dc/releases/download/Release/otel-dc-ai_1.0.0_linux_amd64.tar.gz ``` -*Note: gradle 7.4 will be installed if you do not have it.* - -## Run ODCAI - -1) Make sure code is built with Java SDK 11+. - -2) Refine configuration file (config/config.yaml) according to your own LLM application. - -3) Start up your OTLP backend which accept OTLP connections. Right now we support following protocols: -- otlp/grpc - -1) Option-1: Run the Data Collector with gradle +2) Extract the package to the desired deployment location: ```bash -./gradlew run +tar vxf otel-dc-ai_1.0.0_linux_amd64.tar.gz ``` -1) Option-2: Extract deployment package generated by gradle: + +## Configuration ```bash -tar vxf otel-dc-ai-*.tar -cd otel-dc-ai-* +cd otel-dc-ai-1.0.0 +vi config/config.yaml ``` +The following options are required： +- `otel.backend.url`：The OTel gRPC address of the agent, for example: http://localhost:4317 +- `otel.service.name`：The Data Collector name, which can be any string you choose. +- `price.prompt.tokens.per.kilo`：The unit price per thousand prompt tokens. +- `price.complete.tokens.per.kilo`：The unit price per thousand complete tokens. -Then, user can create the proper configuration files: - - config/config.yaml - - config/logging.properties - -Run the Data Collector with following command according to your current OS: - - bin/odcai -## Create a new data collector +## Run ODCAI +Run the Data Collector with the following command according to your current system: +```bash +nohup ./bin/otel-dc-ai & +``` -Please refer to "[How to create a new data collector](docs/developer/new-llm.md)". \ No newline at end of file +## Reference +If your platform is not supported by the pre-built binaries or if you prefer to compile from source, you can follow the steps in [DEVELOP.md](DEVELOP.md) diff --git a/ai/build.gradle b/ai/build.gradle index d071e06..cdbc53a 100644 --- a/ai/build.gradle +++ b/ai/build.gradle @@ -4,7 +4,7 @@ plugins { } group = "com.instana.dc" -version = "0.0.2" +version = "1.0.0" sourceCompatibility = 11 targetCompatibility = 11 diff --git a/ai/config/config.yaml b/ai/config/config.yaml index 3ace1ff..21d833f 100644 --- a/ai/config/config.yaml +++ b/ai/config/config.yaml @@ -2,6 +2,6 @@ llm.application: LLM_DC instances: - otel.backend.url: http://localhost:4317 - otel.service.name: WatsonxDC - price.prompt.tokens.per.kilo: 0.3 - price.complete.tokens.per.kilo: 0.3 + otel.service.name: LLMDC1 + price.prompt.tokens.per.kilo: 0.0 + price.complete.tokens.per.kilo: 0.0 From 7fb88b362437f0b506a4cdd8a32ed8b363550b9f Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Tue, 7 May 2024 14:30:27 -0700 Subject: [PATCH 11/13] Rename the name and subpath from ai to llm --- README.md | 14 +++++++------- ai/settings.gradle | 2 -- {ai => llm}/CHANGELOG.md | 0 {ai => llm}/DEVELOP.md | 14 +++++++------- {ai => llm}/README.md | 14 +++++++------- {ai => llm}/architecture.md | 6 +++--- {ai => llm}/build.gradle | 2 +- {ai => llm}/config/config.yaml | 0 {ai => llm}/config/logging.properties | 2 +- {ai => llm}/gradle/wrapper/gradle-wrapper.jar | Bin .../gradle/wrapper/gradle-wrapper.properties | 0 {ai => llm}/gradlew | 0 {ai => llm}/gradlew.bat | 0 {ai => llm}/libs/otel-dc-0.9.7.jar | Bin llm/settings.gradle | 2 ++ .../java/com/instana/dc/llm}/AbstractLLMDc.java | 6 +++--- .../java/com/instana/dc/llm}/DataCollector.java | 4 ++-- .../java/com/instana/dc/llm}/LLMDcRegistry.java | 6 +++--- .../main/java/com/instana/dc/llm}/LLMDcUtil.java | 2 +- .../com/instana/dc/llm}/LLMRawMetricRegistry.java | 4 ++-- .../java/com/instana/dc/llm}/impl/llm/LLMDc.java | 10 +++++----- .../dc/llm}/impl/llm/MetricsCollectorService.java | 2 +- 22 files changed, 45 insertions(+), 45 deletions(-) delete mode 100644 ai/settings.gradle rename {ai => llm}/CHANGELOG.md (100%) rename {ai => llm}/DEVELOP.md (87%) rename {ai => llm}/README.md (81%) rename {ai => llm}/architecture.md (79%) rename {ai => llm}/build.gradle (96%) rename {ai => llm}/config/config.yaml (100%) rename {ai => llm}/config/logging.properties (90%) rename {ai => llm}/gradle/wrapper/gradle-wrapper.jar (100%) rename {ai => llm}/gradle/wrapper/gradle-wrapper.properties (100%) rename {ai => llm}/gradlew (100%) rename {ai => llm}/gradlew.bat (100%) rename {ai => llm}/libs/otel-dc-0.9.7.jar (100%) create mode 100644 llm/settings.gradle rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/AbstractLLMDc.java (96%) rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/DataCollector.java (97%) rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/LLMDcRegistry.java (87%) rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/LLMDcUtil.java (99%) rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/LLMRawMetricRegistry.java (94%) rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/impl/llm/LLMDc.java (97%) rename {ai/src/main/java/com/instana/dc/ai => llm/src/main/java/com/instana/dc/llm}/impl/llm/MetricsCollectorService.java (99%) diff --git a/README.md b/README.md index 065dea9..28ddc90 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ ODCD (OpenTelemetry based Data Collector for Telemetry Data) is a collection of ### Requirements - Data Collectors for Host **Java 8+** - Data Collectors for Databases **Java 8+** -- Data Collectors for AI **Java 11+** +- Data Collectors for LLM **Java 11+** ## Build & Installation @@ -38,11 +38,11 @@ cd otel-dc/host ```bash cd otel-dc/rdb ``` -- Data Collectors for AI +- Data Collectors for LLM ```bash -cd otel-dc/ai +cd otel-dc/llm ``` -3) Build with Gradle +1) Build with Gradle ```bash ./gradlew clean build ``` @@ -115,17 +115,17 @@ export JAVA_OPTS=-Dconfig/logging.properties nohup bin/otel-dc-host > /dev/null 2>&1 & ``` -- **ai** +- **llm** ```bash export DC_CONFIG=config/config.yaml export JAVA_OPTS=-Dconfig/logging.properties -bin/otel-dc-ai +bin/otel-dc-llm ``` Or run Data Collector in background ```bash export DC_CONFIG=config/config.yaml export JAVA_OPTS=-Dconfig/logging.properties -nohup bin/otel-dc-ai > /dev/null 2>&1 & +nohup bin/otel-dc-llm > /dev/null 2>&1 & ``` *Note:* diff --git a/ai/settings.gradle b/ai/settings.gradle deleted file mode 100644 index e5704f3..0000000 --- a/ai/settings.gradle +++ /dev/null @@ -1,2 +0,0 @@ -rootProject.name = 'otel-dc-ai' - diff --git a/ai/CHANGELOG.md b/llm/CHANGELOG.md similarity index 100% rename from ai/CHANGELOG.md rename to llm/CHANGELOG.md diff --git a/ai/DEVELOP.md b/llm/DEVELOP.md similarity index 87% rename from ai/DEVELOP.md rename to llm/DEVELOP.md index e9a9408..6055e1b 100644 --- a/ai/DEVELOP.md +++ b/llm/DEVELOP.md @@ -1,4 +1,4 @@ -# ODCAI (OTel based Data Collector for AI) +# ODCL (OTel based Data Collector for LLM) **[Semantic Convention](docs/semconv)** | **[Support](docs/support/README.md)** | @@ -7,7 +7,7 @@ **[License](LICENSE)** --- -ODCAI (OTel based Data Collector for AI) is the tool or template to generate OpenTelemetry metrics for various +ODCL (OTel based Data Collector for LLM) is the tool or template to generate OpenTelemetry metrics for various LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. ## Requirements @@ -24,7 +24,7 @@ java -version 2) Get the source code from `github.ibm.com`. ```bash git clone https://github.ibm.com/instana-llmetry/otel-dc.git -cd otel-dc/ai +cd otel-dc/llm ``` 3) Build with Gradle @@ -33,7 +33,7 @@ cd otel-dc/ai ``` *Note: gradle 7.4 will be installed if you do not have it.* -## Run ODCAI +## Run ODCL 1) Make sure code is built with Java SDK 11+. @@ -48,8 +48,8 @@ cd otel-dc/ai ``` 1) Option-2: Extract deployment package generated by gradle: ```bash -tar vxf otel-dc-ai-*.tar -cd otel-dc-ai-* +tar vxf otel-dc-llm-*.tar +cd otel-dc-llm-* ``` Then, user can create the proper configuration files: @@ -57,4 +57,4 @@ Then, user can create the proper configuration files: - config/logging.properties Run the Data Collector with following command according to your current OS: - - bin/odcai + - bin/odcl diff --git a/ai/README.md b/llm/README.md similarity index 81% rename from ai/README.md rename to llm/README.md index fecd1ec..3fffac2 100644 --- a/ai/README.md +++ b/llm/README.md @@ -1,6 +1,6 @@ -# ODCAI (OTel based Data Collector for AI) +# ODCL (OTel based Data Collector for LLM) -ODCAI (OTel based Data Collector for AI) is the tool or template to generate OpenTelemetry metrics for various LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. +ODCL (OTel based Data Collector for LLM) is the tool or template to generate OpenTelemetry metrics for various LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. ## Requirements @@ -15,16 +15,16 @@ java -version 1) Download the installation package: ```bash -curl -O https://github.com/instana/otel-dc/releases/download/Release/otel-dc-ai_1.0.0_linux_amd64.tar.gz +curl -O https://github.com/instana/otel-dc/releases/download/Release/otel-dc-llm_1.0.0_linux_amd64.tar.gz ``` 2) Extract the package to the desired deployment location: ```bash -tar vxf otel-dc-ai_1.0.0_linux_amd64.tar.gz +tar vxf otel-dc-llm_1.0.0_linux_amd64.tar.gz ``` ## Configuration ```bash -cd otel-dc-ai-1.0.0 +cd otel-dc-llm-1.0.0 vi config/config.yaml ``` The following options are required： @@ -34,10 +34,10 @@ The following options are required： - `price.complete.tokens.per.kilo`：The unit price per thousand complete tokens. -## Run ODCAI +## Run ODCL Run the Data Collector with the following command according to your current system: ```bash -nohup ./bin/otel-dc-ai & +nohup ./bin/otel-dc-llm & ``` ## Reference diff --git a/ai/architecture.md b/llm/architecture.md similarity index 79% rename from ai/architecture.md rename to llm/architecture.md index bf8f950..232b79a 100644 --- a/ai/architecture.md +++ b/llm/architecture.md @@ -1,14 +1,14 @@ -# ODCAI (OTel based Data Collector for AI) +# ODCL (OTel based Data Collector for LLM) --- -ODCAI (OTel based Data Collector for AI) is the tool or template to generate OpenTelemetry metrics for various +ODCL (OTel based Data Collector for LLM) is the tool or template to generate OpenTelemetry metrics for various LLM and LLM Applications. All implementation are based on predefined OpenTelemetry Semantic Conventions. ## Runtime Architecture ![image](https://media.github.ibm.com/user/104429/files/7e1ba353-477c-448f-aeda-94ee69827586) -## AI Data Collectors +## LLM Data Collectors Watsonx: https://github.ibm.com/instana-llmetry/llm-dc-otel/tree/main/watsonx-dc OpenAI: TBD diff --git a/ai/build.gradle b/llm/build.gradle similarity index 96% rename from ai/build.gradle rename to llm/build.gradle index cdbc53a..6c8650d 100644 --- a/ai/build.gradle +++ b/llm/build.gradle @@ -34,7 +34,7 @@ dependencies { } application { - mainClass = 'com.instana.dc.ai.DataCollector' + mainClass = 'com.instana.dc.llm.DataCollector' } applicationDistribution.from("config") { diff --git a/ai/config/config.yaml b/llm/config/config.yaml similarity index 100% rename from ai/config/config.yaml rename to llm/config/config.yaml diff --git a/ai/config/logging.properties b/llm/config/logging.properties similarity index 90% rename from ai/config/logging.properties rename to llm/config/logging.properties index 91585c0..9dcf7bf 100644 --- a/ai/config/logging.properties +++ b/llm/config/logging.properties @@ -6,7 +6,7 @@ java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter # default file output is in user's home directory. java.util.logging.FileHandler.level=INFO -java.util.logging.FileHandler.pattern = %h/ai-dc%u.log +java.util.logging.FileHandler.pattern = %h/llm-dc%u.log java.util.logging.FileHandler.limit = 500000 java.util.logging.FileHandler.count = 10 java.util.logging.FileHandler.formatter = java.util.logging.XMLFormatter \ No newline at end of file diff --git a/ai/gradle/wrapper/gradle-wrapper.jar b/llm/gradle/wrapper/gradle-wrapper.jar similarity index 100% rename from ai/gradle/wrapper/gradle-wrapper.jar rename to llm/gradle/wrapper/gradle-wrapper.jar diff --git a/ai/gradle/wrapper/gradle-wrapper.properties b/llm/gradle/wrapper/gradle-wrapper.properties similarity index 100% rename from ai/gradle/wrapper/gradle-wrapper.properties rename to llm/gradle/wrapper/gradle-wrapper.properties diff --git a/ai/gradlew b/llm/gradlew similarity index 100% rename from ai/gradlew rename to llm/gradlew diff --git a/ai/gradlew.bat b/llm/gradlew.bat similarity index 100% rename from ai/gradlew.bat rename to llm/gradlew.bat diff --git a/ai/libs/otel-dc-0.9.7.jar b/llm/libs/otel-dc-0.9.7.jar similarity index 100% rename from ai/libs/otel-dc-0.9.7.jar rename to llm/libs/otel-dc-0.9.7.jar diff --git a/llm/settings.gradle b/llm/settings.gradle new file mode 100644 index 0000000..ddb3111 --- /dev/null +++ b/llm/settings.gradle @@ -0,0 +1,2 @@ +rootProject.name = 'otel-dc-llm' + diff --git a/ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java b/llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java similarity index 96% rename from ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java rename to llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java index f9b2529..5cc152e 100644 --- a/ai/src/main/java/com/instana/dc/ai/AbstractLLMDc.java +++ b/llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java @@ -2,10 +2,10 @@ * (c) Copyright IBM Corp. 2023 * (c) Copyright Instana Inc. */ -package com.instana.dc.ai; +package com.instana.dc.llm; import com.instana.dc.AbstractDc; -import com.instana.dc.ai.DataCollector.CustomDcConfig; +import com.instana.dc.llm.DataCollector.CustomDcConfig; import com.instana.dc.resources.ContainerResource; import io.opentelemetry.api.OpenTelemetry; import io.opentelemetry.api.common.Attributes; @@ -22,7 +22,7 @@ import java.util.logging.Logger; import static com.instana.dc.DcUtil.*; -import static com.instana.dc.ai.LLMDcUtil.*; +import static com.instana.dc.llm.LLMDcUtil.*; import static io.opentelemetry.api.common.AttributeKey.stringKey; public abstract class AbstractLLMDc extends AbstractDc { private static final Logger logger = Logger.getLogger(AbstractLLMDc.class.getName()); diff --git a/ai/src/main/java/com/instana/dc/ai/DataCollector.java b/llm/src/main/java/com/instana/dc/llm/DataCollector.java similarity index 97% rename from ai/src/main/java/com/instana/dc/ai/DataCollector.java rename to llm/src/main/java/com/instana/dc/llm/DataCollector.java index 0891777..ca807dc 100644 --- a/ai/src/main/java/com/instana/dc/ai/DataCollector.java +++ b/llm/src/main/java/com/instana/dc/llm/DataCollector.java @@ -2,13 +2,13 @@ * (c) Copyright IBM Corp. 2023 * (c) Copyright Instana Inc. */ -package com.instana.dc.ai; +package com.instana.dc.llm; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.instana.dc.IDc; -import com.instana.dc.ai.LLMDcRegistry; +import com.instana.dc.llm.LLMDcRegistry; import java.io.File; import java.util.ArrayList; diff --git a/ai/src/main/java/com/instana/dc/ai/LLMDcRegistry.java b/llm/src/main/java/com/instana/dc/llm/LLMDcRegistry.java similarity index 87% rename from ai/src/main/java/com/instana/dc/ai/LLMDcRegistry.java rename to llm/src/main/java/com/instana/dc/llm/LLMDcRegistry.java index 1a18125..c62c71a 100644 --- a/ai/src/main/java/com/instana/dc/ai/LLMDcRegistry.java +++ b/llm/src/main/java/com/instana/dc/llm/LLMDcRegistry.java @@ -2,10 +2,10 @@ * (c) Copyright IBM Corp. 2023 * (c) Copyright Instana Inc. */ -package com.instana.dc.ai; +package com.instana.dc.llm; -import com.instana.dc.ai.AbstractLLMDc; -import com.instana.dc.ai.impl.llm.LLMDc; +import com.instana.dc.llm.AbstractLLMDc; +import com.instana.dc.llm.impl.llm.LLMDc; import java.util.HashMap; import java.util.Map; diff --git a/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java b/llm/src/main/java/com/instana/dc/llm/LLMDcUtil.java similarity index 99% rename from ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java rename to llm/src/main/java/com/instana/dc/llm/LLMDcUtil.java index dae1d22..463db2b 100644 --- a/ai/src/main/java/com/instana/dc/ai/LLMDcUtil.java +++ b/llm/src/main/java/com/instana/dc/llm/LLMDcUtil.java @@ -2,7 +2,7 @@ * (c) Copyright IBM Corp. 2023 * (c) Copyright Instana Inc. */ -package com.instana.dc.ai; +package com.instana.dc.llm; //import static com.instana.agent.sensorsdk.semconv.SemanticAttributes.*; diff --git a/ai/src/main/java/com/instana/dc/ai/LLMRawMetricRegistry.java b/llm/src/main/java/com/instana/dc/llm/LLMRawMetricRegistry.java similarity index 94% rename from ai/src/main/java/com/instana/dc/ai/LLMRawMetricRegistry.java rename to llm/src/main/java/com/instana/dc/llm/LLMRawMetricRegistry.java index c115e8e..d66d828 100644 --- a/ai/src/main/java/com/instana/dc/ai/LLMRawMetricRegistry.java +++ b/llm/src/main/java/com/instana/dc/llm/LLMRawMetricRegistry.java @@ -2,7 +2,7 @@ * (c) Copyright IBM Corp. 2023 * (c) Copyright Instana Inc. */ -package com.instana.dc.ai; +package com.instana.dc.llm; import com.instana.dc.RawMetric; @@ -11,7 +11,7 @@ import static com.instana.dc.InstrumentType.GAUGE; import static com.instana.dc.InstrumentType.UPDOWN_COUNTER; -import static com.instana.dc.ai.LLMDcUtil.*; +import static com.instana.dc.llm.LLMDcUtil.*; public class LLMRawMetricRegistry { private final Map map = new ConcurrentHashMap() {{ diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java b/llm/src/main/java/com/instana/dc/llm/impl/llm/LLMDc.java similarity index 97% rename from ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java rename to llm/src/main/java/com/instana/dc/llm/impl/llm/LLMDc.java index b0b1b7d..e0f4eab 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/LLMDc.java +++ b/llm/src/main/java/com/instana/dc/llm/impl/llm/LLMDc.java @@ -2,11 +2,11 @@ * (c) Copyright IBM Corp. 2023 * (c) Copyright Instana Inc. */ -package com.instana.dc.ai.impl.llm; +package com.instana.dc.llm.impl.llm; -import com.instana.dc.ai.AbstractLLMDc; -import com.instana.dc.ai.DataCollector.CustomDcConfig; -import com.instana.dc.ai.impl.llm.MetricsCollectorService.OtelMetric; +import com.instana.dc.llm.AbstractLLMDc; +import com.instana.dc.llm.DataCollector.CustomDcConfig; +import com.instana.dc.llm.impl.llm.MetricsCollectorService.OtelMetric; import java.util.logging.Logger; import java.util.*; @@ -20,7 +20,7 @@ import com.linecorp.armeria.server.healthcheck.HealthCheckService; //import static com.instana.agent.sensorsdk.semconv.SemanticAttributes.*; -import static com.instana.dc.ai.LLMDcUtil.*; +import static com.instana.dc.llm.LLMDcUtil.*; @SuppressWarnings("null") public class LLMDc extends AbstractLLMDc { diff --git a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java b/llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java similarity index 99% rename from ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java rename to llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java index 148563e..2d2eaeb 100644 --- a/ai/src/main/java/com/instana/dc/ai/impl/llm/MetricsCollectorService.java +++ b/llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java @@ -1,4 +1,4 @@ -package com.instana.dc.ai.impl.llm; +package com.instana.dc.llm.impl.llm; import com.google.common.collect.ImmutableList; import io.grpc.stub.StreamObserver; From e7a795e596cc5a173906cf55863a97ec88276d2b Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Tue, 7 May 2024 16:44:20 -0700 Subject: [PATCH 12/13] Refine the name for the infrastructure --- llm/config/config.yaml | 2 +- llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/llm/config/config.yaml b/llm/config/config.yaml index 21d833f..eb08d6a 100644 --- a/llm/config/config.yaml +++ b/llm/config/config.yaml @@ -2,6 +2,6 @@ llm.application: LLM_DC instances: - otel.backend.url: http://localhost:4317 - otel.service.name: LLMDC1 + otel.service.name: DC1 price.prompt.tokens.per.kilo: 0.0 price.complete.tokens.per.kilo: 0.0 diff --git a/llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java b/llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java index 5cc152e..b60fd69 100644 --- a/llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java +++ b/llm/src/main/java/com/instana/dc/llm/AbstractLLMDc.java @@ -59,7 +59,7 @@ public AbstractLLMDc(Map properties, CustomDcConfig cdcConfig) { otelBackendUrl = (String) properties.get(OTEL_BACKEND_URL); otelUsingHttp = (Boolean) properties.getOrDefault(OTEL_BACKEND_USING_HTTP, Boolean.FALSE); serviceName = (String) properties.get(OTEL_SERVICE_NAME); - serviceInstanceId = "LLMONITOR:" + serviceName + "@" + getHostName(); + serviceInstanceId = serviceName + "@" + getHostName(); this.cdcConfig = cdcConfig; } @@ -67,6 +67,7 @@ public AbstractLLMDc(Map properties, CustomDcConfig cdcConfig) { public Resource getResourceAttributes() { Resource resource = Resource.getDefault() .merge(Resource.create(Attributes.of( + stringKey("llm.platform"), "LLM", stringKey(SERVICE_NAME), serviceName, stringKey(SERVICE_INSTANCE_ID), serviceInstanceId ))) From b8b59d22039f65e5b2700b05f2112d3a1b9e5b04 Mon Sep 17 00:00:00 2001 From: JINSONG WANG Date: Tue, 7 May 2024 22:08:39 -0700 Subject: [PATCH 13/13] Add OpenAI metrics collections --- .../instana/dc/llm/impl/llm/MetricsCollectorService.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java b/llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java index 2d2eaeb..5dfa13d 100644 --- a/llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java +++ b/llm/src/main/java/com/instana/dc/llm/impl/llm/MetricsCollectorService.java @@ -116,7 +116,8 @@ public void export( switch (metric.getDataCase()) { case SUM: - if (metric.getName().compareTo("llm.watsonx.completions.tokens") == 0) { + if (metric.getName().compareTo("llm.watsonx.completions.tokens") == 0 || + metric.getName().compareTo("llm.openai.chat_completions.tokens") == 0) { List sumDataPoints = metric.getSum().getDataPointsList(); for (NumberDataPoint dataPoint : sumDataPoints) { @@ -157,7 +158,8 @@ public void export( } break; case HISTOGRAM: - if (metric.getName().compareTo("llm.watsonx.completions.duration") == 0) { + if (metric.getName().compareTo("llm.watsonx.completions.duration") == 0 || + metric.getName().compareTo("llm.openai.chat_completions.duration") == 0) { List histDataPoints = metric.getHistogram().getDataPointsList(); for (HistogramDataPoint dataPoint : histDataPoints) {