is there any way to change the value of timestamp field?

[1123183721]

A note for the community

• Please vote on this issue by adding a 👍 reaction to the original issue to help the community and maintainers prioritize this request
• If you are interested in working on this issue or have submitted a pull request, please leave a comment

Problem

The loki record value timestamp and timestamp in message are different, is there any way to change the value of timestamp?

Configuration

data_dir = "/vector-data-dir"
# I'm afraid to open it in case it triggers some bugs.
schema.log_namespace = false
api.enabled = true
api.address = "0.0.0.0:8686"
[sources.in]
type = "kafka"
bootstrap_servers = "xxxxx"
topics = ["a"]
group_id = "xxx"
decoding.codec = "json"
[sources.in.librdkafka_options]
"allow.auto.create.topics" = "false"
"auto.offset.reset" = "earliest"

[transforms.trans]
type = "remap"
inputs = ["in"]
# event value is a json, e.g., {"message": "hello", "@collect_time":  "2024-06-06T07:09:23.015146042Z"}
source = '''
        .timestamp=to_unix_timestamp(parse_timestamp!(.@collect_time, format: "%F T %T%.9f Z"), unit: "nanoseconds")
    '''

[sinks.out]
type = "loki"
inputs = ["trans"]
endpoint = "http://xxxxx"
encoding.codec = "json"
compression = "snappy"
encoding.only_fields = ["message", "timestamp"]
out_of_order_action = "accept"
remove_label_fields = true
remove_timestamp = false

[sinks.out.labels]
job = "test"

Version

0.38.0

Debug Output

No response

Example Data

{
    "status": "success",
    "data": {
        "resultType": "streams",
        "result": [
            {
                "stream": {
                    "job": "test"
                },
                "values": [
                    [
                        "1717658642800327233",
                        "{\"message\":\"hello\",\"timestamp\":1717657763015146042}"
                    ]
                ]
            }
        ],
        "stats": {
            "summary": {
                "bytesProcessedPerSecond": 6652,
                "linesProcessedPerSecond": 130,
                "totalBytesProcessed": 51,
                "totalLinesProcessed": 1,
                "execTime": 0.007666,
                "queueTime": 0.000344,
                "subqueries": 0,
                "totalEntriesReturned": 1,
                "splits": 2,
                "shards": 0,
                "totalPostFilterLines": 1,
                "totalStructuredMetadataBytesProcessed": 0
            },
            "querier": {
                "store": {
                    "totalChunksRef": 0,
                    "totalChunksDownloaded": 0,
                    "chunksDownloadTime": 0,
                    "chunk": {
                        "headChunkBytes": 0,
                        "headChunkLines": 0,
                        "decompressedBytes": 0,
                        "decompressedLines": 0,
                        "compressedBytes": 0,
                        "totalDuplicates": 0,
                        "postFilterLines": 0,
                        "headChunkStructuredMetadataBytes": 0,
                        "decompressedStructuredMetadataBytes": 0
                    },
                    "chunkRefsFetchTime": 2816982
                }
            },
            "ingester": {
                "totalReached": 6,
                "totalChunksMatched": 1,
                "totalBatches": 7,
                "totalLinesSent": 1,
                "store": {
                    "totalChunksRef": 0,
                    "totalChunksDownloaded": 0,
                    "chunksDownloadTime": 0,
                    "chunk": {
                        "headChunkBytes": 51,
                        "headChunkLines": 1,
                        "decompressedBytes": 0,
                        "decompressedLines": 0,
                        "compressedBytes": 0,
                        "totalDuplicates": 0,
                        "postFilterLines": 1,
                        "headChunkStructuredMetadataBytes": 0,
                        "decompressedStructuredMetadataBytes": 0
                    },
                    "chunkRefsFetchTime": 0
                }
            },
            "cache": {
                "chunk": {
                    "entriesFound": 0,
                    "entriesRequested": 0,
                    "entriesStored": 0,
                    "bytesReceived": 0,
                    "bytesSent": 0,
                    "requests": 0,
                    "downloadTime": 0
                },
                "index": {
                    "entriesFound": 0,
                    "entriesRequested": 0,
                    "entriesStored": 0,
                    "bytesReceived": 0,
                    "bytesSent": 0,
                    "requests": 0,
                    "downloadTime": 0
                },
                "result": {
                    "entriesFound": 0,
                    "entriesRequested": 0,
                    "entriesStored": 0,
                    "bytesReceived": 0,
                    "bytesSent": 0,
                    "requests": 0,
                    "downloadTime": 0
                },
                "statsResult": {
                    "entriesFound": 0,
                    "entriesRequested": 0,
                    "entriesStored": 0,
                    "bytesReceived": 0,
                    "bytesSent": 0,
                    "requests": 0,
                    "downloadTime": 0
                }
            }
        }
    }
}

Additional Context

No response

References

No response

[jszwedko]

I think you may need to remove the to_unix_timestamp call so that .timestamp is an actual timestamp type and not an integer.

[1123183721]

I read the code, just like you said. Looks like it's time to learn rust.