Skip to content

Commit a636758

Browse files
kruskallaxw
kruskall
and
axw
authored
feat: update protobuf models to use uint64 for timestamps (#136)
* feat: update protobuf models to use uint64 * feat: regenerate protobuf models * feat: update code use new uint64 * fix: support larger time range Co-authored-by: Andrew Wilkins <axwalk@gmail.com> --------- Co-authored-by: Andrew Wilkins <axwalk@gmail.com>
1 parent 5085cc4 commit a636758

31 files changed

Lines changed: 430 additions & 555 deletions

‎codec/fullevent_test.go‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,12 +23,11 @@ import (
2323

2424
"github.com/elastic/apm-data/model/modelpb"
2525
"google.golang.org/protobuf/types/known/durationpb"
26-
"google.golang.org/protobuf/types/known/timestamppb"
2726
)
2827

2928
func fullEvent(t *testing.B) *modelpb.APMEvent {
3029
return &modelpb.APMEvent{
31-
Timestamp: timestamppb.New(time.Unix(1, 1)),
30+
Timestamp: uint64(time.Second.Nanoseconds() + 1),
3231
Span: &modelpb.Span{
3332
Message: &modelpb.Message{
3433
Body: "body",

‎input/elasticapm/internal/modeldecoder/modeldecodertest/populator.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -122,7 +122,7 @@ func SetStructValues(in interface{}, values *Values, opts ...SetStructValuesOpti
122122
switch fKind := f.Kind(); fKind {
123123
case reflect.String:
124124
fieldVal = reflect.ValueOf(values.Str)
125-
case reflect.Int, reflect.Int32, reflect.Int64:
125+
case reflect.Int, reflect.Int32, reflect.Int64, reflect.Uint64:
126126
fieldVal = reflect.ValueOf(values.Int).Convert(f.Type())
127127
case reflect.Float64:
128128
fieldVal = reflect.ValueOf(values.Float).Convert(f.Type())

‎input/elasticapm/internal/modeldecoder/rumv3/decoder.go‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,6 @@ import (
3232
"github.com/elastic/apm-data/input/elasticapm/internal/modeldecoder/nullable"
3333
"github.com/elastic/apm-data/model/modelpb"
3434
"google.golang.org/protobuf/types/known/durationpb"
35-
"google.golang.org/protobuf/types/known/timestamppb"
3635
)
3736

3837
var (
@@ -248,7 +247,7 @@ func mapToErrorModel(from *errorEvent, event *modelpb.APMEvent) {
248247
event.ParentId = from.ParentID.Val
249248
}
250249
if !from.Timestamp.Val.IsZero() {
251-
event.Timestamp = timestamppb.New(from.Timestamp.Val)
250+
event.Timestamp = modelpb.FromTime(from.Timestamp.Val)
252251
}
253252
if from.TraceID.IsSet() {
254253
event.Trace = &modelpb.Trace{
@@ -628,9 +627,7 @@ func mapToSpanModel(from *span, event *modelpb.APMEvent) {
628627
if from.Start.IsSet() {
629628
// event.Timestamp is initialized to the time the payload was
630629
// received; offset that by "start" milliseconds for RUM.
631-
event.Timestamp = timestamppb.New(event.Timestamp.AsTime().Add(
632-
time.Duration(float64(time.Millisecond) * from.Start.Val),
633-
))
630+
event.Timestamp += uint64(time.Duration(float64(time.Millisecond) * from.Start.Val).Nanoseconds())
634631
}
635632
}
636633

‎input/elasticapm/internal/modeldecoder/rumv3/error_test.go‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import (
2828
"github.com/stretchr/testify/assert"
2929
"github.com/stretchr/testify/require"
3030
"google.golang.org/protobuf/testing/protocmp"
31-
"google.golang.org/protobuf/types/known/timestamppb"
3231

3332
"github.com/elastic/apm-data/input/elasticapm/internal/decoder"
3433
"github.com/elastic/apm-data/input/elasticapm/internal/modeldecoder"
@@ -47,17 +46,17 @@ func TestResetErrorOnRelease(t *testing.T) {
4746

4847
func TestDecodeNestedError(t *testing.T) {
4948
t.Run("decode", func(t *testing.T) {
50-
now := time.Now().UTC()
49+
now := modelpb.FromTime(time.Now())
5150
eventBase := initializedMetadata()
52-
eventBase.Timestamp = timestamppb.New(now)
51+
eventBase.Timestamp = now
5352
input := modeldecoder.Input{Base: eventBase}
5453
str := `{"e":{"id":"a-b-c","timestamp":1599996822281000,"log":{"mg":"abc"}}}`
5554
dec := decoder.NewJSONDecoder(strings.NewReader(str))
5655
var batch modelpb.Batch
5756
require.NoError(t, DecodeNestedError(dec, &input, &batch))
5857
require.Len(t, batch, 1)
5958
require.NotNil(t, batch[0].Error)
60-
assert.Equal(t, time.Unix(1599996822, 281000000).UTC(), batch[0].Timestamp.AsTime())
59+
assert.Equal(t, modelpb.FromTime(time.Unix(1599996822, 281000000)), batch[0].Timestamp)
6160
assert.Empty(t, cmp.Diff(&modelpb.Error{
6261
Id: "a-b-c",
6362
Log: &modelpb.ErrorLog{
@@ -72,7 +71,7 @@ func TestDecodeNestedError(t *testing.T) {
7271
dec = decoder.NewJSONDecoder(strings.NewReader(str))
7372
batch = modelpb.Batch{}
7473
require.NoError(t, DecodeNestedError(dec, &input, &batch))
75-
assert.Equal(t, now, batch[0].Timestamp.AsTime())
74+
assert.Equal(t, now, batch[0].Timestamp)
7675

7776
// test decode
7877
err := DecodeNestedError(decoder.NewJSONDecoder(strings.NewReader(`malformed`)), &input, &batch)
@@ -146,24 +145,24 @@ func TestDecodeMapToErrorModel(t *testing.T) {
146145
}
147146
var input errorEvent
148147
var out1, out2 modelpb.APMEvent
149-
reqTime := time.Now().Add(time.Second).UTC()
150-
out1.Timestamp = timestamppb.New(reqTime)
148+
reqTime := modelpb.FromTime(time.Now().Add(time.Second))
149+
out1.Timestamp = reqTime
151150
defaultVal := modeldecodertest.DefaultValues()
152151
modeldecodertest.SetStructValues(&input, defaultVal)
153152
mapToErrorModel(&input, &out1)
154153
input.Reset()
155154
modeldecodertest.AssertStructValues(t, out1.Error, exceptions, defaultVal)
156155

157156
// leave event timestamp unmodified if eventTime is zero
158-
out1.Timestamp = timestamppb.New(reqTime)
157+
out1.Timestamp = reqTime
159158
modeldecodertest.SetStructValues(&input, defaultVal)
160159
mapToErrorModel(&input, &out1)
161160
input.Reset()
162161
modeldecodertest.AssertStructValues(t, out1.Error, exceptions, defaultVal)
163162

164163
// reuse input model for different event
165164
// ensure memory is not shared by reusing input model
166-
out2.Timestamp = timestamppb.New(reqTime)
165+
out2.Timestamp = reqTime
167166
otherVal := modeldecodertest.NonDefaultValues()
168167
modeldecodertest.SetStructValues(&input, otherVal)
169168
mapToErrorModel(&input, &out2)

‎input/elasticapm/internal/modeldecoder/rumv3/transaction_test.go‎

Lines changed: 14 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import (
2828
"github.com/stretchr/testify/assert"
2929
"github.com/stretchr/testify/require"
3030
"google.golang.org/protobuf/testing/protocmp"
31-
"google.golang.org/protobuf/types/known/timestamppb"
3231

3332
"github.com/elastic/apm-data/input/elasticapm/internal/decoder"
3433
"github.com/elastic/apm-data/input/elasticapm/internal/modeldecoder"
@@ -47,9 +46,9 @@ func TestResetTransactionOnRelease(t *testing.T) {
4746

4847
func TestDecodeNestedTransaction(t *testing.T) {
4948
t.Run("decode", func(t *testing.T) {
50-
now := time.Now().UTC()
49+
now := modelpb.FromTime(time.Now())
5150
eventBase := initializedMetadata()
52-
eventBase.Timestamp = timestamppb.New(now)
51+
eventBase.Timestamp = now
5352
input := modeldecoder.Input{Base: eventBase}
5453
str := `{"x":{"n":"tr-a","d":100,"id":"100","tid":"1","t":"request","yc":{"sd":2},"y":[{"n":"a","d":10,"t":"http","id":"123","s":20}],"me":[{"sa":{"ysc":{"v":5}},"y":{"t":"span_type","su":"span_subtype"}}]}}`
5554
dec := decoder.NewJSONDecoder(strings.NewReader(str))
@@ -62,7 +61,7 @@ func TestDecodeNestedTransaction(t *testing.T) {
6261

6362
assert.Equal(t, "request", batch[0].Transaction.Type)
6463
// fall back to request time
65-
assert.Equal(t, now, batch[0].Timestamp.AsTime())
64+
assert.Equal(t, now, batch[0].Timestamp)
6665

6766
// Ensure nested metricsets are decoded. RUMv3 only sends
6867
// breakdown metrics, so the Metricsets will be empty and
@@ -78,11 +77,11 @@ func TestDecodeNestedTransaction(t *testing.T) {
7877
Subtype: "span_subtype",
7978
SelfTime: &modelpb.AggregatedDuration{Count: 5},
8079
}, batch[1].Span, protocmp.Transform()))
81-
assert.Equal(t, now, batch[1].Timestamp.AsTime())
80+
assert.Equal(t, now, batch[1].Timestamp)
8281

8382
// ensure nested spans are decoded
84-
start := time.Duration(20 * 1000 * 1000)
85-
assert.Equal(t, now.Add(start), batch[2].Timestamp.AsTime()) // add start to timestamp
83+
start := uint64(time.Duration(20 * 1000 * 1000).Nanoseconds())
84+
assert.Equal(t, now+start, batch[2].Timestamp) // add start to timestamp
8685
assert.Equal(t, "100", batch[2].Transaction.Id)
8786
assert.Equal(t, "1", batch[2].Trace.Id)
8887
assert.Equal(t, "100", batch[2].ParentId)
@@ -93,8 +92,8 @@ func TestDecodeNestedTransaction(t *testing.T) {
9392
})
9493

9594
t.Run("decode-marks", func(t *testing.T) {
96-
now := time.Now()
97-
eventBase := modelpb.APMEvent{Timestamp: timestamppb.New(now)}
95+
now := modelpb.FromTime(time.Now())
96+
eventBase := modelpb.APMEvent{Timestamp: now}
9897
input := modeldecoder.Input{Base: &eventBase}
9998
str := `{"x":{"d":100,"id":"100","tid":"1","t":"request","yc":{"sd":2},"k":{"a":{"dc":0.1,"di":0.2,"ds":0.3,"de":0.4,"fb":0.5,"fp":0.6,"lp":0.7,"long":0.8},"nt":{"fs":0.1,"ls":0.2,"le":0.3,"cs":0.4,"ce":0.5,"qs":0.6,"rs":0.7,"re":0.8,"dl":0.9,"di":0.11,"ds":0.21,"de":0.31,"dc":0.41,"es":0.51,"ee":6,"long":0.99},"long":{"long":0.1}}}}`
10099
dec := decoder.NewJSONDecoder(strings.NewReader(str))
@@ -215,16 +214,16 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
215214

216215
var input transaction
217216
var out1, out2 modelpb.APMEvent
218-
reqTime := time.Now().Add(time.Second)
219-
out1.Timestamp = timestamppb.New(reqTime)
217+
reqTime := modelpb.FromTime(time.Now().Add(time.Second))
218+
out1.Timestamp = reqTime
220219
defaultVal := modeldecodertest.DefaultValues()
221220
modeldecodertest.SetStructValues(&input, defaultVal)
222221
mapToTransactionModel(&input, &out1)
223222
input.Reset()
224223
modeldecodertest.AssertStructValues(t, out1.Transaction, exceptions, defaultVal)
225224

226225
// ensure memory is not shared by reusing input model
227-
out2.Timestamp = timestamppb.New(reqTime)
226+
out2.Timestamp = reqTime
228227
otherVal := modeldecodertest.NonDefaultValues()
229228
modeldecodertest.SetStructValues(&input, otherVal)
230229
mapToTransactionModel(&input, &out2)
@@ -267,16 +266,16 @@ func TestDecodeMapToTransactionModel(t *testing.T) {
267266

268267
var input span
269268
var out1, out2 modelpb.APMEvent
270-
reqTime := time.Now().Add(time.Second)
271-
out1.Timestamp = timestamppb.New(reqTime)
269+
reqTime := modelpb.FromTime(time.Now().Add(time.Second))
270+
out1.Timestamp = reqTime
272271
defaultVal := modeldecodertest.DefaultValues()
273272
modeldecodertest.SetStructValues(&input, defaultVal)
274273
mapToSpanModel(&input, &out1)
275274
input.Reset()
276275
modeldecodertest.AssertStructValues(t, out1.Span, exceptions, defaultVal)
277276

278277
// ensure memory is not shared by reusing input model
279-
out2.Timestamp = timestamppb.New(reqTime)
278+
out2.Timestamp = reqTime
280279
otherVal := modeldecodertest.NonDefaultValues()
281280
modeldecodertest.SetStructValues(&input, otherVal)
282281
mapToSpanModel(&input, &out2)

‎input/elasticapm/internal/modeldecoder/v2/decoder.go‎

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ import (
3737
"github.com/elastic/apm-data/input/otlp"
3838
"github.com/elastic/apm-data/model/modelpb"
3939
"google.golang.org/protobuf/types/known/durationpb"
40-
"google.golang.org/protobuf/types/known/timestamppb"
4140

4241
"go.opentelemetry.io/collector/pdata/pcommon"
4342
"go.opentelemetry.io/collector/pdata/ptrace"
@@ -470,7 +469,7 @@ func mapToErrorModel(from *errorEvent, event *modelpb.APMEvent) {
470469
event.ParentId = from.ParentID.Val
471470
}
472471
if !from.Timestamp.Val.IsZero() {
473-
event.Timestamp = timestamppb.New(from.Timestamp.Val)
472+
event.Timestamp = modelpb.FromTime(from.Timestamp.Val)
474473
}
475474
if from.TraceID.IsSet() {
476475
event.Trace = &modelpb.Trace{
@@ -731,7 +730,7 @@ func mapToMetricsetModel(from *metricset, event *modelpb.APMEvent) bool {
731730
event.Metricset = &modelpb.Metricset{Name: "app"}
732731

733732
if !from.Timestamp.Val.IsZero() {
734-
event.Timestamp = timestamppb.New(from.Timestamp.Val)
733+
event.Timestamp = modelpb.FromTime(from.Timestamp.Val)
735734
}
736735

737736
if len(from.Samples) > 0 {
@@ -1179,18 +1178,12 @@ func mapToSpanModel(from *span, event *modelpb.APMEvent) {
11791178
out.Sync = &val
11801179
}
11811180
if !from.Timestamp.Val.IsZero() {
1182-
event.Timestamp = timestamppb.New(from.Timestamp.Val)
1181+
event.Timestamp = modelpb.FromTime(from.Timestamp.Val)
11831182
} else if from.Start.IsSet() {
11841183
// event.Timestamp should have been initialized to the time the
11851184
// payload was received; offset that by "start" milliseconds for
11861185
// RUM.
1187-
base := time.Time{}
1188-
if event.Timestamp != nil {
1189-
base = event.Timestamp.AsTime()
1190-
}
1191-
event.Timestamp = timestamppb.New(base.Add(
1192-
time.Duration(float64(time.Millisecond) * from.Start.Val),
1193-
))
1186+
event.Timestamp += uint64(time.Duration(float64(time.Millisecond) * from.Start.Val).Nanoseconds())
11941187
}
11951188
if from.TraceID.IsSet() {
11961189
event.Trace = &modelpb.Trace{
@@ -1411,7 +1404,7 @@ func mapToTransactionModel(from *transaction, event *modelpb.APMEvent) {
14111404
out.SpanCount.Started = &started
14121405
}
14131406
if !from.Timestamp.Val.IsZero() {
1414-
event.Timestamp = timestamppb.New(from.Timestamp.Val)
1407+
event.Timestamp = modelpb.FromTime(from.Timestamp.Val)
14151408
}
14161409
if from.TraceID.IsSet() {
14171410
event.Trace = &modelpb.Trace{
@@ -1467,7 +1460,7 @@ func mapToLogModel(from *log, event *modelpb.APMEvent) {
14671460
mapToFAASModel(from.FAAS, event.Faas)
14681461
}
14691462
if !from.Timestamp.Val.IsZero() {
1470-
event.Timestamp = timestamppb.New(from.Timestamp.Val)
1463+
event.Timestamp = modelpb.FromTime(from.Timestamp.Val)
14711464
}
14721465
if from.TraceID.IsSet() {
14731466
event.Trace = &modelpb.Trace{

‎input/elasticapm/internal/modeldecoder/v2/error_test.go‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import (
2828
"github.com/stretchr/testify/assert"
2929
"github.com/stretchr/testify/require"
3030
"google.golang.org/protobuf/testing/protocmp"
31-
"google.golang.org/protobuf/types/known/timestamppb"
3231

3332
"github.com/elastic/apm-data/input/elasticapm/internal/decoder"
3433
"github.com/elastic/apm-data/input/elasticapm/internal/modeldecoder"
@@ -47,18 +46,18 @@ func TestResetErrorOnRelease(t *testing.T) {
4746

4847
func TestDecodeNestedError(t *testing.T) {
4948
t.Run("decode", func(t *testing.T) {
50-
now := time.Now().UTC()
49+
now := modelpb.FromTime(time.Now())
5150
defaultVal := modeldecodertest.DefaultValues()
5251
_, eventBase := initializedInputMetadata(defaultVal)
53-
eventBase.Timestamp = timestamppb.New(now)
52+
eventBase.Timestamp = now
5453
input := modeldecoder.Input{Base: eventBase}
5554
str := `{"error":{"id":"a-b-c","timestamp":1599996822281000,"log":{"message":"abc"}}}`
5655
dec := decoder.NewJSONDecoder(strings.NewReader(str))
5756
var batch modelpb.Batch
5857
require.NoError(t, DecodeNestedError(dec, &input, &batch))
5958
require.Len(t, batch, 1)
6059
require.NotNil(t, batch[0].Error)
61-
assert.Equal(t, time.Unix(1599996822, 281000000).UTC(), batch[0].Timestamp.AsTime())
60+
assert.Equal(t, modelpb.FromTime(time.Unix(1599996822, 281000000)), batch[0].Timestamp)
6261
assert.Empty(t, cmp.Diff(&modelpb.Error{
6362
Id: "a-b-c",
6463
Log: &modelpb.ErrorLog{Message: "abc"},
@@ -69,7 +68,7 @@ func TestDecodeNestedError(t *testing.T) {
6968
batch = modelpb.Batch{}
7069
require.NoError(t, DecodeNestedError(dec, &input, &batch))
7170
// if no timestamp is provided, leave base event time unmodified
72-
assert.Equal(t, now, batch[0].Timestamp.AsTime())
71+
assert.Equal(t, now, batch[0].Timestamp)
7372

7473
err := DecodeNestedError(decoder.NewJSONDecoder(strings.NewReader(`malformed`)), &input, &batch)
7574
require.Error(t, err)

‎input/elasticapm/internal/modeldecoder/v2/log_test.go‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@ import (
2424

2525
"github.com/stretchr/testify/assert"
2626
"github.com/stretchr/testify/require"
27-
"google.golang.org/protobuf/types/known/timestamppb"
2827

2928
"github.com/elastic/apm-data/input/elasticapm/internal/decoder"
3029
"github.com/elastic/apm-data/input/elasticapm/internal/modeldecoder"
@@ -50,7 +49,7 @@ func TestDecodeNestedLog(t *testing.T) {
5049
require.NoError(t, DecodeNestedLog(dec, &input, &batch))
5150
require.Len(t, batch, 1)
5251
assert.Equal(t, "something happened", batch[0].Message)
53-
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", batch[0].Timestamp.AsTime().String())
52+
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", modelpb.ToTime(batch[0].Timestamp).String())
5453
assert.Equal(t, "trace-id", batch[0].Trace.Id)
5554
assert.Equal(t, "transaction-id", batch[0].Transaction.Id)
5655
assert.Equal(t, "warn", batch[0].Log.Level)
@@ -68,13 +67,13 @@ func TestDecodeNestedLog(t *testing.T) {
6867
})
6968

7069
t.Run("withoutTimestamp", func(t *testing.T) {
71-
now := time.Now().UTC()
72-
input := modeldecoder.Input{Base: &modelpb.APMEvent{Timestamp: timestamppb.New(now)}}
70+
now := modelpb.FromTime(time.Now())
71+
input := modeldecoder.Input{Base: &modelpb.APMEvent{Timestamp: now}}
7372
str := `{"log":{"message":"something happened"}}`
7473
dec := decoder.NewJSONDecoder(strings.NewReader(str))
7574
var batch modelpb.Batch
7675
require.NoError(t, DecodeNestedLog(dec, &input, &batch))
77-
assert.Equal(t, now, batch[0].Timestamp.AsTime())
76+
assert.Equal(t, now, batch[0].Timestamp)
7877
})
7978

8079
t.Run("withError", func(t *testing.T) {
@@ -84,7 +83,7 @@ func TestDecodeNestedLog(t *testing.T) {
8483
var batch modelpb.Batch
8584
require.NoError(t, DecodeNestedLog(dec, &input, &batch))
8685
require.Len(t, batch, 1)
87-
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", batch[0].Timestamp.AsTime().String())
86+
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", modelpb.ToTime(batch[0].Timestamp).String())
8887
assert.Equal(t, "trace-id", batch[0].Trace.Id)
8988
assert.Equal(t, "transaction-id", batch[0].Transaction.Id)
9089
assert.Equal(t, "error", batch[0].Log.Level)
@@ -111,7 +110,7 @@ func TestDecodeNestedLog(t *testing.T) {
111110
var batch modelpb.Batch
112111
require.NoError(t, DecodeNestedLog(dec, &input, &batch))
113112
require.Len(t, batch, 1)
114-
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", batch[0].Timestamp.AsTime().String())
113+
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", modelpb.ToTime(batch[0].Timestamp).String())
115114
assert.Equal(t, "trace-id", batch[0].Trace.Id)
116115
assert.Equal(t, "transaction-id", batch[0].Transaction.Id)
117116
assert.Equal(t, "error", batch[0].Log.Level)
@@ -138,7 +137,7 @@ func TestDecodeNestedLog(t *testing.T) {
138137
var batch modelpb.Batch
139138
require.NoError(t, DecodeNestedLog(dec, &input, &batch))
140139
require.Len(t, batch, 1)
141-
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", batch[0].Timestamp.AsTime().String())
140+
assert.Equal(t, "2022-09-08 06:02:51 +0000 UTC", modelpb.ToTime(batch[0].Timestamp).String())
142141
assert.Equal(t, "trace-id", batch[0].Trace.Id)
143142
assert.Equal(t, "transaction-id", batch[0].Transaction.Id)
144143
assert.Equal(t, "error", batch[0].Log.Level)

0 commit comments

Comments
 (0)