feat(tracing/sql): emit breadcrumbs

request_logger.go

@@ -35,16 +35,6 @@ func RequestLogger(l zerolog.Logger, requestHeaders, responseHeaders bool) fiber
 	reqCh := make(chan request, reqLoggerBufSize)
 	resCh := make(chan response, reqLoggerBufSize)
 
-	formatHeaderAttribute := func(t, k string, v []string, i int) string {
-		attr := fmt.Sprintf("%s.headers.%s", t, k)
-
-		if len(v) != 1 {
-			attr = fmt.Sprintf("%s.%d", attr, i)
-		}
-
-		return attr
-	}
-
 	go func() {
 		for {
 			select {
@@ -170,3 +160,13 @@ func RequestLogger(l zerolog.Logger, requestHeaders, responseHeaders bool) fiber
 		return err
 	}
 }
+
+func formatHeaderAttribute(t, k string, v []string, i int) string {
+	attr := fmt.Sprintf("%s.headers.%s", t, k)
+
+	if len(v) != 1 {
+		attr = fmt.Sprintf("%s.%d", attr, i)
+	}
+
+	return attr
+}

span.go

@@ -116,6 +116,77 @@ func (s *Span) CaptureMessage(message string) *Span {
 	return s
 }
 
+type (
+	SeverityLevel  string
+	BreadcrumbType string
+)
+
+const (
+	SeverityDebug   SeverityLevel = "debug"
+	SeverityInfo    SeverityLevel = "info"
+	SeverityWarning SeverityLevel = "warning"
+	SeverityError   SeverityLevel = "error"
+	SeverityFatal   SeverityLevel = "fatal"
+
+	BreadcrumbTypeDefault BreadcrumbType = "default"
+	// BreadcrumbCatagoryConsole is rendered as a BreadcrumbTypeDebug
+	BreadcrumbCatagoryConsole string = "console"
+
+	BreadcrumbTypeDebug BreadcrumbType = "debug"
+
+	BreadcrumbTypeError     BreadcrumbType = "error"
+	BreadcrumbCatagoryError string         = "error"
+
+	// BreadcrumbTypeNavigration requires the `from` and `to` attributes
+	BreadcrumbTypeNavigration     BreadcrumbType = "navigation"
+	BreadcrumbCatagoryNavigration string         = "navigation"
+
+	// BreadcrumbTypeHTTP requires the `url`, `method` and `status_code` attributes
+	BreadcrumbTypeHTTP     BreadcrumbType = "http"
+	BreadcrumbCatagoryHTTP string         = "http"
+
+	BreadcrumbTypeInfo BreadcrumbType = "info"
+
+	BreadcrumbTypeQuery BreadcrumbType = "query"
+
+	BreadcrumbTypeTransaction           BreadcrumbType = "transaction"
+	BreadcrumbCategorySentryTransaction string         = "sentry.transaction"
+
+	// BreadcrumbTypeUI is also applicable for all `ui.*` categories
+	BreadcrumbTypeUI BreadcrumbType = "ui"
+
+	BreadcrumbTypeUser      BreadcrumbType = "user"
+	BreadcrumbCategoryClick string         = "click"
+)
+
+func (s *Span) CaptureBreadcrumb(level SeverityLevel, type_ BreadcrumbType, category string, message string, data map[string]any) *Span {
+	now := time.Now()
+
+	if hub := sentry.GetHubFromContext(s.Context()); hub != nil {
+		hub.AddBreadcrumb(&sentry.Breadcrumb{
+			Level:     sentry.Level(level),
+			Type:      string(type_),
+			Category:  category,
+			Message:   message,
+			Data:      data,
+			Timestamp: now,
+		}, nil)
+	}
+
+	attrs := []attribute.KeyValue{
+		attribute.String("category", category),
+		attribute.String("type", string(type_)),
+		attribute.String("level", string(level)),
+	}
+	for k, v := range data {
+		attrs = append(attrs, attribute.String(k, fmt.Sprintf("%v", v)))
+	}
+
+	s.otel.AddEvent(message, trace.WithAttributes(attrs...), trace.WithTimestamp(now))
+
+	return s
+}
+
 func (s *Span) SetName(name string) *Span {
 	s.otel.SetName(name)
 	s.sentry.Name = name

sql_tracing.go

@@ -15,6 +15,8 @@ import (
 	"go.opentelemetry.io/otel/trace"
 )
 
+// TODO: Port to github.com/loghole/dbhook for proper error handling
+
 const (
 	sqlClientID = libBase + "#TracedSQL"
 )
@@ -79,16 +81,17 @@ func (h *tracedSQLHooks) Before(ctx context.Context, query string, args ...inter
 }
 
 func (h *tracedSQLHooks) After(ctx context.Context, query string, args ...interface{}) (context.Context, error) {
-	if s := SpanFromContext(ctx); s != nil {
-		s.End()
-	}
-
 	if val, ok := ctx.Value(dbCtxKey).(dbCtxVal); ok {
 		attrs := val.attrs
 
 		h.mDuration.Record(ctx, float64(time.Since(val.start).Milliseconds()), metric.WithAttributes(attrs...))
 	}
 
+	if s := SpanFromContext(ctx); s != nil {
+		s.CaptureBreadcrumb(SeverityDebug, BreadcrumbTypeQuery, "started", query, map[string]any{"args": args}).
+			End()
+	}
+
 	return ctx, nil
 }
 

traced_transport.go

@@ -35,14 +35,14 @@ func (t *Telemetry) NewTracedTransport(transport http.RoundTripper, forwardTrace
 }
 
 func (t *HTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
-	span := t.telemetry.
+	s := t.telemetry.
 		StartSpan(req.Context(), "http.client", fmt.Sprintf("%s %s", req.Method, req.URL), WithOtelOptions(trace.WithSpanKind(trace.SpanKindClient))).
 		AddAttributes(
 			semconv.HTTPRequestMethodKey.String(req.Method),
 			semconv.URLFull(req.URL.String()),
 		)
-	defer span.End()
+	defer s.End()
-	ctx := span.Context()
+	ctx := s.Context()
 	req = req.WithContext(ctx)
 
 	if t.forwardTrace {
@@ -50,13 +50,19 @@ func (t *HTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	}
 
 	for _, header := range t.tracedRequestHeaders {
-		if v := req.Header.Get(header); v != "" {
+		vv := req.Header.Values(header)
-			span.AddAttributes(attribute.String(fmt.Sprintf("http.request.header.%s", header), v))
+		for i, v := range vv {
+			s.AddAttributes(attribute.String(formatHeaderAttribute("http.request", header, vv, i), v))
 		}
 	}
 
 	resp, err := t.Transport.RoundTrip(req)
 	if err != nil {
+		s.CaptureBreadcrumb(SeverityError, BreadcrumbTypeHTTP, BreadcrumbCatagoryHTTP, fmt.Sprintf("Failed to send request to %s: %v", req.URL.String(), err), map[string]any{
+			"url":    req.URL.String(),
+			"method": req.Method,
+		})
+
 		if hub := sentry.GetHubFromContext(ctx); hub != nil {
 			hub.CaptureException(err)
 		}
@@ -64,13 +70,30 @@ func (t *HTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 		return resp, err
 	}
 
-	span.
+	{
+		severity := SeverityDebug
+		if resp.StatusCode < http.StatusBadRequest {
+			severity = SeverityInfo
+		} else if resp.StatusCode < http.StatusInternalServerError {
+			severity = SeverityError
+		} else {
+			severity = SeverityError
+		}
+		s.CaptureBreadcrumb(severity, BreadcrumbTypeHTTP, BreadcrumbCatagoryHTTP, req.URL.String(), map[string]any{
+			"url":    req.URL.String(),
+			"method": req.Method,
+			"status": resp.StatusCode,
+		})
+	}
+
+	s.
 		AddAttributes(semconv.HTTPResponseStatusCode(resp.StatusCode)).
 		SetStatus(httpStatusToSpanStatus(resp.StatusCode, false), "")
 
 	for _, header := range t.tracedResponseHeaders {
-		if v := resp.Header.Get(header); v != "" {
+		vv := resp.Header.Values(header)
-			span.AddAttributes(attribute.String(fmt.Sprintf("http.response.header.%s", header), v))
+		for i, v := range vv {
+			s.AddAttributes(attribute.String(formatHeaderAttribute("http.response", header, vv, i), v))
 		}
 	}