diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..4cf2ac0 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +gitstreams diff --git a/OTEL_SETUP.md b/OTEL_SETUP.md new file mode 100644 index 0000000..65da2d4 --- /dev/null +++ b/OTEL_SETUP.md @@ -0,0 +1,139 @@ +# OpenTelemetry Instrumentation + +gitstreams includes optional OpenTelemetry instrumentation to help monitor and understand the sync operation performance. + +## Features + +The instrumentation provides visibility into: +- Overall sync operation duration +- Time to fetch followed users +- Per-user API call timings (starred repos, owned repos, events) +- Pagination details (pages fetched, items per page) + +## Setup + +### Prerequisites + +You'll need an OpenTelemetry collector running. Here's a quick setup using Docker: + +1. Create a `docker-compose.yml` file: + +```yaml +version: '3' +services: + otel-collector: + image: otel/opentelemetry-collector:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4318:4318" # OTLP HTTP receiver + - "13133:13133" # Health check + - "55679:55679" # zpages + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" # Jaeger UI + - "14250:14250" # gRPC receiver +``` + +2. Create `otel-collector-config.yaml`: + +```yaml +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + debug: + verbosity: detailed + jaeger: + endpoint: jaeger:14250 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug, jaeger] +``` + +3. Start the collector: + +```bash +docker-compose up -d +``` + +### Running gitstreams with OpenTelemetry + +Set the OTEL environment variable to enable tracing: + +```bash +export GITHUB_TOKEN=your_token_here +export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4318 +export OTEL_SERVICE_NAME=gitstreams # Optional, defaults to "gitstreams" + +./gitstreams +``` + +### Viewing Traces + +Open Jaeger UI in your browser: + +``` +http://localhost:16686 +``` + +Select the "gitstreams" service and explore the traces to see: +- Total sync operation time +- Per-user processing time +- API call breakdowns +- Pagination details + +## Environment Variables + +- `OTEL_EXPORTER_OTLP_ENDPOINT`: OTLP endpoint (required to enable tracing) + - Format: `hostname:port` (e.g., `localhost:4318`) +- `OTEL_SERVICE_NAME`: Service name for traces (optional, defaults to "gitstreams") + +## Spans + +### Top-level spans + +- `fetchActivity`: Overall sync operation + - Attributes: `user_count` (number of users followed) + +### Per-user spans + +- `fetchUserActivity`: Processing a single user + - Attributes: `user` (username) +- `getStarredRepos`: Fetching starred repositories + - Attributes: `user` (username) +- `getOwnedRepos`: Fetching owned repositories + - Attributes: `user` (username) +- `getRecentEvents`: Fetching recent events + - Attributes: `user` (username) + +### Pagination spans + +- `github.getPaginated`: Overall pagination operation + - Attributes: `path` (API path), `total_pages`, `total_results` +- `github.fetchPage`: Individual page fetch + - Attributes: `path` (full path with params), `page` (page number), `results` (items in page) + +## Example: Analyzing Performance + +After running with OTEL enabled, you can: + +1. Identify slow users: Look at `fetchUserActivity` spans to see which users take longest +2. Understand pagination overhead: Check `github.fetchPage` spans to see how many pages are fetched +3. API rate limiting: Monitor timing patterns to detect rate limit delays +4. Overall performance: Track `fetchActivity` duration over time + +## Disabling Instrumentation + +Simply omit the `OTEL_EXPORTER_OTLP_ENDPOINT` environment variable. The instrumentation code will use a no-op tracer with zero overhead. diff --git a/README.md b/README.md index d4a95a2..c285d66 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,22 @@ The generated report includes: - **Relative timestamps** — "2 hours ago", "yesterday", "last week" - **Fun taglines** — dynamic header message based on activity volume +## OpenTelemetry Instrumentation (Optional) + +gitstreams includes optional OpenTelemetry instrumentation to monitor sync operation performance. Enable it by setting: + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT=localhost:4318 +``` + +See [OTEL_SETUP.md](OTEL_SETUP.md) for detailed setup instructions, including Docker Compose configuration for a local collector and Jaeger UI. + +Benefits: +- Monitor sync operation duration +- Track per-user API call timings +- Analyze pagination patterns +- Identify performance bottlenecks + ## Development ### Git Hooks (Recommended) diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..38e8d34 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,24 @@ +version: '3' +services: + otel-collector: + image: otel/opentelemetry-collector:latest + command: ["--config=/etc/otel-collector-config.yaml"] + volumes: + - ./otel-collector-config.yaml:/etc/otel-collector-config.yaml + ports: + - "4318:4318" # OTLP HTTP receiver + - "13133:13133" # Health check + - "55679:55679" # zpages + networks: + - otel + + jaeger: + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" # Jaeger UI + - "14250:14250" # gRPC receiver + networks: + - otel + +networks: + otel: diff --git a/github/client.go b/github/client.go index fab36e8..bd2397d 100644 --- a/github/client.go +++ b/github/client.go @@ -13,6 +13,10 @@ import ( "strings" "sync" "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) const defaultBaseURL = "https://api.github.com" @@ -297,6 +301,11 @@ func (c *Client) parseRateLimitHeaders(resp *http.Response) { // It handles GitHub's pagination by requesting 100 items per page until // no more results are returned. func (c *Client) getPaginated(ctx context.Context, basePath string, result any) error { + tracer := otel.Tracer("gitstreams") + ctx, span := tracer.Start(ctx, "github.getPaginated", + trace.WithAttributes(attribute.String("path", basePath))) + defer span.End() + // Use reflection to work with any slice type resultVal := reflect.ValueOf(result) if resultVal.Kind() != reflect.Ptr || resultVal.Elem().Kind() != reflect.Slice { @@ -316,16 +325,28 @@ func (c *Client) getPaginated(ctx context.Context, basePath string, result any) } path := fmt.Sprintf("%s%spage=%d&per_page=%d", basePath, separator, page, perPage) + // Create a span for this page + _, pageSpan := tracer.Start(ctx, "github.fetchPage", + trace.WithAttributes( + attribute.String("path", path), + attribute.Int("page", page))) + // Create a new slice to hold this page's results pageResult := reflect.New(sliceVal.Type()).Interface() if err := c.get(ctx, path, pageResult); err != nil { + pageSpan.RecordError(err) + pageSpan.End() + span.RecordError(err) return err } // Get the slice value from the pointer pageSlice := reflect.ValueOf(pageResult).Elem() + pageSpan.SetAttributes(attribute.Int("results", pageSlice.Len())) + pageSpan.End() + // If we got no results, we're done if pageSlice.Len() == 0 { break @@ -344,6 +365,9 @@ func (c *Client) getPaginated(ctx context.Context, basePath string, result any) // Set the final result resultVal.Elem().Set(sliceVal) + span.SetAttributes( + attribute.Int("total_pages", page), + attribute.Int("total_results", sliceVal.Len())) return nil } diff --git a/go.mod b/go.mod index 029a391..354ffea 100644 --- a/go.mod +++ b/go.mod @@ -4,16 +4,35 @@ go 1.24.1 require ( github.com/mattn/go-isatty v0.0.20 + go.opentelemetry.io/otel v1.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 + go.opentelemetry.io/otel/sdk v1.39.0 + go.opentelemetry.io/otel/trace v1.39.0 modernc.org/sqlite v1.44.3 ) require ( + github.com/cenkalti/backoff/v5 v5.0.3 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 // indirect + go.opentelemetry.io/otel/metric v1.39.0 // indirect + go.opentelemetry.io/proto/otlp v1.9.0 // indirect golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/net v0.47.0 // indirect + golang.org/x/sys v0.39.0 // indirect + golang.org/x/text v0.31.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect + google.golang.org/grpc v1.77.0 // indirect + google.golang.org/protobuf v1.36.10 // indirect modernc.org/libc v1.67.6 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.11.0 // indirect diff --git a/go.sum b/go.sum index 3f65330..b7b8e23 100644 --- a/go.sum +++ b/go.sum @@ -1,28 +1,85 @@ +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3 h1:NmZ1PKzSTQbuGHw9DGPFomqkkLWMC+vZCkfs+FHv1Vg= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.3/go.mod h1:zQrxl1YP88HQlA6i9c63DSVPFklWpGX4OWAc9bFuaH4= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.39.0 h1:8yPrr/S0ND9QEfTfdP9V+SiwT4E0G7Y5MO7p85nis48= +go.opentelemetry.io/otel v1.39.0/go.mod h1:kLlFTywNWrFyEdH0oj2xK0bFYZtHRYUdv1NklR/tgc8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0 h1:f0cb2XPmrqn4XMy9PNliTgRKJgS5WcL/u0/WRYGz4t0= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.39.0/go.mod h1:vnakAaFckOMiMtOIhFI2MNH4FYrZzXCYxmb1LlhoGz8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0 h1:Ckwye2FpXkYgiHX7fyVrN1uA/UYd9ounqqTuSNAv0k4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.39.0/go.mod h1:teIFJh5pW2y+AN7riv6IBPX2DuesS3HgP39mwOspKwU= +go.opentelemetry.io/otel/metric v1.39.0 h1:d1UzonvEZriVfpNKEVmHXbdf909uGTOQjA0HF0Ls5Q0= +go.opentelemetry.io/otel/metric v1.39.0/go.mod h1:jrZSWL33sD7bBxg1xjrqyDjnuzTUB0x1nBERXd7Ftcs= +go.opentelemetry.io/otel/sdk v1.39.0 h1:nMLYcjVsvdui1B/4FRkwjzoRVsMK8uL/cj0OyhKzt18= +go.opentelemetry.io/otel/sdk v1.39.0/go.mod h1:vDojkC4/jsTJsE+kh+LXYQlbL8CgrEcwmt1ENZszdJE= +go.opentelemetry.io/otel/sdk/metric v1.39.0 h1:cXMVVFVgsIf2YL6QkRF4Urbr/aMInf+2WKg+sEJTtB8= +go.opentelemetry.io/otel/sdk/metric v1.39.0/go.mod h1:xq9HEVH7qeX69/JnwEfp6fVq5wosJsY1mt4lLfYdVew= +go.opentelemetry.io/otel/trace v1.39.0 h1:2d2vfpEDmCJ5zVYz7ijaJdOF59xLomrvj7bjt6/qCJI= +go.opentelemetry.io/otel/trace v1.39.0/go.mod h1:88w4/PnZSazkGzz/w84VHpQafiU4EtqqlVdxWy+rNOA= +go.opentelemetry.io/proto/otlp v1.9.0 h1:l706jCMITVouPOqEnii2fIAuO3IVGBRPV5ICjceRb/A= +go.opentelemetry.io/proto/otlp v1.9.0/go.mod h1:xE+Cx5E/eEHw+ISFkwPLwCZefwVjY+pqKg1qcK03+/4= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY= golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70= golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA= golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= +golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY= +golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU= +golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I= +golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk= +golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM= +golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM= golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ= golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 h1:fCvbg86sFXwdrl5LgVcTEvNC+2txB5mgROGmRL5mrls= +google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:+rXWjjaukWZun3mLfjmVnQi18E1AsFbDN9QdJ5YXLto= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 h1:gRkg/vSppuSQoDjxyiGfN4Upv/h/DQmIR10ZU8dh4Ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217/go.mod h1:7i2o+ce6H/6BluujYR+kqX3GKH+dChPTQU19wjRPiGk= +google.golang.org/grpc v1.77.0 h1:wVVY6/8cGA6vvffn+wWK5ToddbgdU3d8MNENr4evgXM= +google.golang.org/grpc v1.77.0/go.mod h1:z0BY1iVj0q8E1uSQCjL9cppRj+gnZjzDnzV0dHhrNig= +google.golang.org/protobuf v1.36.10 h1:AYd7cD/uASjIL6Q9LiTjz8JLcrh/88q5UObnmY3aOOE= +google.golang.org/protobuf v1.36.10/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.27.1 h1:9W30zRlYrefrDV2JE2O8VDtJ1yPGownxciz5rrbQZis= modernc.org/cc/v4 v4.27.1/go.mod h1:uVtb5OGqUKpoLWhqwNQo/8LwvoiEBLvZXIQ/SmO6mL0= modernc.org/ccgo/v4 v4.30.1 h1:4r4U1J6Fhj98NKfSjnPUN7Ze2c6MnAdL0hWw6+LrJpc= diff --git a/main.go b/main.go index b3dffa3..20faab4 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "flag" "fmt" "io" + "log/slog" "os" "os/exec" "path/filepath" @@ -16,9 +17,12 @@ import ( "github.com/justinabrahms/gitstreams/diff" "github.com/justinabrahms/gitstreams/github" "github.com/justinabrahms/gitstreams/notify" + "github.com/justinabrahms/gitstreams/otel" "github.com/justinabrahms/gitstreams/progress" "github.com/justinabrahms/gitstreams/report" "github.com/justinabrahms/gitstreams/storage" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" ) const ( @@ -55,6 +59,8 @@ type Dependencies struct { ReportGenerator func() (ReportGenerator, error) OpenBrowser func(url string) error Now func() time.Time + Tracer trace.Tracer + Logger *slog.Logger } // GitHubClient defines the GitHub API operations we need. @@ -100,6 +106,8 @@ func DefaultDependencies() *Dependencies { }, OpenBrowser: openBrowser, Now: time.Now, + Tracer: otel.Tracer(), + Logger: slog.Default(), } } @@ -123,6 +131,21 @@ func run(stdout, stderr io.Writer, args []string, deps *Dependencies) int { return 1 } + // Initialize OpenTelemetry (optional, only if OTEL env vars are set) + ctx := context.Background() + _, cleanup, err := otel.Setup(ctx, deps.Logger) + if err != nil { + _, _ = fmt.Fprintf(stderr, "Warning: failed to initialize OpenTelemetry: %v\n", err) + // Don't fail on OTEL setup errors, just warn + } + defer func() { + if cleanup != nil { + if cleanupErr := cleanup(); cleanupErr != nil { + _, _ = fmt.Fprintf(stderr, "Warning: failed to cleanup OpenTelemetry: %v\n", cleanupErr) + } + } + }() + // Open storage first to support both live and historical modes store, err := deps.StoreFactory(cfg.DBPath) if err != nil { @@ -463,10 +486,19 @@ func parseSinceDate(dateStr string, now time.Time) (time.Time, error) { } func fetchActivity(ctx context.Context, client GitHubClient, now, cutoff time.Time, w, progressW io.Writer, verbose bool) (*diff.Snapshot, error) { + tracer := otel.Tracer() + ctx, span := tracer.Start(ctx, "fetchActivity") + defer span.End() + + // Fetch followed users + ctx, usersSpan := tracer.Start(ctx, "getFollowedUsers") users, err := client.GetFollowedUsers(ctx) + usersSpan.End() if err != nil { + span.RecordError(err) return nil, fmt.Errorf("fetching followed users: %w", err) } + span.SetAttributes(attribute.Int("user_count", len(users))) snapshot := diff.NewSnapshot(now) @@ -484,12 +516,19 @@ func fetchActivity(ctx context.Context, client GitHubClient, now, cutoff time.Ti _, _ = fmt.Fprintf(w, "Fetching activity for %s...\n", user.Login) } + // Create a span for this user's activity + _, userSpan := tracer.Start(ctx, "fetchUserActivity", + trace.WithAttributes(attribute.String("user", user.Login))) + activity := diff.UserActivity{ Username: user.Login, } // Fetch starred repos - filter by repo creation date + _, starredSpan := tracer.Start(ctx, "getStarredRepos", + trace.WithAttributes(attribute.String("user", user.Login))) starred, err := client.GetStarredReposByUsername(ctx, user.Login) + starredSpan.End() if err != nil { if verbose { _, _ = fmt.Fprintf(w, " Warning: could not fetch starred repos for %s: %v\n", user.Login, err) @@ -504,7 +543,10 @@ func fetchActivity(ctx context.Context, client GitHubClient, now, cutoff time.Ti } // Fetch owned repos - filter by creation or recent push date + _, ownedSpan := tracer.Start(ctx, "getOwnedRepos", + trace.WithAttributes(attribute.String("user", user.Login))) owned, err := client.GetOwnedReposByUsername(ctx, user.Login) + ownedSpan.End() if err != nil { if verbose { _, _ = fmt.Fprintf(w, " Warning: could not fetch owned repos for %s: %v\n", user.Login, err) @@ -519,7 +561,10 @@ func fetchActivity(ctx context.Context, client GitHubClient, now, cutoff time.Ti } // Fetch events - filter by event creation date + _, eventsSpan := tracer.Start(ctx, "getRecentEvents", + trace.WithAttributes(attribute.String("user", user.Login))) events, err := client.GetRecentEvents(ctx, user.Login) + eventsSpan.End() if err != nil { if verbose { _, _ = fmt.Fprintf(w, " Warning: could not fetch events for %s: %v\n", user.Login, err) @@ -534,6 +579,7 @@ func fetchActivity(ctx context.Context, client GitHubClient, now, cutoff time.Ti } snapshot.Users[user.Login] = activity + userSpan.End() } // Stop progress indicator diff --git a/main_test.go b/main_test.go index 0e5ffc5..c93e1bf 100644 --- a/main_test.go +++ b/main_test.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "log/slog" "path/filepath" "strings" "testing" @@ -14,6 +15,7 @@ import ( "github.com/justinabrahms/gitstreams/diff" "github.com/justinabrahms/gitstreams/github" "github.com/justinabrahms/gitstreams/notify" + "github.com/justinabrahms/gitstreams/otel" "github.com/justinabrahms/gitstreams/report" "github.com/justinabrahms/gitstreams/storage" ) @@ -192,6 +194,8 @@ func TestRun_SuccessfulRun_NoChanges(t *testing.T) { ReportGenerator: func() (ReportGenerator, error) { return mockGenInst, nil }, OpenBrowser: func(url string) error { browserOpened = true; return nil }, Now: fixedTime, + Tracer: otel.Tracer(), + Logger: slog.Default(), } result := run(&stdout, &stderr, []string{ diff --git a/otel-collector-config.yaml b/otel-collector-config.yaml new file mode 100644 index 0000000..ad9d6f8 --- /dev/null +++ b/otel-collector-config.yaml @@ -0,0 +1,19 @@ +receivers: + otlp: + protocols: + http: + endpoint: 0.0.0.0:4318 + +exporters: + debug: + verbosity: detailed + jaeger: + endpoint: jaeger:14250 + tls: + insecure: true + +service: + pipelines: + traces: + receivers: [otlp] + exporters: [debug, jaeger] diff --git a/otel/otel.go b/otel/otel.go new file mode 100644 index 0000000..6f1cebf --- /dev/null +++ b/otel/otel.go @@ -0,0 +1,106 @@ +// Package otel provides optional OpenTelemetry instrumentation for gitstreams. +package otel + +import ( + "context" + "fmt" + "log/slog" + "os" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" + "go.opentelemetry.io/otel/sdk/resource" + sdktrace "go.opentelemetry.io/otel/sdk/trace" + semconv "go.opentelemetry.io/otel/semconv/v1.21.0" + "go.opentelemetry.io/otel/trace" + "go.opentelemetry.io/otel/trace/noop" +) + +const ( + serviceName = "gitstreams" +) + +// Setup initializes OpenTelemetry if OTEL environment variables are configured. +// Returns a tracer provider and a cleanup function. If OTEL is not configured, +// returns a no-op tracer and a no-op cleanup function. +// +// The cleanup function should be called before the application exits to ensure +// all spans are exported. +// +// Environment variables: +// - OTEL_EXPORTER_OTLP_ENDPOINT: The OTLP endpoint (e.g., http://localhost:4318) +// - OTEL_SERVICE_NAME: Optional service name (defaults to "gitstreams") +func Setup(ctx context.Context, logger *slog.Logger) (trace.TracerProvider, func() error, error) { + // Use default logger if none provided + if logger == nil { + logger = slog.Default() + } + + // Check if OTEL is configured + endpoint := os.Getenv("OTEL_EXPORTER_OTLP_ENDPOINT") + if endpoint == "" { + logger.Debug("OpenTelemetry not configured (OTEL_EXPORTER_OTLP_ENDPOINT not set)") + return noop.NewTracerProvider(), func() error { return nil }, nil + } + + logger.Info("Initializing OpenTelemetry", + "endpoint", endpoint, + "service", serviceName, + ) + + // Create resource with service information + res, err := resource.New(ctx, + resource.WithAttributes( + semconv.ServiceName(getServiceName()), + ), + ) + if err != nil { + return nil, nil, fmt.Errorf("creating resource: %w", err) + } + + // Create OTLP HTTP exporter + exporter, err := otlptracehttp.New(ctx, + otlptracehttp.WithEndpoint(endpoint), + otlptracehttp.WithInsecure(), // For local development + ) + if err != nil { + return nil, nil, fmt.Errorf("creating OTLP exporter: %w", err) + } + + // Create tracer provider with batch span processor + tp := sdktrace.NewTracerProvider( + sdktrace.WithBatcher(exporter), + sdktrace.WithResource(res), + ) + + // Set global tracer provider + otel.SetTracerProvider(tp) + + // Return cleanup function + cleanup := func() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := tp.Shutdown(ctx); err != nil { + return fmt.Errorf("shutting down tracer provider: %w", err) + } + logger.Debug("OpenTelemetry tracer provider shut down") + return nil + } + + logger.Info("OpenTelemetry initialized successfully") + return tp, cleanup, nil +} + +// getServiceName returns the service name from environment or default. +func getServiceName() string { + if name := os.Getenv("OTEL_SERVICE_NAME"); name != "" { + return name + } + return serviceName +} + +// Tracer returns a tracer for the gitstreams package. +func Tracer() trace.Tracer { + return otel.Tracer(serviceName) +}