diff --git a/.dagger/.gitattributes b/.dagger/.gitattributes deleted file mode 100644 index 3a454933cb..0000000000 --- a/.dagger/.gitattributes +++ /dev/null @@ -1,4 +0,0 @@ -/dagger.gen.go linguist-generated -/internal/dagger/** linguist-generated -/internal/querybuilder/** linguist-generated -/internal/telemetry/** linguist-generated diff --git a/.dagger/.gitignore b/.dagger/.gitignore deleted file mode 100644 index 773338bfb9..0000000000 --- a/.dagger/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/dagger.gen.go -/internal/dagger -/internal/querybuilder -/internal/telemetry -/.env diff --git a/.dagger/build.go b/.dagger/build.go deleted file mode 100644 index ce7ff07720..0000000000 --- a/.dagger/build.go +++ /dev/null @@ -1,280 +0,0 @@ -package main - -import ( - "context" - "strings" - "time" - - "github.com/sourcegraph/conc/pool" - - "github.com/openmeterio/openmeter/.dagger/internal/dagger" -) - -// Build individual artifacts. (Useful for testing and development) -func (m *Openmeter) Build() *Build { - return &Build{ - Source: m.Source, - } -} - -type Build struct { - // +private - Source *dagger.Directory -} - -func (m *Build) All( - ctx context.Context, - - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) error { - p := pool.New().WithErrors().WithContext(ctx) - - p.Go(syncFunc(m.ContainerImage(platform))) - p.Go(syncFunc(m.HelmChart("openmeter", ""))) - p.Go(syncFunc(m.HelmChart("benthos-collector", ""))) - - return p.Wait() -} - -func (m *Build) containerImages(version string) []*dagger.Container { - platforms := []dagger.Platform{ - "linux/amd64", - "linux/arm64", - } - - variants := make([]*dagger.Container, 0, len(platforms)) - - for _, platform := range platforms { - variants = append(variants, m.containerImage(platform, version)) - } - - return variants -} - -// Build a container image. -func (m *Build) ContainerImage( - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) *dagger.Container { - return m.containerImage(platform, "") -} - -func (m *Build) containerImage(platform dagger.Platform, version string) *dagger.Container { - return dag.Container(dagger.ContainerOpts{Platform: platform}). - From(alpineBaseImage). - WithExec([]string{"apk", "add", "--update", "--no-cache", "ca-certificates", "tzdata", "bash"}). - WithLabel("org.opencontainers.image.title", "openmeter"). - WithLabel("org.opencontainers.image.description", "Cloud Metering for AI, Billing and FinOps. Collect and aggregate millions of usage events in real-time."). - WithLabel("org.opencontainers.image.url", "https://github.com/openmeterio/openmeter"). - WithLabel("org.opencontainers.image.source", "https://github.com/openmeterio/openmeter"). - WithLabel("org.opencontainers.image.licenses", "Apache-2.0"). - With(func(c *dagger.Container) *dagger.Container { - if version != "" { - c = c.WithLabel("org.opencontainers.image.version", version) - } - - return c - }). - WithFile("/usr/local/bin/openmeter", m.Binary().api(platform, version)). - WithFile("/usr/local/bin/openmeter-sink-worker", m.Binary().sinkWorker(platform, version)). - WithFile("/usr/local/bin/openmeter-balance-worker", m.Binary().balanceWorker(platform, version)). - WithFile("/usr/local/bin/openmeter-notification-service", m.Binary().notificationService(platform, version)). - WithLabel("org.opencontainers.image.created", time.Now().String()) // TODO: embed commit timestamp -} - -// Build binaries. -func (m *Build) Binary() *Binary { - return &Binary{ - Source: m.Source, - } -} - -type Binary struct { - // +private - Source *dagger.Directory -} - -// Build all binaries. -func (m *Binary) All( - ctx context.Context, - - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) error { - p := pool.New().WithErrors().WithContext(ctx) - - p.Go(syncFunc(m.Api(platform))) - p.Go(syncFunc(m.SinkWorker(platform))) - p.Go(syncFunc(m.BalanceWorker(platform))) - p.Go(syncFunc(m.NotificationService(platform))) - p.Go(syncFunc(m.BenthosCollector(platform))) - - return p.Wait() -} - -// Build the API server binary. -func (m *Binary) Api( - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) *dagger.File { - return m.api(platform, "") -} - -func (m *Binary) api(platform dagger.Platform, version string) *dagger.File { - return m.buildCross(platform, version, "./cmd/server").WithName("server") -} - -// Build the sink worker binary. -func (m *Binary) SinkWorker( - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) *dagger.File { - return m.sinkWorker(platform, "") -} - -func (m *Binary) sinkWorker(platform dagger.Platform, version string) *dagger.File { - return m.buildCross(platform, version, "./cmd/sink-worker").WithName("sink-worker") -} - -// Build the balance worker binary. -func (m *Binary) BalanceWorker( - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) *dagger.File { - return m.balanceWorker(platform, "") -} - -func (m *Binary) balanceWorker(platform dagger.Platform, version string) *dagger.File { - return m.buildCross(platform, version, "./cmd/balance-worker").WithName("balance-worker") -} - -// Build the notification service binary. -func (m *Binary) NotificationService( - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) *dagger.File { - return m.notificationService(platform, "") -} - -func (m *Binary) notificationService(platform dagger.Platform, version string) *dagger.File { - return m.buildCross(platform, version, "./cmd/notification-service").WithName("notification-service") -} - -func (m *Binary) buildCross(platform dagger.Platform, version string, pkg string) *dagger.File { - if version == "" { - version = "unknown" - } - - goMod := goModuleCross(platform) - - binary := goMod. - WithSource(m.Source). - Build(dagger.GoWithSourceBuildOpts{ - Pkg: pkg, - Trimpath: true, - Tags: []string{"musl"}, - Ldflags: []string{ - "-s", "-w", - "-linkmode", "external", - "-extldflags", `"-static"`, - "-X", "main.version=" + version, - }, - }) - - return goMod. - Container(). - WithFile("/out/binary", binary). - WithExec([]string{"xx-verify", "/out/binary"}). - File("/out/binary") -} - -// Build the sink worker binary. -func (m *Binary) BenthosCollector( - // Target platform in "[os]/[platform]/[version]" format (e.g., "darwin/arm64/v7", "windows/amd64", "linux/arm64"). - // +optional - platform dagger.Platform, -) *dagger.File { - return m.benthosCollector(platform, "") -} - -func (m *Binary) benthosCollector(platform dagger.Platform, version string) *dagger.File { - return m.build(platform, version, "./cmd/benthos-collector").WithName("benthos") -} - -func (m *Binary) build(platform dagger.Platform, version string, pkg string) *dagger.File { - if version == "" { - version = "unknown" - } - - return goModule(). - With(func(m *dagger.Go) *dagger.Go { - if platform != "" { - m = m.WithPlatform(platform) - } - - return m - }). - WithSource(m.Source). - Build(dagger.GoWithSourceBuildOpts{ - Pkg: pkg, - Trimpath: true, - Ldflags: []string{ - "-s", "-w", - "-X", "main.version=" + version, - }, - }) -} - -func goModule() *dagger.Go { - return dag.Go(dagger.GoOpts{Version: goBuildVersion}). - WithModuleCache(cacheVolume("go-mod")). - WithBuildCache(cacheVolume("go-build")) -} - -func (m *Build) HelmChart( - // Name of the chart to build. - name string, - - // Release version. - // +optional - version string, -) *dagger.File { - return m.helmChart(name, version).File() -} - -func (m *Build) helmChart(name string, version string) *dagger.HelmPackage { - opts := dagger.HelmChartPackageOpts{ - DependencyUpdate: true, - } - - if version != "" { - opts.Version = strings.TrimPrefix(version, "v") - opts.AppVersion = version - } - - return helmChart(m.Source, name).Package(opts) -} - -func helmChart(source *dagger.Directory, name string) *dagger.HelmChart { - chart := source.Directory("deploy/charts").Directory(name) - - readme := dag.HelmDocs(dagger.HelmDocsOpts{Version: helmDocsVersion}).Generate(chart, dagger.HelmDocsGenerateOpts{ - Templates: []*dagger.File{ - source.File("deploy/charts/template.md"), - chart.File("README.tmpl.md"), - }, - SortValuesOrder: "file", - }) - - chart = chart.WithFile("README.md", readme) - - return dag.Helm(dagger.HelmOpts{Version: helmVersion}).Chart(chart) -} diff --git a/.dagger/generate.go b/.dagger/generate.go deleted file mode 100644 index 2e2e472c01..0000000000 --- a/.dagger/generate.go +++ /dev/null @@ -1,37 +0,0 @@ -package main - -import ( - "github.com/openmeterio/openmeter/.dagger/internal/dagger" -) - -// Generate various artifacts. -func (m *Openmeter) Generate() *Generate { - return &Generate{ - Source: m.Source, - } -} - -type Generate struct { - // +private - Source *dagger.Directory -} - -// Generate the Python SDK. -func (m *Generate) PythonSdk() *dagger.Directory { - // We build our image as the official autorest Dockerfile is outdated - // and not compatible with the latest autorest. - // More specifically, the latest autorest npm package depends on - // other Azure packages that require a higher node version. - // Official image: https://github.com/Azure/autorest/blob/63ffe68961e24ed8aa59a2ca4c16a8019c271e45/docker/base/ubuntu/Dockerfile - - // Autorest is incompatible with latest node version - return dag.Container(). - From(NODEJS_CONTAINER_IMAGE). - WithExec([]string{"apt", "update"}). - WithExec([]string{"apt", "install", "-y", "python3", "python3-pip", "python3-venv"}). - WithExec([]string{"npm", "install", "-g", "autorest"}). - WithDirectory("/work", m.Source.Directory("api")). - WithWorkdir("/work/client/python"). - WithExec([]string{"autorest", "config.yaml"}). - Directory("/work/client/python") -} diff --git a/.dagger/go.mod b/.dagger/go.mod deleted file mode 100644 index a22302aab2..0000000000 --- a/.dagger/go.mod +++ /dev/null @@ -1,60 +0,0 @@ -module github.com/openmeterio/openmeter/.dagger - -go 1.25.5 - -require ( - github.com/Khan/genqlient v0.8.1 - github.com/sourcegraph/conc v0.3.0 - github.com/vektah/gqlparser/v2 v2.5.30 - go.opentelemetry.io/otel v1.42.0 - go.opentelemetry.io/otel/sdk v1.42.0 - go.opentelemetry.io/otel/trace v1.42.0 -) - -require ( - github.com/99designs/gqlgen v0.17.81 // indirect - github.com/cenkalti/backoff/v5 v5.0.3 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.18.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.18.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 // indirect - go.opentelemetry.io/otel/log v0.18.0 // indirect - go.opentelemetry.io/otel/sdk/log v0.18.0 // indirect - go.opentelemetry.io/proto/otlp v1.10.0 // indirect - golang.org/x/sync v0.19.0 // indirect - google.golang.org/grpc v1.79.3 // indirect -) - -require ( - dagger.io/dagger v0.19.11 - github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/go-logr/logr v1.4.3 // indirect - github.com/go-logr/stdr v1.2.2 // indirect - github.com/google/uuid v1.6.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect - github.com/sosodev/duration v1.3.1 // indirect - go.opentelemetry.io/auto/sdk v1.2.1 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 // indirect - go.opentelemetry.io/otel/metric v1.42.0 // indirect - go.opentelemetry.io/otel/sdk/metric v1.42.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - golang.org/x/net v0.51.0 // indirect - golang.org/x/sys v0.41.0 // indirect - golang.org/x/text v0.34.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 // indirect - google.golang.org/protobuf v1.36.11 // indirect -) - -replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 - -replace go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp => go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 - -replace go.opentelemetry.io/otel/log => go.opentelemetry.io/otel/log v0.14.0 - -replace go.opentelemetry.io/otel/sdk/log => go.opentelemetry.io/otel/sdk/log v0.14.0 diff --git a/.dagger/go.sum b/.dagger/go.sum deleted file mode 100644 index d17995767d..0000000000 --- a/.dagger/go.sum +++ /dev/null @@ -1,109 +0,0 @@ -dagger.io/dagger v0.19.11 h1:Cra3wL1oaZsqXJcnPydocx3bIDD5tM7XCuwcn2Uh+2Q= -dagger.io/dagger v0.19.11/go.mod h1:BjAJWl4Lx7XRW7nooNjBi0ZAC5Ici2pkthkdBIZdbTI= -github.com/99designs/gqlgen v0.17.81 h1:kCkN/xVyRb5rEQpuwOHRTYq83i0IuTQg9vdIiwEerTs= -github.com/99designs/gqlgen v0.17.81/go.mod h1:vgNcZlLwemsUhYim4dC1pvFP5FX0pr2Y+uYUoHFb1ig= -github.com/Khan/genqlient v0.8.1 h1:wtOCc8N9rNynRLXN3k3CnfzheCUNKBcvXmVv5zt6WCs= -github.com/Khan/genqlient v0.8.1/go.mod h1:R2G6DzjBvCbhjsEajfRjbWdVglSH/73kSivC9TLWVjU= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= -github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= -github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= -github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= -github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= -github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= -github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= -github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= -github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= -github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= -github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= -github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= -github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0 h1:HWRh5R2+9EifMyIHV7ZV+MIZqgz+PMpZ14Jynv3O2Zs= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.28.0/go.mod h1:JfhWUomR1baixubs02l85lZYYOm7LV6om4ceouMv45c= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= -github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= -github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= -github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= -github.com/sosodev/duration v1.3.1/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= -github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= -github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= -github.com/vektah/gqlparser/v2 v2.5.30 h1:EqLwGAFLIzt1wpx1IPpY67DwUujF1OfzgEyDsLrN6kE= -github.com/vektah/gqlparser/v2 v2.5.30/go.mod h1:D1/VCZtV3LPnQrcPBeR/q5jkSQIPti0uYCP/RI0gIeo= -go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= -go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= -go.opentelemetry.io/otel v1.42.0 h1:lSQGzTgVR3+sgJDAU/7/ZMjN9Z+vUip7leaqBKy4sho= -go.opentelemetry.io/otel v1.42.0/go.mod h1:lJNsdRMxCUIWuMlVJWzecSMuNjE7dOYyWlqOXWkdqCc= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0 h1:OMqPldHt79PqWKOMYIAQs3CxAi7RLgPxwfFSwr4ZxtM= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploggrpc v0.14.0/go.mod h1:1biG4qiqTxKiUCtoWDPpL3fB3KxVwCiGw81j3nKMuHE= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0 h1:QQqYw3lkrzwVsoEX0w//EhH/TCnpRdEenKBOOEIMjWc= -go.opentelemetry.io/otel/exporters/otlp/otlplog/otlploghttp v0.14.0/go.mod h1:gSVQcr17jk2ig4jqJ2DX30IdWH251JcNAecvrqTxH1s= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0 h1:MdKucPl/HbzckWWEisiNqMPhRrAOQX8r4jTuGr636gk= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.42.0/go.mod h1:RolT8tWtfHcjajEH5wFIZ4Dgh5jpPdFXYV9pTAk/qjc= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0 h1:H7O6RlGOMTizyl3R08Kn5pdM06bnH8oscSj7o11tmLA= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v1.42.0/go.mod h1:mBFWu/WOVDkWWsR7Tx7h6EpQB8wsv7P0Yrh0Pb7othc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0 h1:THuZiwpQZuHPul65w4WcwEnkX2QIuMT+UFoOrygtoJw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.42.0/go.mod h1:J2pvYM5NGHofZ2/Ru6zw/TNWnEQp5crgyDeSrYpXkAw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0 h1:zWWrB1U6nqhS/k6zYB74CjRpuiitRtLLi68VcgmOEto= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.42.0/go.mod h1:2qXPNBX1OVRC0IwOnfo1ljoid+RD0QK3443EaqVlsOU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0 h1:uLXP+3mghfMf7XmV4PkGfFhFKuNWoCvvx5wP/wOXo0o= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.42.0/go.mod h1:v0Tj04armyT59mnURNUJf7RCKcKzq+lgJs6QSjHjaTc= -go.opentelemetry.io/otel/log v0.14.0 h1:2rzJ+pOAZ8qmZ3DDHg73NEKzSZkhkGIua9gXtxNGgrM= -go.opentelemetry.io/otel/log v0.14.0/go.mod h1:5jRG92fEAgx0SU/vFPxmJvhIuDU9E1SUnEQrMlJpOno= -go.opentelemetry.io/otel/metric v1.42.0 h1:2jXG+3oZLNXEPfNmnpxKDeZsFI5o4J+nz6xUlaFdF/4= -go.opentelemetry.io/otel/metric v1.42.0/go.mod h1:RlUN/7vTU7Ao/diDkEpQpnz3/92J9ko05BIwxYa2SSI= -go.opentelemetry.io/otel/sdk v1.42.0 h1:LyC8+jqk6UJwdrI/8VydAq/hvkFKNHZVIWuslJXYsDo= -go.opentelemetry.io/otel/sdk v1.42.0/go.mod h1:rGHCAxd9DAph0joO4W6OPwxjNTYWghRWmkHuGbayMts= -go.opentelemetry.io/otel/sdk/log v0.14.0 h1:JU/U3O7N6fsAXj0+CXz21Czg532dW2V4gG1HE/e8Zrg= -go.opentelemetry.io/otel/sdk/log v0.14.0/go.mod h1:imQvII+0ZylXfKU7/wtOND8Hn4OpT3YUoIgqJVksUkM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0 h1:Ijbtz+JKXl8T2MngiwqBlPaHqc4YCaP/i13Qrow6gAM= -go.opentelemetry.io/otel/sdk/log/logtest v0.14.0/go.mod h1:dCU8aEL6q+L9cYTqcVOk8rM9Tp8WdnHOPLiBgp0SGOA= -go.opentelemetry.io/otel/sdk/metric v1.42.0 h1:D/1QR46Clz6ajyZ3G8SgNlTJKBdGp84q9RKCAZ3YGuA= -go.opentelemetry.io/otel/sdk/metric v1.42.0/go.mod h1:Ua6AAlDKdZ7tdvaQKfSmnFTdHx37+J4ba8MwVCYM5hc= -go.opentelemetry.io/otel/trace v1.42.0 h1:OUCgIPt+mzOnaUTpOQcBiM/PLQ/Op7oq6g4LenLmOYY= -go.opentelemetry.io/otel/trace v1.42.0/go.mod h1:f3K9S+IFqnumBkKhRJMeaZeNk9epyhnCmQh/EysQCdc= -go.opentelemetry.io/proto/otlp v1.10.0 h1:IQRWgT5srOCYfiWnpqUYz9CVmbO8bFmKcwYxpuCSL2g= -go.opentelemetry.io/proto/otlp v1.10.0/go.mod h1:/CV4QoCR/S9yaPj8utp3lvQPoqMtxXdzn7ozvvozVqk= -go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= -go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= -golang.org/x/net v0.51.0 h1:94R/GTO7mt3/4wIKpcR5gkGmRLOuE/2hNGeWq/GBIFo= -golang.org/x/net v0.51.0/go.mod h1:aamm+2QF5ogm02fjy5Bb7CQ0WMt1/WVM7FtyaTLlA9Y= -golang.org/x/sync v0.19.0 h1:vV+1eWNmZ5geRlYjzm2adRgW2/mcpevXNg50YZtPCE4= -golang.org/x/sync v0.19.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k= -golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= -golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk= -golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA= -gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= -gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= -google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57 h1:JLQynH/LBHfCTSbDWl+py8C+Rg/k1OVH3xfcaiANuF0= -google.golang.org/genproto/googleapis/api v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:kSJwQxqmFXeo79zOmbrALdflXQeAYcUbgS7PbpMknCY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57 h1:mWPCjDEyshlQYzBpMNHaEof6UX1PmHcaUODUywQ0uac= -google.golang.org/genproto/googleapis/rpc v0.0.0-20260209200024-4cfbd4190f57/go.mod h1:j9x/tPzZkyxcgEFkiKEEGxfvyumM01BEtsW8xzOahRQ= -google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE= -google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ= -google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= -google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/.dagger/main.go b/.dagger/main.go deleted file mode 100644 index 589eec312c..0000000000 --- a/.dagger/main.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -import ( - "github.com/openmeterio/openmeter/.dagger/internal/dagger" -) - -type Openmeter struct { - // Project source directory - // This will become useful once pulling from remote becomes available - // - // +private - Source *dagger.Directory -} - -func New( - // Project source directory. - // - // +defaultPath="/" - // +ignore=[".devenv", ".direnv", ".github", ".vscode", "api/client/javascript/dist", "api/client/javascript/node_modules", "api/spec/node_modules", "api/spec/output", "tmp", "go.work", "go.work.sum"] - source *dagger.Directory, -) *Openmeter { - return &Openmeter{ - Source: source, - } -} diff --git a/.dagger/migrate.go b/.dagger/migrate.go deleted file mode 100644 index 565736468a..0000000000 --- a/.dagger/migrate.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "context" - "fmt" - "strconv" - - "github.com/sourcegraph/conc/pool" - - "github.com/openmeterio/openmeter/.dagger/internal/dagger" -) - -func (m *Openmeter) Migrate() *Migrate { - return &Migrate{ - Source: m.Source, - } -} - -type Migrate struct { - Source *dagger.Directory -} - -// GenerateSqlcTestdata creates a complete SQLC testdata directory for a given migration version -func (m *Migrate) GenerateSqlcTestdata( - ctx context.Context, - // Migration version (timestamp format like 20240826120919) - version int, -) (*dagger.Directory, error) { - versionStr := strconv.Itoa(version) - - // Create postgres service - postgres := dag.Postgres(dagger.PostgresOpts{ - Version: postgresVersion, - Name: fmt.Sprintf("sqlc-gen-%s", versionStr), - }) - - // Use golang-migrate CLI directly - migrateContainer := dag.Container(). - From("migrate/migrate:latest"). - WithServiceBinding("postgres", postgres.Service()). - WithDirectory("/migrations", m.Source.Directory("tools/migrate/migrations")). - WithExec([]string{ - "migrate", - "-path", "/migrations", - "-database", "postgres://postgres:postgres@postgres:5432/postgres?sslmode=disable&x-migrations-table=schema_om", - "goto", versionStr, - }) - - // Wait for migration to complete - _, err := migrateContainer.Sync(ctx) - if err != nil { - return nil, fmt.Errorf("migration failed: %w", err) - } - - // Create schema dump using pg_dump (without sleep, let it fail fast if not ready) - schemaContainer := dag.Container(). - From("postgres:"+postgresVersion). - WithServiceBinding("postgres", postgres.Service()). - WithEnvVariable("PGPASSWORD", "postgres"). - WithExec([]string{ - "pg_dump", - "-h", "postgres", - "-U", "postgres", - "-d", "postgres", - "-s", - "--no-owner", - "--no-acl", - }) - - schemaDump, err := schemaContainer.Stdout(ctx) - if err != nil { - return nil, fmt.Errorf("schema dump failed: %w", err) - } - - // Create sqlc.yaml config - sqlcConfig := `version: "2" -sql: - - engine: "postgresql" - queries: "sqlc/queries.sql" - schema: "sqlc/db-schema.sql" - gen: - go: - package: "db" - out: "db" -` - - // Create empty queries.sql placeholder for SQLC - emptyQueries := `-- Add your SQL queries here --- Example: --- name: GetExampleByID :one --- SELECT * FROM example_table WHERE id = $1; - --- Placeholder query for SQLC validation --- name: GetSchemaVersion :one -SELECT version FROM schema_om ORDER BY version DESC LIMIT 1; -` - - // Build the result directory with sqlc.yaml, schema, and placeholder queries in sqlc/ subdirectory - result := dag.Directory(). - WithNewFile("sqlc.yaml", sqlcConfig). - WithNewFile("sqlc/db-schema.sql", schemaDump). - WithNewFile("sqlc/queries.sql", emptyQueries) - - // Run sqlc generate to create Go types - sqlcContainer := dag.Container(). - From("sqlc/sqlc:latest"). - WithDirectory("/work", result). - WithWorkdir("/work"). - WithExec([]string{"/workspace/sqlc", "generate"}) - - // Get the generated files from the container - generatedFiles := sqlcContainer.Directory("/work") - - return generatedFiles, nil -} - -func (m *Migrate) Check(ctx context.Context) error { - app := goModuleCross(""). - WithSource(m.Source). - Container(). - WithEnvVariable("GOFLAGS", "-tags=musl") - - bin := dag.Container(dagger.ContainerOpts{ - Platform: "linux/amd64", - }).From(AtlasContainerImage).File("atlas") - - atlas := app. - WithFile("/bin/atlas", bin). - WithDirectory("openmeter/ent", m.Source.Directory("openmeter/ent")). - WithDirectory("tools/migrate/migrations", m.Source.Directory("tools/migrate/migrations")). - WithFile("atlas.hcl", m.Source.File("atlas.hcl")) - - p := pool.New().WithErrors().WithContext(ctx) - - // Always validate schema is generated - p.Go(func(ctx context.Context) error { - result := app. - WithExec([]string{"go", "generate", "-x", "./openmeter/ent/..."}). - Directory("openmeter/ent") - - source := m.Source.Directory("openmeter/ent") - - err := diff(ctx, source, result) - if err != nil { - return fmt.Errorf("schema is not in sync with generated code") - } - return nil - }) - - // Always validate migrations are in sync with schema - p.Go(func(ctx context.Context) error { - postgres := dag.Postgres(dagger.PostgresOpts{ - Version: postgresVersion, - Name: "no-diff", - }) - - result := atlas. - WithServiceBinding("postgres", postgres.Service()). - WithExec([]string{"atlas", "migrate", "--env", "ci", "diff", "test"}). - Directory("tools/migrate/migrations") - - source := m.Source.Directory("tools/migrate/migrations") - err := diff(ctx, source, result) - if err != nil { - return fmt.Errorf("migrations are not in sync with schema") - } - - return nil - }) - - // Always lint last 10 migrations - p.Go(syncFunc( - atlas. - WithServiceBinding("postgres", dag.Postgres(dagger.PostgresOpts{ - Version: postgresVersion, - Name: "last-10", - }).Service()). - WithExec([]string{"atlas", "migrate", "--env", "ci", "lint", "--latest", "10"}), - )) - - // Validate checksum is intact - p.Go(syncFunc( - atlas. - WithServiceBinding("postgres", dag.Postgres(dagger.PostgresOpts{ - Version: postgresVersion, - Name: "validate", - }).Service()). - WithExec([]string{"atlas", "migrate", "--env", "ci", "validate"}), - )) - - return p.Wait() -} diff --git a/.dagger/release.go b/.dagger/release.go deleted file mode 100644 index 6a51d3ecf6..0000000000 --- a/.dagger/release.go +++ /dev/null @@ -1,123 +0,0 @@ -package main - -import ( - "context" - "errors" - "fmt" - "strings" - "time" - - "github.com/openmeterio/openmeter/.dagger/internal/dagger" -) - -func (m *Openmeter) Release(ctx context.Context, version string, githubActor string, githubToken *dagger.Secret, pypiToken *dagger.Secret, npmToken *dagger.Secret) error { - p := newPipeline(ctx) - - p.addJobs( - func(ctx context.Context) error { - return m.pushHelmChart(ctx, "openmeter", version, githubActor, githubToken) - }, - - func(ctx context.Context) error { - return m.pushHelmChart(ctx, "benthos-collector", version, githubActor, githubToken) - }, - - func(ctx context.Context) error { - if githubToken == nil { - return errors.New("GitHub token is required to publish a release") - } - - releaseAssets := m.releaseAssets(version) - - return dag.Gh(dagger.GhOpts{ - Token: githubToken, - Repo: "openmeterio/openmeter", - }).Release().Create(ctx, version, version, dagger.GhReleaseCreateOpts{ - Files: releaseAssets, - GenerateNotes: true, - Latest: dagger.GhLatestLatestTrue, - VerifyTag: true, - }) - }, - - // Disabled for now as we don't have a way to generate the SDK yet - // func(ctx context.Context) error { - // return m.publishPythonSdk(ctx, version, pypiToken) - // }, - func(ctx context.Context) error { - return m.PublishJavascriptSdk(ctx, version, "latest", npmToken) - }, - ) - - return p.wait() -} - -func (m *Openmeter) pushHelmChart(ctx context.Context, name string, version string, githubActor string, githubToken *dagger.Secret) error { - return m.Build(). - helmChart(name, version). - WithRegistryAuth("ghcr.io", githubActor, githubToken). - Publish(ctx, "oci://ghcr.io/openmeterio/helm-charts") -} - -func (m *Openmeter) releaseAssets(version string) []*dagger.File { - binaryArchives := m.binaryArchives(version) - checksums := dag.Checksum().Sha256().Calculate(binaryArchives) - - return append(binaryArchives, checksums) -} - -func (m *Openmeter) binaryArchives(version string) []*dagger.File { - platforms := []dagger.Platform{ - "linux/amd64", - "linux/arm64", - - "darwin/amd64", - "darwin/arm64", - } - - archives := make([]*dagger.File, 0, len(platforms)) - - for _, platform := range platforms { - archives = append(archives, m.binaryArchive(version, platform)) - } - - return archives -} - -func (m *Openmeter) binaryArchive(version string, platform dagger.Platform) *dagger.File { - var archiver interface { - Archive(name string, source *dagger.Directory) *dagger.File - } = dag.Archivist().TarGz() - - if strings.HasPrefix(string(platform), "windows/") { - archiver = dag.Archivist().Zip() - } - - return archiver.Archive( - fmt.Sprintf("benthos-collector_%s", strings.ReplaceAll(string(platform), "/", "_")), - dag.Directory(). - WithFile("", m.Build().Binary().benthosCollector(platform, version)). - WithFile("", m.Source.File("README.md")). - WithFile("", m.Source.File("LICENSE")), - ) -} - -// TODO: keep in sync with api/client/javascript/Makefile for now, if the release process is moved to nix, can be removed -func (m *Openmeter) PublishJavascriptSdk(ctx context.Context, version string, tag string, npmToken *dagger.Secret) error { - // TODO: generate SDK on the fly? - _, err := dag.Container(). - From(NODEJS_CONTAINER_IMAGE). - WithExec([]string{"npm", "install", "-g", fmt.Sprintf("corepack@v%s", COREPACK_VERSION)}). - WithExec([]string{"corepack", "enable"}). - WithDirectory("/work", m.Source.Directory("api")). - WithWorkdir("/work/client/javascript"). - WithExec([]string{"pnpm", "install", "--frozen-lockfile"}). - WithExec([]string{"pnpm", "version", version, "--no-git-tag-version"}). - WithEnvVariable("CACHE_BUSTER", time.Now().Format(time.RFC3339Nano)). - WithExec([]string{"sh", "-c", "echo '//registry.npmjs.org/:_authToken=${NPM_TOKEN}' > /root/.npmrc"}). - WithSecretVariable("NPM_TOKEN", npmToken). - WithExec([]string{"pnpm", "publish", "--no-git-checks", "--tag", tag}). - Sync(ctx) - - return err -} diff --git a/.dagger/utils.go b/.dagger/utils.go deleted file mode 100644 index acadda7185..0000000000 --- a/.dagger/utils.go +++ /dev/null @@ -1,110 +0,0 @@ -package main - -import ( - "context" - "fmt" - - "github.com/sourcegraph/conc/pool" - - "github.com/openmeterio/openmeter/.dagger/internal/dagger" -) - -type syncable[T any] interface { - Sync(ctx context.Context) (T, error) -} - -func sync[T any](ctx context.Context, s syncable[T]) error { - _, err := s.Sync(ctx) - if err != nil { - return err - } - - return nil -} - -func syncFunc[T any](s syncable[T]) func(context.Context) error { - return func(ctx context.Context) error { - return sync(ctx, s) - } -} - -func wrapSyncable[T any](s syncable[T]) func(context.Context) error { - return func(ctx context.Context) error { - _, err := s.Sync(ctx) - return err - } -} - -func wrapSyncables[T syncable[T]](ss []T) func(context.Context) error { - return func(ctx context.Context) error { - for _, s := range ss { - _, err := s.Sync(ctx) - if err != nil { - return err - } - } - return nil - } -} - -type pipeline struct { - pool *pool.ContextPool -} - -func newPipeline(ctx context.Context) *pipeline { - return &pipeline{ - pool: pool.New().WithErrors().WithContext(ctx), - } -} - -func (p *pipeline) addJob(job func(context.Context) error) { - p.pool.Go(job) -} - -func (p *pipeline) addJobs(jobs ...func(context.Context) error) { - for _, job := range jobs { - p.pool.Go(job) - } -} - -func (p *pipeline) wait() error { - return p.pool.Wait() -} - -func addSyncableStep[T any](p *pipeline, s syncable[T]) { - p.pool.Go(syncFunc(s)) -} - -func cacheVolume(name string) *dagger.CacheVolume { - return dag.CacheVolume(fmt.Sprintf("openmeter-%s", name)) -} - -func diff(ctx context.Context, d1, d2 *dagger.Directory) error { - _, err := dag.Container(dagger.ContainerOpts{Platform: ""}). - From(alpineBaseImage). - WithDirectory("src", d1). - WithDirectory("res", d2). - WithExec([]string{"diff", "-u", "-r", "-q", "src", "res"}). - Sync(ctx) - - return err -} - -func goModuleCross(platform dagger.Platform) *dagger.Go { - container := goModule(). - WithCgoEnabled(). // TODO: set env var instead? - Container(). - With(func(c *dagger.Container) *dagger.Container { - if platform != "" { - c = c.WithEnvVariable("TARGETPLATFORM", string(platform)) - } - - return c - }). - WithDirectory("/", dag.Container().From(xxBaseImage).Rootfs()). - WithExec([]string{"apk", "add", "--update", "--no-cache", "ca-certificates", "make", "git", "curl", "clang", "lld"}). - WithExec([]string{"xx-apk", "add", "--update", "--no-cache", "musl-dev", "gcc"}). - WithExec([]string{"xx-go", "--wrap"}) - - return dag.Go(dagger.GoOpts{Container: container}) -} diff --git a/.dagger/versions.go b/.dagger/versions.go deleted file mode 100644 index d52800d588..0000000000 --- a/.dagger/versions.go +++ /dev/null @@ -1,20 +0,0 @@ -package main - -const ( - golangciLintVersion = "v2.2.1" - helmVersion = "3.16.4" - helmDocsVersion = "v1.14.2" - spectralVersion = "6.13.1" -) - -const ( - // COREPACK_VERSION defines the corepack version to be used in CI pipelines - // NOTE: Temporary measure to overcome the following issue: https://github.com/nodejs/corepack/issues/612 - COREPACK_VERSION = "0.31.0" - - // NODEJS_CONTAINER_IMAGE defines the container image to be used for Nodejs. - NODEJS_CONTAINER_IMAGE = "node:22.13-alpine3.21@sha256:e2b39f7b64281324929257d0f8004fb6cb4bf0fdfb9aa8cedb235a766aec31da" - - // AtlasContainerImage defines the container image to be used for testing database migrations. - AtlasContainerImage = "arigaio/atlas:0.36.0@sha256:fc61beb75283c17eb30e0ccacd80423b774f83f20293a644e1e5f5a9e53d1cab" -) diff --git a/.dagger/versions_pinned.go b/.dagger/versions_pinned.go deleted file mode 100644 index c325e7521e..0000000000 --- a/.dagger/versions_pinned.go +++ /dev/null @@ -1,17 +0,0 @@ -package main - -const ( - kafkaVersion = "3.6" - clickhouseVersion = "24.10" - redisVersion = "7.0.12" - postgresVersion = "14.9" - svixVersion = "v1.82" - - // TODO: add update mechanism for versions below - - // Alpine is required for our current build (due to Kafka and CGO), but it doesn't seem to work well with golangci-lint - goBuildVersion = "1.25.5-alpine3.23@sha256:ac09a5f469f307e5da71e766b0bd59c9c49ea460a528cc3e6686513d64a6f1fb" - xxBaseImage = "tonistiigi/xx:1.7.0@sha256:010d4b66aed389848b0694f91c7aaee9df59a6f20be7f5d12e53663a37bd14e2" - - alpineBaseImage = "alpine:3.22@sha256:4b7ce07002c69e8f3d704a9c5d6fd3053be500b7f1c69fc0d80990c2ad8dd412" -) diff --git a/.fossa.yaml b/.fossa.yaml index f026570306..fab870b6bc 100644 --- a/.fossa.yaml +++ b/.fossa.yaml @@ -6,6 +6,5 @@ project: paths: exclude: - - ./.dagger/ - ./.direnv/ - ./.devenv/ diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f6a1f09377..a9af6e7e1c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -8,9 +8,6 @@ on: permissions: contents: read -env: - DAGGER_VERSION: 0.20.0 - concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: ${{ !contains(github.ref, 'refs/heads/main') && !contains(github.ref, 'refs/tags/') }} @@ -388,36 +385,44 @@ jobs: environment: dev steps: - # Required as a workaround for Dagger to properly detect Git metadata - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: fetch-depth: 0 # Needed to compare against base branch persist-credentials: false - - name: Run pipeline - uses: dagger/dagger-for-github@27b130bf0f79a7f6fbbbe0fbca6760dc9bb40a77 # v8.4.1 + - name: Set up Nix + uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 with: - verb: call - args: migrate check - cloud-token: ${{ secrets.DAGGER_CLOUD_NO_CACHE_TOKEN }} - version: ${{ env.DAGGER_VERSION }} + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + keep-env-derivations = true + keep-outputs = true - - name: Export Dagger Engine logs - id: export-dagger-engine-logs - run: docker logs $(docker container list --all --filter 'name=^dagger-engine-*' --format '{{.Names}}') > engine.stdout.log 2> engine.stderr.log - if: always() - continue-on-error: true + - name: Restore Nix store + uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 + with: + primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }} + restore-prefixes-first-match: | + ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- + ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} + ${{ runner.os }}-openmeter-nix-build-main- + ${{ runner.os }}-openmeter-nix-build- - - name: Upload Dagger Engine logs as artifact - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 - if: always() && steps.export-dagger-engine-logs.outcome == 'success' + - name: Restore go.mod cache if exists + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 with: - name: "[${{ github.job }}] Dagger Engine logs" - path: | - engine.stdout.log - engine.stderr.log - retention-days: 14 + path: .devenv/state/go + key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }} + restore-keys: | + ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }} + ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}- + ${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }} + ${{ runner.os }}-openmeter-go-modules-main + + - name: Run migration checks + run: nix develop --impure .#ci -c make migrate-check lint: name: Lint diff --git a/.github/workflows/npm-release.yaml b/.github/workflows/npm-release.yaml new file mode 100644 index 0000000000..caa916d984 --- /dev/null +++ b/.github/workflows/npm-release.yaml @@ -0,0 +1,78 @@ +# Reusable workflow: publish @openmeter/sdk to npm via OIDC Trusted Publishing. +# +# npm validates the top-level (caller) workflow filename for trusted publishing, +# not this reusable file. The trusted publisher entry on npmjs.com is therefore +# configured against the caller workflow (e.g. release.yaml) + environment prod. +# See: https://github.com/npm/documentation/issues/1755 + +name: NPM Release + +on: + workflow_call: + inputs: + version: + description: "Version to publish (without leading v; pnpm strips it anyway)." + required: true + type: string + dist-tag: + description: "npm dist-tag to publish under (e.g. latest, beta)." + required: true + type: string + +# Restrictive default; the publish job opens id-token: write itself. +permissions: + contents: read + +jobs: + publish: + name: Publish @openmeter/sdk + runs-on: depot-ubuntu-latest-8 + environment: prod + + # npm Trusted Publishing: GitHub mints a short-lived OIDC token that npm + # exchanges for a publish token. Requires id-token: write and a matching + # trusted publisher configuration on npmjs.com. + permissions: + contents: read + id-token: write + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Nix + uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + keep-env-derivations = true + keep-outputs = true + + - name: Restore Nix store + uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 + with: + primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }} + restore-prefixes-first-match: | + ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- + ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} + ${{ runner.os }}-openmeter-nix-build-main- + ${{ runner.os }}-openmeter-nix-build- + + # Trusted publishing requires npm CLI >= 11.5.1 on PATH so npm can + # perform the OIDC handshake. setup-node provides a current npm and + # writes a registry-scoped .npmrc with no _authToken. + - name: Set up Node.js + uses: actions/setup-node@53b83947a5a98c8d113130e565377fae1a50d02f # v6.3.0 + with: + node-version: 22.x + registry-url: 'https://registry.npmjs.org' + + - name: Publish NPM package + run: nix develop --impure .#ci -c make -C api/client/javascript publish-javascript-sdk + env: + JS_SDK_RELEASE_VERSION: ${{ inputs.version }} + JS_SDK_RELEASE_TAG: ${{ inputs.dist-tag }} + NPM_CONFIG_PROVENANCE: "true" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 2d64b3eeea..4d360f112c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -6,16 +6,26 @@ on: - "v[0-9]+.[0-9]+.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-dev.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-beta.[0-9]+" + branches: + - main + # Manual trigger lets maintainers publish a JS SDK beta on demand without a commit. + workflow_dispatch: permissions: contents: read -env: - DAGGER_VERSION: 0.19.11 +# Serialize releases by ref so two overlapping tag pushes or main pushes +# can't race on the npm/GHCR registries. Tags never cancel each other; +# main-push betas can safely supersede earlier in-flight runs. +concurrency: + group: release-${{ github.ref }} + cancel-in-progress: ${{ github.ref_type != 'tag' }} jobs: artifacts: name: Artifacts + # Artifact publishing (container images, etc.) is tag-only. + if: github.ref_type == 'tag' uses: ./.github/workflows/artifacts.yaml with: publish: true @@ -25,38 +35,237 @@ jobs: id-token: write security-events: write - dagger: - name: Dagger + helm-release: + name: Helm Charts + # Helm chart releases are tag-only. + if: github.ref_type == 'tag' runs-on: depot-ubuntu-latest-8 environment: prod permissions: - contents: write + contents: read packages: write - id-token: write - security-events: write + + strategy: + fail-fast: false + matrix: + chart: [openmeter, benthos-collector] steps: - # Required as a workaround for Dagger to properly detect Git metadata - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: Run pipeline - uses: dagger/dagger-for-github@27b130bf0f79a7f6fbbbe0fbca6760dc9bb40a77 # v8.4.1 + - name: Set up Nix + uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + keep-env-derivations = true + keep-outputs = true + + - name: Restore Nix store + uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 with: - verb: call - args: release --version ${{ github.ref_name }} --github-actor ${{ github.actor }} --github-token env:GITHUB_TOKEN --pypi-token env:PYPI_TOKEN --npm-token env:NPM_TOKEN - cloud-token: ${{ secrets.DAGGER_CLOUD_NO_CACHE_TOKEN }} - version: ${{ env.DAGGER_VERSION }} + primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }} + restore-prefixes-first-match: | + ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- + ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} + ${{ runner.os }}-openmeter-nix-build-main- + ${{ runner.os }}-openmeter-nix-build- + + - name: Package chart + # Untrusted values (github.ref_name, matrix.chart) passed via env to + # avoid shell injection through ${{ ... }} interpolation in run:. + run: nix develop --impure .#ci -c make package-helm-chart CHART="$CHART" VERSION="$VERSION" env: - GITHUB_TOKEN: ${{ github.token }} - PYPI_TOKEN: ${{ secrets.PYPI_TOKEN }} - NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + CHART: ${{ matrix.chart }} + VERSION: ${{ github.ref_name }} + + - name: Login to GitHub Container Registry + run: echo "$GH_TOKEN" | nix develop --impure .#ci -c helm registry login ghcr.io --username "$GH_ACTOR" --password-stdin + env: + GH_TOKEN: ${{ github.token }} + GH_ACTOR: ${{ github.actor }} + + - name: Push chart to GHCR + run: | + nix develop --impure .#ci -c helm push \ + "build/helm/${CHART}-${GITHUB_REF_NAME#v}.tgz" \ + oci://ghcr.io/openmeterio/helm-charts + env: + CHART: ${{ matrix.chart }} + GITHUB_REF_NAME: ${{ github.ref_name }} + + binary-build: + name: Binary (${{ matrix.goos }}/${{ matrix.goarch }}) + # Release binaries are tag-only. + if: github.ref_type == 'tag' + runs-on: depot-ubuntu-latest-8 + + strategy: + fail-fast: false + matrix: + include: + - goos: linux + goarch: amd64 + - goos: linux + goarch: arm64 + - goos: darwin + goarch: amd64 + - goos: darwin + goarch: arm64 + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Set up Nix + uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 + with: + github_access_token: ${{ secrets.GITHUB_TOKEN }} + nix_conf: | + access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} + keep-env-derivations = true + keep-outputs = true + + - name: Restore Nix store + uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 + with: + primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }} + restore-prefixes-first-match: | + ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- + ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} + ${{ runner.os }}-openmeter-nix-build-main- + ${{ runner.os }}-openmeter-nix-build- + + - name: Restore go.mod cache if exists + uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4 + with: + path: .devenv/state/go + key: ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }} + restore-keys: | + ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }} + ${{ runner.os }}-openmeter-go-modules-${{ github.ref_name }}- + ${{ runner.os }}-openmeter-go-modules-main-${{ hashFiles('flake.*', 'go.*', '.github/workflows/*.yaml') }} + ${{ runner.os }}-openmeter-go-modules-main + + - name: Build benthos-collector binary + # Untrusted github.ref_name passed via env; matrix values are + # workflow-controlled but routed via env for consistency. + run: | + nix develop --impure .#ci -c make build-benthos-collector-release \ + GOOS="$GOOS" GOARCH="$GOARCH" VERSION="$VERSION" + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + VERSION: ${{ github.ref_name }} + + - name: Archive benthos-collector binary + run: | + nix develop --impure .#ci -c make archive-benthos-collector-release \ + GOOS="$GOOS" GOARCH="$GOARCH" + env: + GOOS: ${{ matrix.goos }} + GOARCH: ${{ matrix.goarch }} + + - name: Upload archive + uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0 + with: + name: benthos-collector_${{ matrix.goos }}_${{ matrix.goarch }} + path: build/release/benthos-collector_${{ matrix.goos }}_${{ matrix.goarch }}.tar.gz + if-no-files-found: error + retention-days: 7 + + github-release: + name: GitHub Release + # GitHub release creation is tag-only. + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + needs: [binary-build] + environment: prod + + permissions: + contents: write + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Download binary archives + uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + path: dist + pattern: "benthos-collector_*" + merge-multiple: true + + - name: Compute checksums + working-directory: dist + run: | + sha256sum benthos-collector_*.tar.gz > checksums.txt + cat checksums.txt + + - name: Create GitHub release + env: + GH_TOKEN: ${{ github.token }} + run: | + gh release create "${GITHUB_REF_NAME}" \ + --title "${GITHUB_REF_NAME}" \ + --generate-notes \ + --verify-tag \ + dist/benthos-collector_*.tar.gz \ + dist/checksums.txt + + # Compute the npm version and dist-tag from the trigger context so the + # reusable workflow below receives them as plain inputs. Tag pushes go to + # the `latest` dist-tag; main pushes and manual dispatches publish a + # per-commit beta. + sdk-javascript-meta: + name: JavaScript SDK Release Meta + runs-on: ubuntu-latest + outputs: + version: ${{ steps.meta.outputs.version }} + dist-tag: ${{ steps.meta.outputs.dist-tag }} + steps: + - name: Determine version and npm dist-tag + id: meta + env: + REF_TYPE: ${{ github.ref_type }} + run: | + if [[ "$REF_TYPE" == "tag" ]]; then + echo "version=${GITHUB_REF_NAME}" >> "$GITHUB_OUTPUT" + echo "dist-tag=latest" >> "$GITHUB_OUTPUT" + else + short_sha="${GITHUB_SHA:0:12}" + echo "version=1.0.0-beta-${short_sha}" >> "$GITHUB_OUTPUT" + echo "dist-tag=beta" >> "$GITHUB_OUTPUT" + fi + + sdk-javascript-release: + name: JavaScript SDK Release + # Runs on both tag pushes (stable/pre-release) and main pushes (per-commit beta). + # npm's trusted publisher entry is keyed on caller workflow file + environment, + # so this single caller must serve both channels. + needs: [sdk-javascript-meta] + uses: ./.github/workflows/npm-release.yaml + with: + version: ${{ needs.sdk-javascript-meta.outputs.version }} + dist-tag: ${{ needs.sdk-javascript-meta.outputs.dist-tag }} + permissions: + contents: read + id-token: write + secrets: inherit sdk-python-release: name: Python SDK Release + # Python SDK releases are tag-only (dev Python releases live in sdk-python-dev-release.yaml). + if: github.ref_type == 'tag' runs-on: depot-ubuntu-latest-8 environment: prod @@ -64,16 +273,13 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: - ref: ${{ github.head_ref }} persist-credentials: false - name: Get short SHA id: get-short-sha run: | - id=$(echo ${{ github.sha }} | cut -c 1-12) - echo "id=${id}" >> $GITHUB_OUTPUT + echo "id=${GITHUB_SHA:0:12}" >> "$GITHUB_OUTPUT" - # If you keep using Nix for your dev shell / tooling, retain these steps - name: Set up Nix uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 with: diff --git a/.github/workflows/sdk-javascript-dev-release.yaml b/.github/workflows/sdk-javascript-dev-release.yaml deleted file mode 100644 index 8d0a43c142..0000000000 --- a/.github/workflows/sdk-javascript-dev-release.yaml +++ /dev/null @@ -1,59 +0,0 @@ -name: JavaScript SDK Beta Release - -permissions: - contents: read - -on: - workflow_dispatch: - push: - branches: - - main - -jobs: - release-beta-sdk: - name: JavaScript SDK Beta Release - runs-on: depot-ubuntu-latest-8 - environment: dev - - steps: - - name: Checkout repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - ref: ${{ github.head_ref }} - persist-credentials: false - - - name: Get short SHA - id: get-short-sha - run: | - id=$(echo ${{github.sha}} | cut -c 1-12) - echo "id=${id}" >> $GITHUB_OUTPUT - - - name: Configure registry auth - run: | - echo "//registry.npmjs.org/:_authToken=${{ secrets.NPM_TOKEN }}" >> ~/.npmrc - - - name: Set up Nix - uses: nixbuild/nix-quick-install-action@2c9db80fb984ceb1bcaa77cdda3fdf8cfba92035 # v34 - with: - github_access_token: ${{ secrets.GITHUB_TOKEN }} - nix_conf: | - access-tokens = github.com=${{ secrets.GITHUB_TOKEN }} - keep-env-derivations = true - keep-outputs = true - - - name: Restore Nix store - uses: nix-community/cache-nix-action/restore@7df957e333c1e5da7721f60227dbba6d06080569 # v7.0.2 - with: - primary-key: ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}-${{ hashFiles('flake.*') }} - restore-prefixes-first-match: | - ${{ runner.os }}-openmeter-nix-build-${{ github.ref_name }}- - ${{ runner.os }}-openmeter-nix-build-main-${{ hashFiles('flake.*') }} - ${{ runner.os }}-openmeter-nix-build-main- - ${{ runner.os }}-openmeter-nix-build- - - - name: Publish NPM package - run: | - nix develop --impure .#ci -c make -C api/client/javascript publish-javascript-sdk - env: - JS_SDK_RELEASE_VERSION: 1.0.0-beta-${{steps.get-short-sha.outputs.id}} - JS_SDK_RELEASE_TAG: beta diff --git a/.golangci-fast.yaml b/.golangci-fast.yaml index d867d98fc2..93603d3d11 100644 --- a/.golangci-fast.yaml +++ b/.golangci-fast.yaml @@ -36,8 +36,6 @@ linters: - legacy - std-error-handling paths: - - .dagger/internal - - .dagger/dagger.gen.go - collector/benthos/internal - third_party$ - builtin$ diff --git a/.golangci.yaml b/.golangci.yaml index 114dd24cf1..ec31c8370b 100644 --- a/.golangci.yaml +++ b/.golangci.yaml @@ -40,8 +40,6 @@ linters: - legacy - std-error-handling paths: - - .dagger/internal - - .dagger/dagger.gen.go - collector/benthos/internal - third_party$ - builtin$ @@ -64,8 +62,6 @@ formatters: exclusions: generated: lax paths: - - .dagger/internal - - .dagger/dagger.gen.go - collector/benthos/internal - third_party$ - builtin$ diff --git a/.vscode/default.code-workspace b/.vscode/default.code-workspace index c0a23cc954..bc6ba11fb9 100644 --- a/.vscode/default.code-workspace +++ b/.vscode/default.code-workspace @@ -4,10 +4,6 @@ "name": "root", "path": "../" }, - { - "name": "ci", - "path": "../.dagger" - }, { "name": "typespec", "path": "../api/spec" diff --git a/AGENTS.md b/AGENTS.md index ae1bc49b1b..e7da4def62 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,15 +4,14 @@ OpenMeter is a usage metering and billing platform for AI and DevTool companies, ## Quick Reference -Use the `Makefile` for all common tasks. Dagger and justfile exist but are seldom used. +Use the `Makefile` for all common tasks. A `justfile` also exists but is seldom used. OpenMeter is a metering and billing platform with usage based pricing and access control. ## Tips for working with the codebase If during your work anything confuses you or something isn't trivial for you, please augment AGENTS.md with your findings so next time it will be easier for you. AGENTS.md files are for you to edit and update as you go so you can interact with the codebase the most effectively. -Development commands are run via `Makefile`, it contains all commonly used commands during development. `Dagger` and `justfile` are also present but seldom used. Use the Makefile commands for common tasks like running tests, generating code, linting, etc. -The Dagger Go builder image is pinned in `.dagger/versions_pinned.go` via `goBuildVersion`; update that constant when the repo-wide Go version is bumped so Dagger checks stay aligned with `go.mod` and Dockerfiles. +Development commands are run via `Makefile`, it contains all commonly used commands during development. A `justfile` is also present but seldom used. Use the Makefile commands for common tasks like running tests, generating code, linting, etc. ## AGENTS.md maintenance diff --git a/Makefile b/Makefile index eb14bfc822..c2617edbb0 100644 --- a/Makefile +++ b/Makefile @@ -50,15 +50,45 @@ generate-all: update-openapi generate-javascript-sdk ## Execute all code generat go generate ./... .PHONY: migrate-check -migrate-check: ## Validate migrations +migrate-check: migrate-check-schema migrate-check-diff migrate-check-lint migrate-check-validate ## Validate migrations + +.PHONY: migrate-check-schema +migrate-check-schema: ## Ensure ent schema is in sync with generated code + $(call print-target) + go generate -x ./openmeter/ent/... + @if ! git diff --quiet -- openmeter/ent || [ -n "$$(git ls-files --others --exclude-standard -- openmeter/ent)" ]; then \ + git --no-pager diff -- openmeter/ent; \ + git ls-files --others --exclude-standard -- openmeter/ent; \ + echo "!!! schema is not in sync with generated code — run 'go generate ./openmeter/ent/...' and commit the changes !!!"; \ + exit 1; \ + fi + +.PHONY: migrate-check-diff +migrate-check-diff: ## Ensure migrations are in sync with schema (runs atlas migrate diff against a clean target) + $(call print-target) + atlas migrate --env local diff migrate-check >/dev/null + @if ! git diff --quiet -- tools/migrate/migrations || [ -n "$$(git ls-files --others --exclude-standard -- tools/migrate/migrations)" ]; then \ + git --no-pager diff -- tools/migrate/migrations; \ + git ls-files --others --exclude-standard -- tools/migrate/migrations; \ + echo "!!! migrations are not in sync with schema — run 'atlas migrate --env local diff ' and commit the generated files !!!"; \ + exit 1; \ + fi + +.PHONY: migrate-check-lint +migrate-check-lint: ## Lint the last 10 migrations $(call print-target) - dagger call migrate check + atlas migrate --env local lint --latest 10 + +.PHONY: migrate-check-validate +migrate-check-validate: ## Validate migration checksums + $(call print-target) + atlas migrate --env local validate .PHONY: generate-sqlc-testdata generate-sqlc-testdata: ## Generate SQLC testdata for a specific version (make generate-sqlc-testdata VERSION=20240826120919) $(call print-target) @if [ -z "$(VERSION)" ]; then echo "Usage: make generate-sqlc-testdata VERSION="; exit 1; fi - dagger call migrate generate-sqlc-testdata --version=$(VERSION) export --path=tools/migrate/testdata/sqlcgen/$(VERSION) + VERSION=$(VERSION) ./tools/migrate/generate-sqlc-testdata.sh .PHONY: generate generate: patch-oapi-templates ## Generate code @@ -77,6 +107,30 @@ build-dir: .PHONY: build build: build-server build-sink-worker build-benthos-collector build-balance-worker build-billing-worker build-notification-service build-jobs ## Build all binaries +# Cross-compile the benthos-collector binary for release archives. +# Usage: make build-benthos-collector-release GOOS=linux GOARCH=amd64 VERSION=v1.2.3 +# Produces build/release/benthos-collector__/benthos (+ README.md, LICENSE) +.PHONY: build-benthos-collector-release +build-benthos-collector-release: ## Cross-compile benthos-collector for release (set GOOS/GOARCH/VERSION) + $(call print-target) + @if [ -z "$(GOOS)" ] || [ -z "$(GOARCH)" ]; then echo "ERROR: GOOS and GOARCH are required"; exit 1; fi + @version="$${VERSION:-unknown}" && \ + outdir="build/release/benthos-collector_$(GOOS)_$(GOARCH)" && \ + rm -rf "$$outdir" && mkdir -p "$$outdir" && \ + CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) \ + go build -trimpath \ + -ldflags "-s -w -X main.version=$${version}" \ + -o "$$outdir/benthos" ./cmd/benthos-collector && \ + cp README.md LICENSE "$$outdir/" + +# Produces build/release/benthos-collector__.tar.gz from the directory above. +.PHONY: archive-benthos-collector-release +archive-benthos-collector-release: ## Archive the cross-compiled benthos-collector (set GOOS/GOARCH) + $(call print-target) + @if [ -z "$(GOOS)" ] || [ -z "$(GOARCH)" ]; then echo "ERROR: GOOS and GOARCH are required"; exit 1; fi + @name="benthos-collector_$(GOOS)_$(GOARCH)" && \ + tar -C build/release -czf "build/release/$${name}.tar.gz" "$$name" + .PHONY: build-server build-server: | build-dir ## Build server binary $(call print-target) @@ -201,6 +255,24 @@ lint-helm: ## Lint Helm charts helm lint deploy/charts/openmeter helm lint deploy/charts/benthos-collector +# Package a helm chart for release. +# Usage: make package-helm-chart CHART=openmeter VERSION=v1.2.3 +# Produces build/helm/-.tgz +.PHONY: package-helm-chart +package-helm-chart: ## Package a helm chart for release (set CHART and VERSION) + $(call print-target) + @if [ -z "$(CHART)" ] || [ -z "$(VERSION)" ]; then echo "ERROR: CHART and VERSION are required"; exit 1; fi + @chart_dir="deploy/charts/$(CHART)" && \ + version_no_v="$(VERSION:v%=%)" && \ + mkdir -p build/helm && \ + helm-docs --log-level info -s file -c "$$chart_dir" \ + -t "deploy/charts/template.md" -t "$$chart_dir/README.tmpl.md" && \ + helm dependency update "$$chart_dir" && \ + helm package "$$chart_dir" \ + --version "$$version_no_v" \ + --app-version "$(VERSION)" \ + --destination build/helm + .PHONY: lint-go lint-go: ## Lint Go code $(call print-target) diff --git a/api/client/javascript/Makefile b/api/client/javascript/Makefile index 22f596b776..9df3c86948 100644 --- a/api/client/javascript/Makefile +++ b/api/client/javascript/Makefile @@ -8,7 +8,6 @@ generate: ## Generate JavaScript SDK pnpm build pnpm test -# Warning: keep in sync with .dagger/release.go for now .PHONY: publish-javascript-sdk publish-javascript-sdk: ## Publish JavaScript SDK $(call print-target) diff --git a/cmd/benthos-collector/version.go b/cmd/benthos-collector/version.go new file mode 100644 index 0000000000..bfda8db05d --- /dev/null +++ b/cmd/benthos-collector/version.go @@ -0,0 +1,11 @@ +package main + +// Provisioned by ldflags. +var version string + +//nolint:gochecknoinits +func init() { + if version == "" { + version = "unknown" + } +} diff --git a/dagger.json b/dagger.json deleted file mode 100644 index 95ba40482b..0000000000 --- a/dagger.json +++ /dev/null @@ -1,85 +0,0 @@ -{ - "name": "openmeter", - "engineVersion": "v0.20.0", - "sdk": { - "source": "go" - }, - "include": [ - "!.devenv", - "!.direnv", - "!.github", - "!.vscode", - "!api/client/javascript/node_modules", - "!tmp", - "!go.work", - "!go.work.sum" - ], - "dependencies": [ - { - "name": "archivist", - "source": "github.com/sagikazarmark/daggerverse/archivist@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "checksum", - "source": "github.com/sagikazarmark/daggerverse/checksum@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "gh", - "source": "github.com/sagikazarmark/daggerverse/gh@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "go", - "source": "github.com/sagikazarmark/daggerverse/go@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "golangci-lint", - "source": "github.com/sagikazarmark/daggerverse/golangci-lint@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "helm", - "source": "github.com/sagikazarmark/daggerverse/helm@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "helm-docs", - "source": "github.com/sagikazarmark/daggerverse/helm-docs@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "kafka", - "source": "github.com/sagikazarmark/daggerverse/kafka@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "openapi-changes", - "source": "github.com/sagikazarmark/daggerverse/openapi-changes@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "postgres", - "source": "github.com/sagikazarmark/daggerverse/postgres@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "python", - "source": "github.com/sagikazarmark/daggerverse/python@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "spectral", - "source": "github.com/sagikazarmark/daggerverse/spectral@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - }, - { - "name": "svix", - "source": "github.com/sagikazarmark/daggerverse/svix@ceffda4aebd349a24fc00e591b4ed9b801535b65", - "pin": "ceffda4aebd349a24fc00e591b4ed9b801535b65" - } - ], - "source": ".dagger" -} diff --git a/flake.lock b/flake.lock index 7603314820..7317271f97 100644 --- a/flake.lock +++ b/flake.lock @@ -33,26 +33,6 @@ "type": "github" } }, - "dagger": { - "inputs": { - "nixpkgs": [ - "nixpkgs" - ] - }, - "locked": { - "lastModified": 1772151302, - "narHash": "sha256-Jq7fwgA0xDFXj1NzywpDfHWT1GNFHutARvh7aU4tEjs=", - "owner": "dagger", - "repo": "nix", - "rev": "e11d4abe0d21295852d16ad4eef5f7c04d6cc8ca", - "type": "github" - }, - "original": { - "owner": "dagger", - "repo": "nix", - "type": "github" - } - }, "devenv": { "inputs": { "cachix": "cachix", @@ -311,7 +291,6 @@ }, "root": { "inputs": { - "dagger": "dagger", "devenv": "devenv", "flake-parts": "flake-parts_2", "nixpkgs": "nixpkgs_2" diff --git a/flake.nix b/flake.nix index 3ddf942c31..85e00fd957 100644 --- a/flake.nix +++ b/flake.nix @@ -3,8 +3,6 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable"; flake-parts.url = "github:hercules-ci/flake-parts"; devenv.url = "github:cachix/devenv"; - dagger.url = "github:dagger/nix"; - dagger.inputs.nixpkgs.follows = "nixpkgs"; }; outputs = inputs@{ flake-parts, ... }: @@ -21,7 +19,6 @@ overlays = [ (final: prev: { - dagger = inputs'.dagger.packages.dagger; atlasx = self'.packages.atlasx; }) ]; @@ -128,8 +125,6 @@ just semver-tool - dagger - go-migrate sqlc @@ -159,19 +154,6 @@ }; ci = devenv.shells.default; - - # Lighteweight target to use inside dagger - dagger = { - languages = { - go = devenv.shells.default.languages.go; - }; - packages = with pkgs; [ - gnumake - git - atlasx - ]; - containers = devenv.shells.default.containers; - }; }; packages = { diff --git a/justfile b/justfile index 8b488fae84..46f15e783e 100644 --- a/justfile +++ b/justfile @@ -3,10 +3,8 @@ default: @just --list generate: - dagger call generate server export --path api - dagger call generate javascript-sdk export --path api/client/javascript + make gen-api - # TODO: move this to dagger? GOWORK=off go generate ./cmd/... go generate ./api/client/go @@ -36,7 +34,3 @@ release: git tag -m "Release ${version}" $version git push origin $version - -# update CI dependency versions -update-versions: - dagger call dev update-versions --github-token env:GITHUB_TOKEN export --path .dagger/versions.go diff --git a/openmeter/testutils/pg_driver.go b/openmeter/testutils/pg_driver.go index 1165ad8923..b843600664 100644 --- a/openmeter/testutils/pg_driver.go +++ b/openmeter/testutils/pg_driver.go @@ -60,8 +60,7 @@ type TestDB struct { func InitPostgresDB(t *testing.T) *TestDB { t.Helper() - // Dagger will set the POSTGRES_HOST environment variable for `make test`. - // If you need to run credit tests without Dagger you can set the POSTGRES_HOST environment variable. + // Set POSTGRES_HOST to point tests at your local Postgres. // For example to use the Postgres in docker compose you can run `POSTGRES_HOST=localhost go test ./internal/credit/...` host := os.Getenv("POSTGRES_HOST") if host == "" { diff --git a/tools/migrate/generate-sqlc-testdata.sh b/tools/migrate/generate-sqlc-testdata.sh new file mode 100755 index 0000000000..8a65d1a753 --- /dev/null +++ b/tools/migrate/generate-sqlc-testdata.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Generate SQLC testdata for a specific migration version. +# +# Given a golang-migrate version timestamp (e.g. 20240826120919) this script: +# 1. Spins up a clean postgres via docker compose (reusing the repo service) +# 2. Runs all migrations up to the requested version against a scratch database +# 3. Dumps the resulting schema with pg_dump +# 4. Writes sqlc.yaml + placeholder queries alongside the schema +# 5. Runs `sqlc generate` to produce the Go structs +# +# Requires: docker compose, migrate (golang-migrate), pg_dump, sqlc (all provided +# by the repo's nix dev shell). +# +# Usage: VERSION=20240826120919 ./tools/migrate/generate-sqlc-testdata.sh +# (or: make generate-sqlc-testdata VERSION=20240826120919) + +set -euo pipefail + +if [[ -z "${VERSION:-}" ]]; then + echo "ERROR: VERSION is required (e.g. VERSION=20240826120919)" >&2 + exit 1 +fi + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +OUT_DIR="${OUT_DIR:-${REPO_ROOT}/tools/migrate/testdata/sqlcgen/${VERSION}}" +MIGRATIONS_DIR="${REPO_ROOT}/tools/migrate/migrations" + +PG_HOST="${POSTGRES_HOST:-127.0.0.1}" +PG_PORT="${POSTGRES_PORT:-5432}" +PG_USER="${POSTGRES_USER:-postgres}" +PG_PASSWORD="${POSTGRES_PASSWORD:-postgres}" +SCRATCH_DB="sqlc_gen_${VERSION}" + +export PGPASSWORD="${PG_PASSWORD}" + +for bin in docker migrate pg_dump psql sqlc; do + if ! command -v "${bin}" >/dev/null 2>&1; then + echo "ERROR: required binary '${bin}' not found in PATH" >&2 + echo "Hint: run inside the nix dev shell (e.g. 'nix develop --impure')" >&2 + exit 1 + fi +done + +echo ">>> Ensuring postgres is running (docker compose up -d postgres)" +(cd "${REPO_ROOT}" && docker compose up -d postgres) +"${REPO_ROOT}/tools/wait-for-compose.sh" postgres + +echo ">>> Creating scratch database ${SCRATCH_DB}" +psql -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_USER}" -d postgres \ + -c "DROP DATABASE IF EXISTS ${SCRATCH_DB};" \ + -c "CREATE DATABASE ${SCRATCH_DB};" + +DB_URL="postgres://${PG_USER}:${PG_PASSWORD}@${PG_HOST}:${PG_PORT}/${SCRATCH_DB}?sslmode=disable&x-migrations-table=schema_om" + +echo ">>> Migrating ${SCRATCH_DB} to version ${VERSION}" +migrate -path "${MIGRATIONS_DIR}" -database "${DB_URL}" goto "${VERSION}" + +mkdir -p "${OUT_DIR}/sqlc" + +echo ">>> Dumping schema to ${OUT_DIR}/sqlc/db-schema.sql" +pg_dump -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_USER}" -d "${SCRATCH_DB}" \ + -s --no-owner --no-acl > "${OUT_DIR}/sqlc/db-schema.sql" + +echo ">>> Writing sqlc.yaml and placeholder queries" +cat > "${OUT_DIR}/sqlc.yaml" <<'YAML' +version: "2" +sql: + - engine: "postgresql" + queries: "sqlc/queries.sql" + schema: "sqlc/db-schema.sql" + gen: + go: + package: "db" + out: "db" +YAML + +cat > "${OUT_DIR}/sqlc/queries.sql" <<'SQL' +-- Add your SQL queries here +-- Example: +-- name: GetExampleByID :one +-- SELECT * FROM example_table WHERE id = $1; + +-- Placeholder query for SQLC validation +-- name: GetSchemaVersion :one +SELECT version FROM schema_om ORDER BY version DESC LIMIT 1; +SQL + +echo ">>> Running sqlc generate in ${OUT_DIR}" +(cd "${OUT_DIR}" && sqlc generate) + +echo ">>> Dropping scratch database ${SCRATCH_DB}" +psql -h "${PG_HOST}" -p "${PG_PORT}" -U "${PG_USER}" -d postgres \ + -c "DROP DATABASE IF EXISTS ${SCRATCH_DB};" >/dev/null + +echo ">>> Done — SQLC testdata written to ${OUT_DIR}"