feat(bun): generate native Go array slices for PostgreSQL array columns

Bun's pgdialect scans/appends native slices directly, so array columns
(text[], integer[], uuid[], ...) always generate as plain []string,
[]int32, etc. with an explicit "array" bun tag, regardless of --types
(sqltypes/stdlib/baselib). The SqlXxxArray wrapper types are no longer
used for Bun array columns (gorm is unaffected and keeps using them).

Adds --array-nullable pointer_slice to represent nullable array columns
as *[]T instead of []T, so callers can distinguish SQL NULL (nil) from
'{}' (pointer to an empty slice). Verified end-to-end against a live
PostgreSQL instance for NULL/{}/populated arrays in every --types mode.

Closes #13
This commit is contained in:
Hein
2026-07-21 12:41:38 +02:00
parent 2cecb4c11c
commit 5d9ff5df03
65 changed files with 9584 additions and 167 deletions
+52
View File
@@ -0,0 +1,52 @@
package pgxpool
import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type errBatchResults struct {
err error
}
func (br errBatchResults) Exec() (pgconn.CommandTag, error) {
return pgconn.CommandTag{}, br.err
}
func (br errBatchResults) Query() (pgx.Rows, error) {
return errRows{err: br.err}, br.err
}
func (br errBatchResults) QueryRow() pgx.Row {
return errRow{err: br.err}
}
func (br errBatchResults) Close() error {
return br.err
}
type poolBatchResults struct {
br pgx.BatchResults
c *Conn
}
func (br *poolBatchResults) Exec() (pgconn.CommandTag, error) {
return br.br.Exec()
}
func (br *poolBatchResults) Query() (pgx.Rows, error) {
return br.br.Query()
}
func (br *poolBatchResults) QueryRow() pgx.Row {
return br.br.QueryRow()
}
func (br *poolBatchResults) Close() error {
err := br.br.Close()
if br.c != nil {
br.c.Release()
br.c = nil
}
return err
}
+134
View File
@@ -0,0 +1,134 @@
package pgxpool
import (
"context"
"sync/atomic"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/puddle/v2"
)
// Conn is an acquired *pgx.Conn from a Pool.
type Conn struct {
res *puddle.Resource[*connResource]
p *Pool
}
// Release returns c to the pool it was acquired from. Once Release has been called, other methods must not be called.
// However, it is safe to call Release multiple times. Subsequent calls after the first will be ignored.
func (c *Conn) Release() {
if c.res == nil {
return
}
conn := c.Conn()
res := c.res
c.res = nil
if c.p.releaseTracer != nil {
c.p.releaseTracer.TraceRelease(c.p, TraceReleaseData{Conn: conn})
}
if conn.IsClosed() || conn.PgConn().IsBusy() || conn.PgConn().TxStatus() != 'I' {
res.Destroy()
// Signal to the health check to run since we just destroyed a connections
// and we might be below minConns now
c.p.triggerHealthCheck()
return
}
// If the pool is consistently being used, we might never get to check the
// lifetime of a connection since we only check idle connections in checkConnsHealth
// so we also check the lifetime here and force a health check
if c.p.isExpired(res) {
atomic.AddInt64(&c.p.lifetimeDestroyCount, 1)
res.Destroy()
// Signal to the health check to run since we just destroyed a connections
// and we might be below minConns now
c.p.triggerHealthCheck()
return
}
if c.p.afterRelease == nil {
res.Release()
return
}
go func() {
if c.p.afterRelease(conn) {
res.Release()
} else {
res.Destroy()
// Signal to the health check to run since we just destroyed a connections
// and we might be below minConns now
c.p.triggerHealthCheck()
}
}()
}
// Hijack assumes ownership of the connection from the pool. Caller is responsible for closing the connection. Hijack
// will panic if called on an already released or hijacked connection.
func (c *Conn) Hijack() *pgx.Conn {
if c.res == nil {
panic("cannot hijack already released or hijacked connection")
}
conn := c.Conn()
res := c.res
c.res = nil
res.Hijack()
return conn
}
func (c *Conn) Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) {
return c.Conn().Exec(ctx, sql, arguments...)
}
func (c *Conn) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) {
return c.Conn().Query(ctx, sql, args...)
}
func (c *Conn) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
return c.Conn().QueryRow(ctx, sql, args...)
}
func (c *Conn) SendBatch(ctx context.Context, b *pgx.Batch) pgx.BatchResults {
return c.Conn().SendBatch(ctx, b)
}
func (c *Conn) CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error) {
return c.Conn().CopyFrom(ctx, tableName, columnNames, rowSrc)
}
// Begin starts a transaction block from the *Conn without explicitly setting a transaction mode (see BeginTx with TxOptions if transaction mode is required).
func (c *Conn) Begin(ctx context.Context) (pgx.Tx, error) {
return c.Conn().Begin(ctx)
}
// BeginTx starts a transaction block from the *Conn with txOptions determining the transaction mode.
func (c *Conn) BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error) {
return c.Conn().BeginTx(ctx, txOptions)
}
func (c *Conn) Ping(ctx context.Context) error {
return c.Conn().Ping(ctx)
}
func (c *Conn) Conn() *pgx.Conn {
return c.connResource().conn
}
func (c *Conn) connResource() *connResource {
return c.res.Value()
}
func (c *Conn) getPoolRow(r pgx.Row) *poolRow {
return c.connResource().getPoolRow(c, r)
}
func (c *Conn) getPoolRows(r pgx.Rows) *poolRows {
return c.connResource().getPoolRows(c, r)
}
+27
View File
@@ -0,0 +1,27 @@
// Package pgxpool is a concurrency-safe connection pool for pgx.
/*
pgxpool implements a nearly identical interface to pgx connections.
Creating a Pool
The primary way of creating a pool is with [pgxpool.New]:
pool, err := pgxpool.New(context.Background(), os.Getenv("DATABASE_URL"))
The database connection string can be in URL or keyword/value format. PostgreSQL settings, pgx settings, and pool settings can be
specified here. In addition, a config struct can be created by [ParseConfig].
config, err := pgxpool.ParseConfig(os.Getenv("DATABASE_URL"))
if err != nil {
// ...
}
config.AfterConnect = func(ctx context.Context, conn *pgx.Conn) error {
// do something with every new connection
}
pool, err := pgxpool.NewWithConfig(context.Background(), config)
A pool returns without waiting for any connections to be established. Acquire a connection immediately after creating
the pool to check if a connection can successfully be established.
*/
package pgxpool
+832
View File
@@ -0,0 +1,832 @@
package pgxpool
import (
"context"
"errors"
"math/rand/v2"
"runtime"
"strconv"
"sync"
"sync/atomic"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/puddle/v2"
)
var (
defaultMaxConns = int32(4)
defaultMinConns = int32(0)
defaultMinIdleConns = int32(0)
defaultMaxConnLifetime = time.Hour
defaultMaxConnIdleTime = time.Minute * 30
defaultHealthCheckPeriod = time.Minute
)
type connResource struct {
conn *pgx.Conn
conns []Conn
poolRows []poolRow
poolRowss []poolRows
maxAgeTime time.Time
}
func (cr *connResource) getConn(p *Pool, res *puddle.Resource[*connResource]) *Conn {
if len(cr.conns) == 0 {
cr.conns = make([]Conn, 128)
}
c := &cr.conns[len(cr.conns)-1]
cr.conns = cr.conns[0 : len(cr.conns)-1]
c.res = res
c.p = p
return c
}
func (cr *connResource) getPoolRow(c *Conn, r pgx.Row) *poolRow {
if len(cr.poolRows) == 0 {
cr.poolRows = make([]poolRow, 128)
}
pr := &cr.poolRows[len(cr.poolRows)-1]
cr.poolRows = cr.poolRows[0 : len(cr.poolRows)-1]
pr.c = c
pr.r = r
return pr
}
func (cr *connResource) getPoolRows(c *Conn, r pgx.Rows) *poolRows {
if len(cr.poolRowss) == 0 {
cr.poolRowss = make([]poolRows, 128)
}
pr := &cr.poolRowss[len(cr.poolRowss)-1]
cr.poolRowss = cr.poolRowss[0 : len(cr.poolRowss)-1]
pr.c = c
pr.r = r
return pr
}
// Pool allows for connection reuse.
type Pool struct {
// 64 bit fields accessed with atomics must be at beginning of struct to guarantee alignment for certain 32-bit
// architectures. See BUGS section of https://pkg.go.dev/sync/atomic and https://github.com/jackc/pgx/issues/1288.
newConnsCount int64
lifetimeDestroyCount int64
idleDestroyCount int64
p *puddle.Pool[*connResource]
config *Config
beforeConnect func(context.Context, *pgx.ConnConfig) error
afterConnect func(context.Context, *pgx.Conn) error
prepareConn func(context.Context, *pgx.Conn) (bool, error)
afterRelease func(*pgx.Conn) bool
beforeClose func(*pgx.Conn)
shouldPing func(context.Context, ShouldPingParams) bool
minConns int32
minIdleConns int32
maxConns int32
maxConnLifetime time.Duration
maxConnLifetimeJitter time.Duration
maxConnIdleTime time.Duration
healthCheckPeriod time.Duration
pingTimeout time.Duration
healthCheckMu sync.Mutex
healthCheckTimer *time.Timer
healthCheckChan chan struct{}
acquireTracer AcquireTracer
releaseTracer ReleaseTracer
closeOnce sync.Once
closeChan chan struct{}
}
// ShouldPingParams are the parameters passed to ShouldPing.
type ShouldPingParams struct {
Conn *pgx.Conn
IdleDuration time.Duration
}
// Config is the configuration struct for creating a pool. It must be created by [ParseConfig] and then it can be
// modified.
type Config struct {
ConnConfig *pgx.ConnConfig
// BeforeConnect is called before a new connection is made. It is passed a copy of the underlying [pgx.ConnConfig] and
// will not impact any existing open connections.
BeforeConnect func(context.Context, *pgx.ConnConfig) error
// AfterConnect is called after a connection is established, but before it is added to the pool.
AfterConnect func(context.Context, *pgx.Conn) error
// BeforeAcquire is called before a connection is acquired from the pool. It must return true to allow the
// acquisition or false to indicate that the connection should be destroyed and a different connection should be
// acquired.
//
// Deprecated: Use PrepareConn instead. If both PrepareConn and BeforeAcquire are set, PrepareConn will take
// precedence, ignoring BeforeAcquire.
BeforeAcquire func(context.Context, *pgx.Conn) bool
// PrepareConn is called before a connection is acquired from the pool. If this function returns true, the connection
// is considered valid, otherwise the connection is destroyed. If the function returns a non-nil error, the instigating
// query will fail with the returned error.
//
// Specifically, this means that:
//
// - If it returns true and a nil error, the query proceeds as normal.
// - If it returns true and an error, the connection will be returned to the pool, and the instigating query will fail with the returned error.
// - If it returns false, and an error, the connection will be destroyed, and the query will fail with the returned error.
// - If it returns false and a nil error, the connection will be destroyed, and the instigating query will be retried on a new connection.
PrepareConn func(context.Context, *pgx.Conn) (bool, error)
// AfterRelease is called after a connection is released, but before it is returned to the pool. It must return true to
// return the connection to the pool or false to destroy the connection.
AfterRelease func(*pgx.Conn) bool
// BeforeClose is called right before a connection is closed and removed from the pool.
BeforeClose func(*pgx.Conn)
// ShouldPing is called after a connection is acquired from the pool. If it returns true, the connection is pinged to check for liveness.
// If this func is not set, the default behavior is to ping connections that have been idle for at least 1 second.
ShouldPing func(context.Context, ShouldPingParams) bool
// MaxConnLifetime is the duration since creation after which a connection will be automatically closed.
MaxConnLifetime time.Duration
// MaxConnLifetimeJitter is the duration after MaxConnLifetime to randomly decide to close a connection.
// This helps prevent all connections from being closed at the exact same time, starving the pool.
MaxConnLifetimeJitter time.Duration
// MaxConnIdleTime is the duration after which an idle connection will be automatically closed by the health check.
MaxConnIdleTime time.Duration
// PingTimeout is the maximum amount of time to wait for a connection to pong before considering it as unhealthy and
// destroying it. If zero, the default is no timeout.
PingTimeout time.Duration
// MaxConns is the maximum size of the pool. The default is the greater of 4 or runtime.NumCPU().
MaxConns int32
// MinConns is the minimum size of the pool. After connection closes, the pool might dip below MinConns. A low
// number of MinConns might mean the pool is empty after MaxConnLifetime until the health check has a chance
// to create new connections.
MinConns int32
// MinIdleConns is the minimum number of idle connections in the pool. You can increase this to ensure that
// there are always idle connections available. This can help reduce tail latencies during request processing,
// as you can avoid the latency of establishing a new connection while handling requests. It is superior
// to MinConns for this purpose.
// Similar to MinConns, the pool might temporarily dip below MinIdleConns after connection closes.
MinIdleConns int32
// HealthCheckPeriod is the duration between checks of the health of idle connections.
HealthCheckPeriod time.Duration
createdByParseConfig bool // Used to enforce created by ParseConfig rule.
}
// Copy returns a deep copy of the config that is safe to use and modify.
// The only exception is the tls.Config:
// according to the tls.Config docs it must not be modified after creation.
func (c *Config) Copy() *Config {
newConfig := new(Config)
*newConfig = *c
newConfig.ConnConfig = c.ConnConfig.Copy()
return newConfig
}
// ConnString returns the connection string as parsed by pgxpool.ParseConfig into pgxpool.Config.
func (c *Config) ConnString() string { return c.ConnConfig.ConnString() }
// New creates a new Pool. See [ParseConfig] for information on connString format.
func New(ctx context.Context, connString string) (*Pool, error) {
config, err := ParseConfig(connString)
if err != nil {
return nil, err
}
return NewWithConfig(ctx, config)
}
// NewWithConfig creates a new [Pool]. config must have been created by [ParseConfig].
func NewWithConfig(ctx context.Context, config *Config) (*Pool, error) {
// Default values are set in ParseConfig. Enforce initial creation by ParseConfig rather than setting defaults from
// zero values.
if !config.createdByParseConfig {
panic("config must be created by ParseConfig")
}
prepareConn := config.PrepareConn
if prepareConn == nil && config.BeforeAcquire != nil {
prepareConn = func(ctx context.Context, conn *pgx.Conn) (bool, error) {
return config.BeforeAcquire(ctx, conn), nil
}
}
p := &Pool{
config: config,
beforeConnect: config.BeforeConnect,
afterConnect: config.AfterConnect,
prepareConn: prepareConn,
afterRelease: config.AfterRelease,
beforeClose: config.BeforeClose,
minConns: config.MinConns,
minIdleConns: config.MinIdleConns,
maxConns: config.MaxConns,
maxConnLifetime: config.MaxConnLifetime,
maxConnLifetimeJitter: config.MaxConnLifetimeJitter,
maxConnIdleTime: config.MaxConnIdleTime,
pingTimeout: config.PingTimeout,
healthCheckPeriod: config.HealthCheckPeriod,
healthCheckChan: make(chan struct{}, 1),
closeChan: make(chan struct{}),
}
if t, ok := config.ConnConfig.Tracer.(AcquireTracer); ok {
p.acquireTracer = t
}
if t, ok := config.ConnConfig.Tracer.(ReleaseTracer); ok {
p.releaseTracer = t
}
if config.ShouldPing != nil {
p.shouldPing = config.ShouldPing
} else {
p.shouldPing = func(ctx context.Context, params ShouldPingParams) bool {
return params.IdleDuration > time.Second
}
}
var err error
p.p, err = puddle.NewPool(
&puddle.Config[*connResource]{
Constructor: func(ctx context.Context) (*connResource, error) {
atomic.AddInt64(&p.newConnsCount, 1)
connConfig := p.config.ConnConfig.Copy()
// Connection will continue in background even if Acquire is canceled. Ensure that a connect won't hang forever.
if connConfig.ConnectTimeout <= 0 {
connConfig.ConnectTimeout = 2 * time.Minute
}
if p.beforeConnect != nil {
if err := p.beforeConnect(ctx, connConfig); err != nil {
return nil, err
}
}
conn, err := pgx.ConnectConfig(ctx, connConfig)
if err != nil {
return nil, err
}
if p.afterConnect != nil {
err = p.afterConnect(ctx, conn)
if err != nil {
conn.Close(ctx)
return nil, err
}
}
jitterSecs := rand.Float64() * config.MaxConnLifetimeJitter.Seconds()
maxAgeTime := time.Now().Add(config.MaxConnLifetime).Add(time.Duration(jitterSecs) * time.Second)
cr := &connResource{
conn: conn,
conns: make([]Conn, 64),
poolRows: make([]poolRow, 64),
poolRowss: make([]poolRows, 64),
maxAgeTime: maxAgeTime,
}
return cr, nil
},
Destructor: func(value *connResource) {
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second)
conn := value.conn
if p.beforeClose != nil {
p.beforeClose(conn)
}
conn.Close(ctx)
select {
case <-conn.PgConn().CleanupDone():
case <-ctx.Done():
}
cancel()
},
MaxSize: config.MaxConns,
},
)
if err != nil {
return nil, err
}
go func() {
targetIdleResources := max(int(p.minConns), int(p.minIdleConns))
p.createIdleResources(ctx, targetIdleResources)
p.backgroundHealthCheck()
}()
return p, nil
}
// ParseConfig builds a Config from connString. It parses connString with the same behavior as [pgx.ParseConfig] with the
// addition of the following variables:
//
// - pool_max_conns: integer greater than 0 (default 4)
// - pool_min_conns: integer 0 or greater (default 0)
// - pool_max_conn_lifetime: duration string (default 1 hour)
// - pool_max_conn_idle_time: duration string (default 30 minutes)
// - pool_health_check_period: duration string (default 1 minute)
// - pool_max_conn_lifetime_jitter: duration string (default 0)
//
// See Config for definitions of these arguments.
//
// # Example Keyword/Value
// user=jack password=secret host=pg.example.com port=5432 dbname=mydb sslmode=verify-ca pool_max_conns=10 pool_max_conn_lifetime=1h30m
//
// # Example URL
// postgres://jack:secret@pg.example.com:5432/mydb?sslmode=verify-ca&pool_max_conns=10&pool_max_conn_lifetime=1h30m
func ParseConfig(connString string) (*Config, error) {
connConfig, err := pgx.ParseConfig(connString)
if err != nil {
return nil, err
}
config := &Config{
ConnConfig: connConfig,
createdByParseConfig: true,
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_max_conns"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_max_conns")
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_max_conns", err)
}
if n < 1 {
return nil, pgconn.NewParseConfigError(connString, "pool_max_conns too small", err)
}
config.MaxConns = int32(n)
} else {
config.MaxConns = defaultMaxConns
if numCPU := int32(runtime.NumCPU()); numCPU > config.MaxConns {
config.MaxConns = numCPU
}
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_min_conns"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_min_conns")
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_min_conns", err)
}
config.MinConns = int32(n)
} else {
config.MinConns = defaultMinConns
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_min_idle_conns"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_min_idle_conns")
n, err := strconv.ParseInt(s, 10, 32)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_min_idle_conns", err)
}
config.MinIdleConns = int32(n)
} else {
config.MinIdleConns = defaultMinIdleConns
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_max_conn_lifetime"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_max_conn_lifetime")
d, err := time.ParseDuration(s)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_max_conn_lifetime", err)
}
config.MaxConnLifetime = d
} else {
config.MaxConnLifetime = defaultMaxConnLifetime
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_max_conn_idle_time"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_max_conn_idle_time")
d, err := time.ParseDuration(s)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_max_conn_idle_time", err)
}
config.MaxConnIdleTime = d
} else {
config.MaxConnIdleTime = defaultMaxConnIdleTime
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_health_check_period"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_health_check_period")
d, err := time.ParseDuration(s)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_health_check_period", err)
}
config.HealthCheckPeriod = d
} else {
config.HealthCheckPeriod = defaultHealthCheckPeriod
}
if s, ok := config.ConnConfig.Config.RuntimeParams["pool_max_conn_lifetime_jitter"]; ok {
delete(connConfig.Config.RuntimeParams, "pool_max_conn_lifetime_jitter")
d, err := time.ParseDuration(s)
if err != nil {
return nil, pgconn.NewParseConfigError(connString, "cannot parse pool_max_conn_lifetime_jitter", err)
}
config.MaxConnLifetimeJitter = d
}
return config, nil
}
// Close closes all connections in the pool and rejects future [Pool.Acquire] calls. Blocks until all connections are returned
// to pool and closed.
func (p *Pool) Close() {
p.closeOnce.Do(func() {
close(p.closeChan)
p.p.Close()
})
}
func (p *Pool) isExpired(res *puddle.Resource[*connResource]) bool {
return time.Now().After(res.Value().maxAgeTime)
}
func (p *Pool) triggerHealthCheck() {
const healthCheckDelay = 500 * time.Millisecond
p.healthCheckMu.Lock()
defer p.healthCheckMu.Unlock()
if p.healthCheckTimer == nil {
// Destroy is asynchronous so we give it time to actually remove itself from
// the pool otherwise we might try to check the pool size too soon
p.healthCheckTimer = time.AfterFunc(healthCheckDelay, func() {
select {
case <-p.closeChan:
case p.healthCheckChan <- struct{}{}:
default:
}
})
return
}
p.healthCheckTimer.Reset(healthCheckDelay)
}
func (p *Pool) backgroundHealthCheck() {
ticker := time.NewTicker(p.healthCheckPeriod)
defer ticker.Stop()
for {
select {
case <-p.closeChan:
return
case <-p.healthCheckChan:
p.checkHealth()
case <-ticker.C:
p.checkHealth()
}
}
}
func (p *Pool) checkHealth() {
for {
// If checkMinConns failed we don't destroy any connections since we couldn't
// even get to minConns
if err := p.checkMinConns(); err != nil {
// Should we log this error somewhere?
break
}
if !p.checkConnsHealth() {
// Since we didn't destroy any connections we can stop looping
break
}
// Technically Destroy is asynchronous but 500ms should be enough for it to
// remove it from the underlying pool
select {
case <-p.closeChan:
return
case <-time.After(500 * time.Millisecond):
}
}
}
// checkConnsHealth will check all idle connections, destroy a connection if
// it's idle or too old, and returns true if any were destroyed
func (p *Pool) checkConnsHealth() bool {
var destroyed bool
totalConns := p.Stat().TotalConns()
resources := p.p.AcquireAllIdle()
for _, res := range resources {
// We're okay going under minConns if the lifetime is up
if p.isExpired(res) && totalConns >= p.minConns {
atomic.AddInt64(&p.lifetimeDestroyCount, 1)
res.Destroy()
destroyed = true
// Since Destroy is async we manually decrement totalConns.
totalConns--
} else if res.IdleDuration() > p.maxConnIdleTime && totalConns > p.minConns {
atomic.AddInt64(&p.idleDestroyCount, 1)
res.Destroy()
destroyed = true
// Since Destroy is async we manually decrement totalConns.
totalConns--
} else {
res.ReleaseUnused()
}
}
return destroyed
}
func (p *Pool) checkMinConns() error {
// TotalConns can include ones that are being destroyed but we should have
// sleep(500ms) around all of the destroys to help prevent that from throwing
// off this check
// Create the number of connections needed to get to both minConns and minIdleConns
stat := p.Stat()
toCreate := max(p.minConns-stat.TotalConns(), p.minIdleConns-stat.IdleConns())
if toCreate > 0 {
return p.createIdleResources(context.Background(), int(toCreate))
}
return nil
}
func (p *Pool) createIdleResources(parentCtx context.Context, targetResources int) error {
ctx, cancel := context.WithCancel(parentCtx)
defer cancel()
errs := make(chan error, targetResources)
for range targetResources {
go func() {
err := p.p.CreateResource(ctx)
// Ignore ErrNotAvailable since it means that the pool has become full since we started creating resource.
if err == puddle.ErrNotAvailable {
err = nil
}
errs <- err
}()
}
var firstError error
for range targetResources {
err := <-errs
if err != nil && firstError == nil {
cancel()
firstError = err
}
}
return firstError
}
// Acquire returns a connection ([Conn]) from the [Pool].
func (p *Pool) Acquire(ctx context.Context) (c *Conn, err error) {
if p.acquireTracer != nil {
ctx = p.acquireTracer.TraceAcquireStart(ctx, p, TraceAcquireStartData{})
defer func() {
var conn *pgx.Conn
if c != nil {
conn = c.Conn()
}
p.acquireTracer.TraceAcquireEnd(ctx, p, TraceAcquireEndData{Conn: conn, Err: err})
}()
}
// Try to acquire from the connection pool up to maxConns + 1 times, so that
// any that fatal errors would empty the pool and still at least try 1 fresh
// connection.
for range int(p.maxConns) + 1 {
res, err := p.p.Acquire(ctx)
if err != nil {
return nil, err
}
cr := res.Value()
shouldPingParams := ShouldPingParams{Conn: cr.conn, IdleDuration: res.IdleDuration()}
if p.shouldPing(ctx, shouldPingParams) {
err := func() error {
pingCtx := ctx
if p.pingTimeout > 0 {
var cancel context.CancelFunc
pingCtx, cancel = context.WithTimeout(ctx, p.pingTimeout)
defer cancel()
}
return cr.conn.Ping(pingCtx)
}()
if err != nil {
res.Destroy()
continue
}
}
if p.prepareConn != nil {
ok, err := p.prepareConn(ctx, cr.conn)
if !ok {
res.Destroy()
}
if err != nil {
if ok {
res.Release()
}
return nil, err
}
if !ok {
continue
}
}
return cr.getConn(p, res), nil
}
return nil, errors.New("pgxpool: too many failed attempts acquiring connection; likely bug in PrepareConn, BeforeAcquire, or ShouldPing hook")
}
// AcquireFunc acquires a [Conn] and calls f with that [Conn]. ctx will only affect the [Pool.Acquire]. It has no effect on the
// call of f. The return value is either an error acquiring the [Conn] or the return value of f. The [Conn] is
// automatically released after the call of f.
func (p *Pool) AcquireFunc(ctx context.Context, f func(*Conn) error) error {
conn, err := p.Acquire(ctx)
if err != nil {
return err
}
defer conn.Release()
return f(conn)
}
// AcquireAllIdle atomically acquires all currently idle connections. Its intended use is for health check and
// keep-alive functionality. It does not update pool statistics.
func (p *Pool) AcquireAllIdle(ctx context.Context) []*Conn {
resources := p.p.AcquireAllIdle()
conns := make([]*Conn, 0, len(resources))
for _, res := range resources {
cr := res.Value()
if p.prepareConn != nil {
ok, err := p.prepareConn(ctx, cr.conn)
if !ok || err != nil {
res.Destroy()
continue
}
}
conns = append(conns, cr.getConn(p, res))
}
return conns
}
// Reset closes all connections, but leaves the pool open. It is intended for use when an error is detected that would
// disrupt all connections (such as a network interruption or a server state change).
//
// It is safe to reset a pool while connections are checked out. Those connections will be closed when they are returned
// to the pool.
func (p *Pool) Reset() {
p.p.Reset()
}
// Config returns a copy of config that was used to initialize this [Pool].
func (p *Pool) Config() *Config { return p.config.Copy() }
// Stat returns a pgxpool.Stat struct with a snapshot of Pool statistics.
func (p *Pool) Stat() *Stat {
return &Stat{
s: p.p.Stat(),
newConnsCount: atomic.LoadInt64(&p.newConnsCount),
lifetimeDestroyCount: atomic.LoadInt64(&p.lifetimeDestroyCount),
idleDestroyCount: atomic.LoadInt64(&p.idleDestroyCount),
}
}
// Exec acquires a connection from the [Pool] and executes the given SQL.
// SQL can be either a prepared statement name or an SQL string.
// Arguments should be referenced positionally from the SQL string as $1, $2, etc.
// The acquired connection is returned to the pool when the [Pool.Exec] function returns.
func (p *Pool) Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) {
c, err := p.Acquire(ctx)
if err != nil {
return pgconn.CommandTag{}, err
}
defer c.Release()
return c.Exec(ctx, sql, arguments...)
}
// Query acquires a connection and executes a query that returns [pgx.Rows].
// Arguments should be referenced positionally from the SQL string as $1, $2, etc.
// See [pgx.Rows] documentation to close the returned [pgx.Rows] and return the acquired connection to the [Pool].
//
// If there is an error, the returned [pgx.Rows] will be returned in an error state.
// If preferred, ignore the error returned from [Pool.Query] and handle errors using the returned [pgx.Rows].
//
// For extra control over how the query is executed, the types [pgx.QueryExecMode], [pgx.QueryResultFormats], and
// [pgx.QueryResultFormatsByOID] may be used as the first args to control exactly how the query is executed. This is rarely
// needed. See the documentation for those types for details.
func (p *Pool) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) {
c, err := p.Acquire(ctx)
if err != nil {
return errRows{err: err}, err
}
rows, err := c.Query(ctx, sql, args...)
if err != nil {
c.Release()
return errRows{err: err}, err
}
return c.getPoolRows(rows), nil
}
// QueryRow acquires a connection and executes a query that is expected
// to return at most one row ([pgx.Row]). Errors are deferred until [pgx.Row]'s
// Scan method is called. If the query selects no rows, [pgx.Row]'s Scan will
// return [pgx.ErrNoRows]. Otherwise, [pgx.Row]'s Scan scans the first selected row
// and discards the rest. The acquired connection is returned to the [Pool] when
// [pgx.Row]'s Scan method is called.
//
// Arguments should be referenced positionally from the SQL string as $1, $2, etc.
//
// For extra control over how the query is executed, the types [pgx.QueryExecMode], [pgx.QueryResultFormats], and
// [pgx.QueryResultFormatsByOID] may be used as the first args to control exactly how the query is executed. This is rarely
// needed. See the documentation for those types for details.
func (p *Pool) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
c, err := p.Acquire(ctx)
if err != nil {
return errRow{err: err}
}
row := c.QueryRow(ctx, sql, args...)
return c.getPoolRow(row)
}
func (p *Pool) SendBatch(ctx context.Context, b *pgx.Batch) pgx.BatchResults {
c, err := p.Acquire(ctx)
if err != nil {
return errBatchResults{err: err}
}
br := c.SendBatch(ctx, b)
return &poolBatchResults{br: br, c: c}
}
// Begin acquires a connection from the [Pool] and starts a transaction. Unlike [database/sql], the context only affects the begin command. i.e. there is no
// auto-rollback on context cancellation. Begin initiates a transaction block without explicitly setting a transaction mode for the block (see [Pool.BeginTx] with [pgx.TxOptions] if transaction mode is required).
// [*Tx] is returned, which implements the [pgx.Tx] interface.
// [Tx.Commit] or [Tx.Rollback] must be called on the returned transaction to finalize the transaction block.
func (p *Pool) Begin(ctx context.Context) (pgx.Tx, error) {
return p.BeginTx(ctx, pgx.TxOptions{})
}
// BeginTx acquires a connection from the [Pool] and starts a transaction with [pgx.TxOptions] determining the transaction mode.
// Unlike [database/sql], the context only affects the begin command. i.e. there is no auto-rollback on context cancellation.
// [*Tx] is returned, which implements the [pgx.Tx] interface.
// [Tx.Commit] or [Tx.Rollback] must be called on the returned transaction to finalize the transaction block.
func (p *Pool) BeginTx(ctx context.Context, txOptions pgx.TxOptions) (pgx.Tx, error) {
c, err := p.Acquire(ctx)
if err != nil {
return nil, err
}
t, err := c.BeginTx(ctx, txOptions)
if err != nil {
c.Release()
return nil, err
}
return &Tx{t: t, c: c}, nil
}
func (p *Pool) CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error) {
c, err := p.Acquire(ctx)
if err != nil {
return 0, err
}
defer c.Release()
return c.Conn().CopyFrom(ctx, tableName, columnNames, rowSrc)
}
// Ping acquires a connection from the [Pool] and executes an empty sql statement against it.
// If the sql returns without error, the database [Pool.Ping] is considered successful, otherwise, the error is returned.
func (p *Pool) Ping(ctx context.Context) error {
c, err := p.Acquire(ctx)
if err != nil {
return err
}
defer c.Release()
return c.Ping(ctx)
}
+116
View File
@@ -0,0 +1,116 @@
package pgxpool
import (
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
type errRows struct {
err error
}
func (errRows) Close() {}
func (e errRows) Err() error { return e.err }
func (errRows) CommandTag() pgconn.CommandTag { return pgconn.CommandTag{} }
func (errRows) FieldDescriptions() []pgconn.FieldDescription { return nil }
func (errRows) Next() bool { return false }
func (e errRows) Scan(dest ...any) error { return e.err }
func (e errRows) Values() ([]any, error) { return nil, e.err }
func (e errRows) RawValues() [][]byte { return nil }
func (e errRows) Conn() *pgx.Conn { return nil }
type errRow struct {
err error
}
func (e errRow) Scan(dest ...any) error { return e.err }
type poolRows struct {
r pgx.Rows
c *Conn
err error
}
func (rows *poolRows) Close() {
rows.r.Close()
if rows.c != nil {
rows.c.Release()
rows.c = nil
}
}
func (rows *poolRows) Err() error {
if rows.err != nil {
return rows.err
}
return rows.r.Err()
}
func (rows *poolRows) CommandTag() pgconn.CommandTag {
return rows.r.CommandTag()
}
func (rows *poolRows) FieldDescriptions() []pgconn.FieldDescription {
return rows.r.FieldDescriptions()
}
func (rows *poolRows) Next() bool {
if rows.err != nil {
return false
}
n := rows.r.Next()
if !n {
rows.Close()
}
return n
}
func (rows *poolRows) Scan(dest ...any) error {
err := rows.r.Scan(dest...)
if err != nil {
rows.Close()
}
return err
}
func (rows *poolRows) Values() ([]any, error) {
values, err := rows.r.Values()
if err != nil {
rows.Close()
}
return values, err
}
func (rows *poolRows) RawValues() [][]byte {
return rows.r.RawValues()
}
func (rows *poolRows) Conn() *pgx.Conn {
return rows.r.Conn()
}
type poolRow struct {
r pgx.Row
c *Conn
err error
}
func (row *poolRow) Scan(dest ...any) error {
if row.err != nil {
return row.err
}
panicked := true
defer func() {
if panicked && row.c != nil {
row.c.Release()
}
}()
err := row.r.Scan(dest...)
panicked = false
if row.c != nil {
row.c.Release()
}
return err
}
+91
View File
@@ -0,0 +1,91 @@
package pgxpool
import (
"time"
"github.com/jackc/puddle/v2"
)
// Stat is a snapshot of Pool statistics.
type Stat struct {
s *puddle.Stat
newConnsCount int64
lifetimeDestroyCount int64
idleDestroyCount int64
}
// AcquireCount returns the cumulative count of successful acquires from the pool.
func (s *Stat) AcquireCount() int64 {
return s.s.AcquireCount()
}
// AcquireDuration returns the total duration of all successful acquires from
// the pool.
func (s *Stat) AcquireDuration() time.Duration {
return s.s.AcquireDuration()
}
// AcquiredConns returns the number of currently acquired connections in the pool.
func (s *Stat) AcquiredConns() int32 {
return s.s.AcquiredResources()
}
// CanceledAcquireCount returns the cumulative count of acquires from the pool
// that were canceled by a context.
func (s *Stat) CanceledAcquireCount() int64 {
return s.s.CanceledAcquireCount()
}
// ConstructingConns returns the number of conns with construction in progress in
// the pool.
func (s *Stat) ConstructingConns() int32 {
return s.s.ConstructingResources()
}
// EmptyAcquireCount returns the cumulative count of successful acquires from the pool
// that waited for a resource to be released or constructed because the pool was
// empty.
func (s *Stat) EmptyAcquireCount() int64 {
return s.s.EmptyAcquireCount()
}
// IdleConns returns the number of currently idle conns in the pool.
func (s *Stat) IdleConns() int32 {
return s.s.IdleResources()
}
// MaxConns returns the maximum size of the pool.
func (s *Stat) MaxConns() int32 {
return s.s.MaxResources()
}
// TotalConns returns the total number of resources currently in the pool.
// The value is the sum of ConstructingConns, AcquiredConns, and
// IdleConns.
func (s *Stat) TotalConns() int32 {
return s.s.TotalResources()
}
// NewConnsCount returns the cumulative count of new connections opened.
func (s *Stat) NewConnsCount() int64 {
return s.newConnsCount
}
// MaxLifetimeDestroyCount returns the cumulative count of connections destroyed
// because they exceeded MaxConnLifetime.
func (s *Stat) MaxLifetimeDestroyCount() int64 {
return s.lifetimeDestroyCount
}
// MaxIdleDestroyCount returns the cumulative count of connections destroyed because
// they exceeded MaxConnIdleTime.
func (s *Stat) MaxIdleDestroyCount() int64 {
return s.idleDestroyCount
}
// EmptyAcquireWaitTime returns the cumulative time waited for successful acquires
// from the pool for a resource to be released or constructed because the pool was
// empty.
func (s *Stat) EmptyAcquireWaitTime() time.Duration {
return s.s.EmptyAcquireWaitTime()
}
+33
View File
@@ -0,0 +1,33 @@
package pgxpool
import (
"context"
"github.com/jackc/pgx/v5"
)
// AcquireTracer traces Acquire.
type AcquireTracer interface {
// TraceAcquireStart is called at the beginning of Acquire.
// The returned context is used for the rest of the call and will be passed to the TraceAcquireEnd.
TraceAcquireStart(ctx context.Context, pool *Pool, data TraceAcquireStartData) context.Context
// TraceAcquireEnd is called when a connection has been acquired.
TraceAcquireEnd(ctx context.Context, pool *Pool, data TraceAcquireEndData)
}
type TraceAcquireStartData struct{}
type TraceAcquireEndData struct {
Conn *pgx.Conn
Err error
}
// ReleaseTracer traces Release.
type ReleaseTracer interface {
// TraceRelease is called at the beginning of Release.
TraceRelease(pool *Pool, data TraceReleaseData)
}
type TraceReleaseData struct {
Conn *pgx.Conn
}
+83
View File
@@ -0,0 +1,83 @@
package pgxpool
import (
"context"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
)
// Tx represents a database transaction acquired from a Pool.
type Tx struct {
t pgx.Tx
c *Conn
}
// Begin starts a pseudo nested transaction implemented with a savepoint.
func (tx *Tx) Begin(ctx context.Context) (pgx.Tx, error) {
return tx.t.Begin(ctx)
}
// Commit commits the transaction and returns the associated connection back to the Pool. Commit will return an error
// where errors.Is(ErrTxClosed) is true if the Tx is already closed, but is otherwise safe to call multiple times. If
// the commit fails with a rollback status (e.g. the transaction was already in a broken state) then ErrTxCommitRollback
// will be returned.
func (tx *Tx) Commit(ctx context.Context) error {
err := tx.t.Commit(ctx)
if tx.c != nil {
tx.c.Release()
tx.c = nil
}
return err
}
// Rollback rolls back the transaction and returns the associated connection back to the Pool. Rollback will return
// where an error where errors.Is(ErrTxClosed) is true if the Tx is already closed, but is otherwise safe to call
// multiple times. Hence, defer tx.Rollback() is safe even if tx.Commit() will be called first in a non-error condition.
func (tx *Tx) Rollback(ctx context.Context) error {
err := tx.t.Rollback(ctx)
if tx.c != nil {
tx.c.Release()
tx.c = nil
}
return err
}
func (tx *Tx) CopyFrom(ctx context.Context, tableName pgx.Identifier, columnNames []string, rowSrc pgx.CopyFromSource) (int64, error) {
return tx.t.CopyFrom(ctx, tableName, columnNames, rowSrc)
}
func (tx *Tx) SendBatch(ctx context.Context, b *pgx.Batch) pgx.BatchResults {
return tx.t.SendBatch(ctx, b)
}
func (tx *Tx) LargeObjects() pgx.LargeObjects {
return tx.t.LargeObjects()
}
// Prepare creates a prepared statement with name and sql. If the name is empty,
// an anonymous prepared statement will be used. sql can contain placeholders
// for bound parameters. These placeholders are referenced positionally as $1, $2, etc.
//
// Prepare is idempotent; i.e. it is safe to call Prepare multiple times with the same
// name and sql arguments. This allows a code path to Prepare and Query/Exec without
// needing to first check whether the statement has already been prepared.
func (tx *Tx) Prepare(ctx context.Context, name, sql string) (*pgconn.StatementDescription, error) {
return tx.t.Prepare(ctx, name, sql)
}
func (tx *Tx) Exec(ctx context.Context, sql string, arguments ...any) (pgconn.CommandTag, error) {
return tx.t.Exec(ctx, sql, arguments...)
}
func (tx *Tx) Query(ctx context.Context, sql string, args ...any) (pgx.Rows, error) {
return tx.t.Query(ctx, sql, args...)
}
func (tx *Tx) QueryRow(ctx context.Context, sql string, args ...any) pgx.Row {
return tx.t.QueryRow(ctx, sql, args...)
}
func (tx *Tx) Conn() *pgx.Conn {
return tx.t.Conn()
}
+909
View File
@@ -0,0 +1,909 @@
// Package stdlib is the compatibility layer from pgx to database/sql.
//
// A database/sql connection can be established through sql.Open.
//
// db, err := sql.Open("pgx", "postgres://pgx_md5:secret@localhost:5432/pgx_test?sslmode=disable")
// if err != nil {
// return err
// }
//
// Or from a keyword/value string.
//
// db, err := sql.Open("pgx", "user=postgres password=secret host=localhost port=5432 database=pgx_test sslmode=disable")
// if err != nil {
// return err
// }
//
// Or from a *pgxpool.Pool.
//
// pool, err := pgxpool.New(context.Background(), os.Getenv("DATABASE_URL"))
// if err != nil {
// return err
// }
//
// db := stdlib.OpenDBFromPool(pool)
//
// Or a pgx.ConnConfig can be used to set configuration not accessible via connection string. In this case the
// pgx.ConnConfig must first be registered with the driver. This registration returns a connection string which is used
// with sql.Open.
//
// connConfig, _ := pgx.ParseConfig(os.Getenv("DATABASE_URL"))
// connConfig.Tracer = &tracelog.TraceLog{Logger: myLogger, LogLevel: tracelog.LogLevelInfo}
// connStr := stdlib.RegisterConnConfig(connConfig)
// db, _ := sql.Open("pgx", connStr)
//
// pgx uses standard PostgreSQL positional parameters in queries. e.g. $1, $2. It does not support named parameters.
//
// db.QueryRow("select * from users where id=$1", userID)
//
// (*sql.Conn) Raw() can be used to get a *pgx.Conn from the standard database/sql.DB connection pool. This allows
// operations that use pgx specific functionality.
//
// // Given db is a *sql.DB
// conn, err := db.Conn(context.Background())
// if err != nil {
// // handle error from acquiring connection from DB pool
// }
//
// err = conn.Raw(func(driverConn any) error {
// conn := driverConn.(*stdlib.Conn).Conn() // conn is a *pgx.Conn
// // Do pgx specific stuff with conn
// conn.CopyFrom(...)
// return nil
// })
// if err != nil {
// // handle error that occurred while using *pgx.Conn
// }
//
// # PostgreSQL Specific Data Types
//
// The pgtype package provides support for PostgreSQL specific types. *pgtype.Map.SQLScanner is an adapter that makes
// these types usable as a sql.Scanner.
//
// m := pgtype.NewMap()
// var a []int64
// err := db.QueryRow("select '{1,2,3}'::bigint[]").Scan(m.SQLScanner(&a))
package stdlib
import (
"context"
"database/sql"
"database/sql/driver"
"errors"
"fmt"
"io"
"math"
"math/rand/v2"
"reflect"
"slices"
"strconv"
"strings"
"sync"
"time"
"github.com/jackc/pgx/v5"
"github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgtype"
"github.com/jackc/pgx/v5/pgxpool"
)
// Only intrinsic types should be binary format with database/sql.
var databaseSQLResultFormats pgx.QueryResultFormatsByOID
var pgxDriver *Driver
func init() {
pgxDriver = &Driver{
configs: make(map[string]*pgx.ConnConfig),
}
// if pgx driver was already registered by different pgx major version then we
// skip registration under the default name.
if !slices.Contains(sql.Drivers(), "pgx") {
sql.Register("pgx", pgxDriver)
}
sql.Register("pgx/v5", pgxDriver)
databaseSQLResultFormats = pgx.QueryResultFormatsByOID{
pgtype.BoolOID: 1,
pgtype.ByteaOID: 1,
pgtype.CIDOID: 1,
pgtype.DateOID: 1,
pgtype.Float4OID: 1,
pgtype.Float8OID: 1,
pgtype.Int2OID: 1,
pgtype.Int4OID: 1,
pgtype.Int8OID: 1,
pgtype.OIDOID: 1,
pgtype.TimestampOID: 1,
pgtype.TimestamptzOID: 1,
pgtype.XIDOID: 1,
}
}
// OptionOpenDB options for configuring the driver when opening a new db pool.
type OptionOpenDB func(*connector)
// ShouldPingParams are passed to OptionShouldPing to decide whether to ping before reusing a connection.
type ShouldPingParams struct {
// Conn is the underlying pgx connection.
Conn *pgx.Conn
// IdleDuration is how long it has been since ResetSession last ran.
IdleDuration time.Duration
}
// OptionShouldPing controls whether stdlib should issue a liveness ping before reusing a connection.
// If the function returns true, stdlib will ping.
// If it returns false, stdlib will skip the ping.
// If not provided, default is ping only when IdleDuration > 1s.
func OptionShouldPing(f func(context.Context, ShouldPingParams) bool) OptionOpenDB {
return func(dc *connector) { dc.ShouldPing = f }
}
// OptionBeforeConnect provides a callback for before connect. It is passed a shallow copy of the ConnConfig that will
// be used to connect, so only its immediate members should be modified. Used only if db is opened with *pgx.ConnConfig.
func OptionBeforeConnect(bc func(context.Context, *pgx.ConnConfig) error) OptionOpenDB {
return func(dc *connector) {
dc.BeforeConnect = bc
}
}
// OptionAfterConnect provides a callback for after connect. Used only if db is opened with *pgx.ConnConfig.
func OptionAfterConnect(ac func(context.Context, *pgx.Conn) error) OptionOpenDB {
return func(dc *connector) {
dc.AfterConnect = ac
}
}
// OptionResetSession provides a callback that can be used to add custom logic prior to executing a query on the
// connection if the connection has been used before.
// If ResetSessionFunc returns ErrBadConn error the connection will be discarded.
func OptionResetSession(rs func(context.Context, *pgx.Conn) error) OptionOpenDB {
return func(dc *connector) {
dc.ResetSession = rs
}
}
// RandomizeHostOrderFunc is a BeforeConnect hook that randomizes the host order in the provided connConfig, so that a
// new host becomes primary each time. This is useful to distribute connections for multi-master databases like
// CockroachDB. If you use this you likely should set https://golang.org/pkg/database/sql/#DB.SetConnMaxLifetime as well
// to ensure that connections are periodically rebalanced across your nodes.
func RandomizeHostOrderFunc(ctx context.Context, connConfig *pgx.ConnConfig) error {
if len(connConfig.Fallbacks) == 0 {
return nil
}
newFallbacks := append([]*pgconn.FallbackConfig{{
Host: connConfig.Host,
Port: connConfig.Port,
TLSConfig: connConfig.TLSConfig,
}}, connConfig.Fallbacks...)
rand.Shuffle(len(newFallbacks), func(i, j int) {
newFallbacks[i], newFallbacks[j] = newFallbacks[j], newFallbacks[i]
})
// Use the one that sorted last as the primary and keep the rest as the fallbacks
newPrimary := newFallbacks[len(newFallbacks)-1]
connConfig.Host = newPrimary.Host
connConfig.Port = newPrimary.Port
connConfig.TLSConfig = newPrimary.TLSConfig
connConfig.Fallbacks = newFallbacks[:len(newFallbacks)-1]
return nil
}
func GetConnector(config pgx.ConnConfig, opts ...OptionOpenDB) driver.Connector {
c := connector{
ConnConfig: config,
BeforeConnect: func(context.Context, *pgx.ConnConfig) error { return nil }, // noop before connect by default
AfterConnect: func(context.Context, *pgx.Conn) error { return nil }, // noop after connect by default
ResetSession: func(context.Context, *pgx.Conn) error { return nil }, // noop reset session by default
driver: pgxDriver,
}
for _, opt := range opts {
opt(&c)
}
return c
}
// GetPoolConnector creates a new driver.Connector from the given *pgxpool.Pool. By using this be sure to set the
// maximum idle connections of the *sql.DB created with this connector to zero since they must be managed from the
// *pgxpool.Pool. This is required to avoid acquiring all the connections from the pgxpool and starving any direct
// users of the pgxpool.
func GetPoolConnector(pool *pgxpool.Pool, opts ...OptionOpenDB) driver.Connector {
c := connector{
pool: pool,
ResetSession: func(context.Context, *pgx.Conn) error { return nil }, // noop reset session by default
driver: pgxDriver,
}
for _, opt := range opts {
opt(&c)
}
return c
}
func OpenDB(config pgx.ConnConfig, opts ...OptionOpenDB) *sql.DB {
c := GetConnector(config, opts...)
return sql.OpenDB(c)
}
// OpenDBFromPool creates a new *sql.DB from the given *pgxpool.Pool. Note that this method automatically sets the
// maximum number of idle connections in *sql.DB to zero, since they must be managed from the *pgxpool.Pool. This is
// required to avoid acquiring all the connections from the pgxpool and starving any direct users of the pgxpool. Note
// that closing the returned *sql.DB will not close the *pgxpool.Pool.
func OpenDBFromPool(pool *pgxpool.Pool, opts ...OptionOpenDB) *sql.DB {
c := GetPoolConnector(pool, opts...)
db := sql.OpenDB(c)
db.SetMaxIdleConns(0)
return db
}
type connector struct {
pgx.ConnConfig
pool *pgxpool.Pool
BeforeConnect func(context.Context, *pgx.ConnConfig) error // function to call before creation of every new connection
AfterConnect func(context.Context, *pgx.Conn) error // function to call after creation of every new connection
ResetSession func(context.Context, *pgx.Conn) error // function is called before a connection is reused
ShouldPing func(context.Context, ShouldPingParams) bool // function to decide if stdlib should ping before reusing a connection
driver *Driver
}
// Connect implement driver.Connector interface
func (c connector) Connect(ctx context.Context) (driver.Conn, error) {
var (
connConfig pgx.ConnConfig
conn *pgx.Conn
close func(context.Context) error
err error
)
if c.pool == nil {
// Create a shallow copy of the config, so that BeforeConnect can safely modify it
connConfig = c.ConnConfig
if err = c.BeforeConnect(ctx, &connConfig); err != nil {
return nil, err
}
if conn, err = pgx.ConnectConfig(ctx, &connConfig); err != nil {
return nil, err
}
if err = c.AfterConnect(ctx, conn); err != nil {
return nil, err
}
close = conn.Close
} else {
var pconn *pgxpool.Conn
pconn, err = c.pool.Acquire(ctx)
if err != nil {
return nil, err
}
conn = pconn.Conn()
close = func(_ context.Context) error {
pconn.Release()
return nil
}
}
return &Conn{
conn: conn,
close: close,
driver: c.driver,
connConfig: connConfig,
resetSessionFunc: c.ResetSession,
shouldPing: c.ShouldPing,
psRefCounts: make(map[*pgconn.StatementDescription]int),
}, nil
}
// Driver implement driver.Connector interface
func (c connector) Driver() driver.Driver {
return c.driver
}
// GetDefaultDriver returns the driver initialized in the init function
// and used when the pgx driver is registered.
func GetDefaultDriver() driver.Driver {
return pgxDriver
}
type Driver struct {
configMutex sync.Mutex
configs map[string]*pgx.ConnConfig
sequence int
}
func (d *Driver) Open(name string) (driver.Conn, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) // Ensure eventual timeout
defer cancel()
connector, err := d.OpenConnector(name)
if err != nil {
return nil, err
}
return connector.Connect(ctx)
}
func (d *Driver) OpenConnector(name string) (driver.Connector, error) {
return &driverConnector{driver: d, name: name}, nil
}
func (d *Driver) registerConnConfig(c *pgx.ConnConfig) string {
d.configMutex.Lock()
connStr := fmt.Sprintf("registeredConnConfig%d", d.sequence)
d.sequence++
d.configs[connStr] = c
d.configMutex.Unlock()
return connStr
}
func (d *Driver) unregisterConnConfig(connStr string) {
d.configMutex.Lock()
delete(d.configs, connStr)
d.configMutex.Unlock()
}
type driverConnector struct {
driver *Driver
name string
}
func (dc *driverConnector) Connect(ctx context.Context) (driver.Conn, error) {
var connConfig *pgx.ConnConfig
dc.driver.configMutex.Lock()
connConfig = dc.driver.configs[dc.name]
dc.driver.configMutex.Unlock()
if connConfig == nil {
var err error
connConfig, err = pgx.ParseConfig(dc.name)
if err != nil {
return nil, err
}
}
conn, err := pgx.ConnectConfig(ctx, connConfig)
if err != nil {
return nil, err
}
c := &Conn{
conn: conn,
close: conn.Close,
driver: dc.driver,
connConfig: *connConfig,
resetSessionFunc: func(context.Context, *pgx.Conn) error { return nil },
psRefCounts: make(map[*pgconn.StatementDescription]int),
}
return c, nil
}
func (dc *driverConnector) Driver() driver.Driver {
return dc.driver
}
// RegisterConnConfig registers a ConnConfig and returns the connection string to use with Open.
func RegisterConnConfig(c *pgx.ConnConfig) string {
return pgxDriver.registerConnConfig(c)
}
// UnregisterConnConfig removes the ConnConfig registration for connStr.
func UnregisterConnConfig(connStr string) {
pgxDriver.unregisterConnConfig(connStr)
}
type Conn struct {
conn *pgx.Conn
close func(context.Context) error
driver *Driver
connConfig pgx.ConnConfig
resetSessionFunc func(context.Context, *pgx.Conn) error // Function is called before a connection is reused
shouldPing func(context.Context, ShouldPingParams) bool // Function to decide if stdlib should ping before reusing a connection
lastResetSessionTime time.Time
// psRefCounts contains reference counts for prepared statements. Prepare uses the underlying pgx logic to generate
// deterministic statement names from the statement text. If this query has already been prepared then the existing
// *pgconn.StatementDescription will be returned. However, this means that if Close is called on the returned Stmt
// then the underlying prepared statement will be closed even when the underlying prepared statement is still in use
// by another database/sql Stmt. To prevent this psRefCounts keeps track of how many database/sql statements are using
// the same underlying statement and only closes the underlying statement when the reference count reaches 0.
psRefCounts map[*pgconn.StatementDescription]int
}
// Conn returns the underlying *pgx.Conn
func (c *Conn) Conn() *pgx.Conn {
return c.conn
}
func (c *Conn) Prepare(query string) (driver.Stmt, error) {
return c.PrepareContext(context.Background(), query)
}
func (c *Conn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) {
if c.conn.IsClosed() {
return nil, driver.ErrBadConn
}
sd, err := c.conn.Prepare(ctx, query, query)
if err != nil {
return nil, err
}
c.psRefCounts[sd]++
return &Stmt{sd: sd, conn: c}, nil
}
func (c *Conn) Close() error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
return c.close(ctx)
}
func (c *Conn) Begin() (driver.Tx, error) {
return c.BeginTx(context.Background(), driver.TxOptions{})
}
func (c *Conn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) {
if c.conn.IsClosed() {
return nil, driver.ErrBadConn
}
var pgxOpts pgx.TxOptions
switch sql.IsolationLevel(opts.Isolation) {
case sql.LevelDefault:
case sql.LevelReadUncommitted:
pgxOpts.IsoLevel = pgx.ReadUncommitted
case sql.LevelReadCommitted:
pgxOpts.IsoLevel = pgx.ReadCommitted
case sql.LevelRepeatableRead, sql.LevelSnapshot:
pgxOpts.IsoLevel = pgx.RepeatableRead
case sql.LevelSerializable:
pgxOpts.IsoLevel = pgx.Serializable
default:
return nil, fmt.Errorf("unsupported isolation: %v", opts.Isolation)
}
if opts.ReadOnly {
pgxOpts.AccessMode = pgx.ReadOnly
}
tx, err := c.conn.BeginTx(ctx, pgxOpts)
if err != nil {
return nil, err
}
return wrapTx{ctx: ctx, tx: tx}, nil
}
func (c *Conn) ExecContext(ctx context.Context, query string, argsV []driver.NamedValue) (driver.Result, error) {
if c.conn.IsClosed() {
return nil, driver.ErrBadConn
}
args := make([]any, len(argsV))
convertNamedArguments(args, argsV)
commandTag, err := c.conn.Exec(ctx, query, args...)
// if we got a network error before we had a chance to send the query, retry
if err != nil {
if pgconn.SafeToRetry(err) {
return nil, driver.ErrBadConn
}
}
return driver.RowsAffected(commandTag.RowsAffected()), err
}
func (c *Conn) QueryContext(ctx context.Context, query string, argsV []driver.NamedValue) (driver.Rows, error) {
if c.conn.IsClosed() {
return nil, driver.ErrBadConn
}
args := make([]any, 1+len(argsV))
args[0] = databaseSQLResultFormats
convertNamedArguments(args[1:], argsV)
rows, err := c.conn.Query(ctx, query, args...)
if err != nil {
if pgconn.SafeToRetry(err) {
return nil, driver.ErrBadConn
}
return nil, err
}
// Preload first row because otherwise we won't know what columns are available when database/sql asks.
more := rows.Next()
if err = rows.Err(); err != nil {
rows.Close()
return nil, err
}
return &Rows{conn: c, rows: rows, skipNext: true, skipNextMore: more}, nil
}
func (c *Conn) Ping(ctx context.Context) error {
if c.conn.IsClosed() {
return driver.ErrBadConn
}
err := c.conn.Ping(ctx)
if err != nil {
// A Ping failure implies some sort of fatal state. The connection is almost certainly already closed by the
// failure, but manually close it just to be sure.
c.Close()
return driver.ErrBadConn
}
return nil
}
func (c *Conn) CheckNamedValue(*driver.NamedValue) error {
// Underlying pgx supports sql.Scanner and driver.Valuer interfaces natively. So everything can be passed through directly.
return nil
}
func (c *Conn) ResetSession(ctx context.Context) error {
if c.conn.IsClosed() {
return driver.ErrBadConn
}
// Discard connection if it has an open transaction. This can happen if the
// application did not properly commit or rollback a transaction.
if c.conn.PgConn().TxStatus() != 'I' {
return driver.ErrBadConn
}
now := time.Now()
idle := now.Sub(c.lastResetSessionTime)
doPing := idle > time.Second // default behavior: ping only if idle > 1s
if c.shouldPing != nil {
doPing = c.shouldPing(ctx, ShouldPingParams{
Conn: c.conn,
IdleDuration: idle,
})
}
if doPing {
if err := c.conn.PgConn().Ping(ctx); err != nil {
return driver.ErrBadConn
}
}
c.lastResetSessionTime = now
return c.resetSessionFunc(ctx, c.conn)
}
type Stmt struct {
sd *pgconn.StatementDescription
conn *Conn
}
func (s *Stmt) Close() error {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
refCount := s.conn.psRefCounts[s.sd]
if refCount == 1 {
delete(s.conn.psRefCounts, s.sd)
} else {
s.conn.psRefCounts[s.sd]--
return nil
}
return s.conn.conn.Deallocate(ctx, s.sd.SQL)
}
func (s *Stmt) NumInput() int {
return len(s.sd.ParamOIDs)
}
func (s *Stmt) Exec(argsV []driver.Value) (driver.Result, error) {
return nil, errors.New("Stmt.Exec deprecated and not implemented")
}
func (s *Stmt) ExecContext(ctx context.Context, argsV []driver.NamedValue) (driver.Result, error) {
return s.conn.ExecContext(ctx, s.sd.SQL, argsV)
}
func (s *Stmt) Query(argsV []driver.Value) (driver.Rows, error) {
return nil, errors.New("Stmt.Query deprecated and not implemented")
}
func (s *Stmt) QueryContext(ctx context.Context, argsV []driver.NamedValue) (driver.Rows, error) {
return s.conn.QueryContext(ctx, s.sd.SQL, argsV)
}
type rowValueFunc func(src []byte) (driver.Value, error)
type Rows struct {
conn *Conn
rows pgx.Rows
valueFuncs []rowValueFunc
skipNext bool
skipNextMore bool
columnNames []string
}
func (r *Rows) Columns() []string {
if r.columnNames == nil {
fields := r.rows.FieldDescriptions()
r.columnNames = make([]string, len(fields))
for i, fd := range fields {
r.columnNames[i] = string(fd.Name)
}
}
return r.columnNames
}
// ColumnTypeDatabaseTypeName returns the database system type name. If the name is unknown the OID is returned.
func (r *Rows) ColumnTypeDatabaseTypeName(index int) string {
if dt, ok := r.conn.conn.TypeMap().TypeForOID(r.rows.FieldDescriptions()[index].DataTypeOID); ok {
return strings.ToUpper(dt.Name)
}
return strconv.FormatInt(int64(r.rows.FieldDescriptions()[index].DataTypeOID), 10)
}
const varHeaderSize = 4
// ColumnTypeLength returns the length of the column type if the column is a
// variable length type. If the column is not a variable length type ok
// should return false.
func (r *Rows) ColumnTypeLength(index int) (int64, bool) {
fd := r.rows.FieldDescriptions()[index]
switch fd.DataTypeOID {
case pgtype.TextOID, pgtype.ByteaOID:
return math.MaxInt64, true
case pgtype.VarcharOID, pgtype.BPCharOID:
return int64(fd.TypeModifier - varHeaderSize), true
case pgtype.VarbitOID:
return int64(fd.TypeModifier), true
default:
return 0, false
}
}
// ColumnTypePrecisionScale should return the precision and scale for decimal
// types. If not applicable, ok should be false.
func (r *Rows) ColumnTypePrecisionScale(index int) (precision, scale int64, ok bool) {
fd := r.rows.FieldDescriptions()[index]
switch fd.DataTypeOID {
case pgtype.NumericOID:
mod := fd.TypeModifier - varHeaderSize
precision = int64((mod >> 16) & 0xffff)
scale = int64(mod & 0xffff)
return precision, scale, true
default:
return 0, 0, false
}
}
// ColumnTypeScanType returns the value type that can be used to scan types into.
func (r *Rows) ColumnTypeScanType(index int) reflect.Type {
fd := r.rows.FieldDescriptions()[index]
switch fd.DataTypeOID {
case pgtype.Float8OID:
return reflect.TypeFor[float64]()
case pgtype.Float4OID:
return reflect.TypeFor[float32]()
case pgtype.Int8OID:
return reflect.TypeFor[int64]()
case pgtype.Int4OID:
return reflect.TypeFor[int32]()
case pgtype.Int2OID:
return reflect.TypeFor[int16]()
case pgtype.BoolOID:
return reflect.TypeFor[bool]()
case pgtype.NumericOID:
return reflect.TypeFor[float64]()
case pgtype.DateOID, pgtype.TimestampOID, pgtype.TimestamptzOID:
return reflect.TypeFor[time.Time]()
case pgtype.ByteaOID:
return reflect.TypeFor[[]byte]()
default:
return reflect.TypeFor[string]()
}
}
func (r *Rows) Close() error {
r.rows.Close()
return r.rows.Err()
}
func (r *Rows) Next(dest []driver.Value) error {
m := r.conn.conn.TypeMap()
fieldDescriptions := r.rows.FieldDescriptions()
if r.valueFuncs == nil {
r.valueFuncs = make([]rowValueFunc, len(fieldDescriptions))
for i, fd := range fieldDescriptions {
dataTypeOID := fd.DataTypeOID
format := fd.Format
switch fd.DataTypeOID {
case pgtype.BoolOID:
var d bool
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return d, err
}
case pgtype.ByteaOID:
var d []byte
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return d, err
}
case pgtype.CIDOID, pgtype.OIDOID, pgtype.XIDOID:
var d pgtype.Uint32
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.DateOID:
var d pgtype.Date
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.Float4OID:
var d float32
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return float64(d), err
}
case pgtype.Float8OID:
var d float64
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return d, err
}
case pgtype.Int2OID:
var d int16
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return int64(d), err
}
case pgtype.Int4OID:
var d int32
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return int64(d), err
}
case pgtype.Int8OID:
var d int64
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return d, err
}
case pgtype.JSONOID, pgtype.JSONBOID:
var d []byte
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
if err != nil {
return nil, err
}
return d, nil
}
case pgtype.TimestampOID:
var d pgtype.Timestamp
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.TimestamptzOID:
var d pgtype.Timestamptz
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
if err != nil {
return nil, err
}
return d.Value()
}
case pgtype.XMLOID:
var d []byte
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
if err != nil {
return nil, err
}
return d, nil
}
default:
var d string
scanPlan := m.PlanScan(dataTypeOID, format, &d)
r.valueFuncs[i] = func(src []byte) (driver.Value, error) {
err := scanPlan.Scan(src, &d)
return d, err
}
}
}
}
var more bool
if r.skipNext {
more = r.skipNextMore
r.skipNext = false
} else {
more = r.rows.Next()
}
if !more {
if r.rows.Err() == nil {
return io.EOF
} else {
return r.rows.Err()
}
}
for i, rv := range r.rows.RawValues() {
if rv != nil {
var err error
dest[i], err = r.valueFuncs[i](rv)
if err != nil {
return fmt.Errorf("convert field %d failed: %w", i, err)
}
} else {
dest[i] = nil
}
}
return nil
}
func convertNamedArguments(args []any, argsV []driver.NamedValue) {
for i, v := range argsV {
if v.Value != nil {
args[i] = v.Value.(any)
} else {
args[i] = nil
}
}
}
type wrapTx struct {
ctx context.Context
tx pgx.Tx
}
func (wtx wrapTx) Commit() error { return wtx.tx.Commit(wtx.ctx) }
func (wtx wrapTx) Rollback() error { return wtx.tx.Rollback(wtx.ctx) }
+79
View File
@@ -0,0 +1,79 @@
# 2.2.2 (September 10, 2024)
* Add empty acquire time to stats (Maxim Ivanov)
* Stop importing nanotime from runtime via linkname (maypok86)
# 2.2.1 (July 15, 2023)
* Fix: CreateResource cannot overflow pool. This changes documented behavior of CreateResource. Previously,
CreateResource could create a resource even if the pool was full. This could cause the pool to overflow. While this
was documented, it was documenting incorrect behavior. CreateResource now returns an error if the pool is full.
# 2.2.0 (February 11, 2023)
* Use Go 1.19 atomics and drop go.uber.org/atomic dependency
# 2.1.2 (November 12, 2022)
* Restore support to Go 1.18 via go.uber.org/atomic
# 2.1.1 (November 11, 2022)
* Fix create resource concurrently with Stat call race
# 2.1.0 (October 28, 2022)
* Concurrency control is now implemented with a semaphore. This simplifies some internal logic, resolves a few error conditions (including a deadlock), and improves performance. (Jan Dubsky)
* Go 1.19 is now required for the improved atomic support.
# 2.0.1 (October 28, 2022)
* Fix race condition when Close is called concurrently with multiple constructors
# 2.0.0 (September 17, 2022)
* Use generics instead of interface{} (Столяров Владимир Алексеевич)
* Add Reset
* Do not cancel resource construction when Acquire is canceled
* NewPool takes Config
# 1.3.0 (August 27, 2022)
* Acquire creates resources in background to allow creation to continue after Acquire is canceled (James Hartig)
# 1.2.1 (December 2, 2021)
* TryAcquire now does not block when background constructing resource
# 1.2.0 (November 20, 2021)
* Add TryAcquire (A. Jensen)
* Fix: remove memory leak / unintentionally pinned memory when shrinking slices (Alexander Staubo)
* Fix: Do not leave pool locked after panic from nil context
# 1.1.4 (September 11, 2021)
* Fix: Deadlock in CreateResource if pool was closed during resource acquisition (Dmitriy Matrenichev)
# 1.1.3 (December 3, 2020)
* Fix: Failed resource creation could cause concurrent Acquire to hang. (Evgeny Vanslov)
# 1.1.2 (September 26, 2020)
* Fix: Resource.Destroy no longer removes itself from the pool before its destructor has completed.
* Fix: Prevent crash when pool is closed while resource is being created.
# 1.1.1 (April 2, 2020)
* Pool.Close can be safely called multiple times
* AcquireAllIDle immediately returns nil if pool is closed
* CreateResource checks if pool is closed before taking any action
* Fix potential race condition when CreateResource and Close are called concurrently. CreateResource now checks if pool is closed before adding newly created resource to pool.
# 1.1.0 (February 5, 2020)
* Use runtime.nanotime for faster tracking of acquire time and last usage time.
* Track resource idle time to enable client health check logic. (Patrick Ellul)
* Add CreateResource to construct a new resource without acquiring it. (Patrick Ellul)
* Fix deadlock race when acquire is cancelled. (Michael Tharp)
+22
View File
@@ -0,0 +1,22 @@
Copyright (c) 2018 Jack Christensen
MIT License
Permission is hereby granted, free of charge, to any person obtaining
a copy of this software and associated documentation files (the
"Software"), to deal in the Software without restriction, including
without limitation the rights to use, copy, modify, merge, publish,
distribute, sublicense, and/or sell copies of the Software, and to
permit persons to whom the Software is furnished to do so, subject to
the following conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+80
View File
@@ -0,0 +1,80 @@
[![Go Reference](https://pkg.go.dev/badge/github.com/jackc/puddle/v2.svg)](https://pkg.go.dev/github.com/jackc/puddle/v2)
![Build Status](https://github.com/jackc/puddle/actions/workflows/ci.yml/badge.svg)
# Puddle
Puddle is a tiny generic resource pool library for Go that uses the standard
context library to signal cancellation of acquires. It is designed to contain
the minimum functionality required for a resource pool. It can be used directly
or it can be used as the base for a domain specific resource pool. For example,
a database connection pool may use puddle internally and implement health checks
and keep-alive behavior without needing to implement any concurrent code of its
own.
## Features
* Acquire cancellation via context standard library
* Statistics API for monitoring pool pressure
* No dependencies outside of standard library and golang.org/x/sync
* High performance
* 100% test coverage of reachable code
## Example Usage
```go
package main
import (
"context"
"log"
"net"
"github.com/jackc/puddle/v2"
)
func main() {
constructor := func(context.Context) (net.Conn, error) {
return net.Dial("tcp", "127.0.0.1:8080")
}
destructor := func(value net.Conn) {
value.Close()
}
maxPoolSize := int32(10)
pool, err := puddle.NewPool(&puddle.Config[net.Conn]{Constructor: constructor, Destructor: destructor, MaxSize: maxPoolSize})
if err != nil {
log.Fatal(err)
}
// Acquire resource from the pool.
res, err := pool.Acquire(context.Background())
if err != nil {
log.Fatal(err)
}
// Use resource.
_, err = res.Value().Write([]byte{1})
if err != nil {
log.Fatal(err)
}
// Release when done.
res.Release()
}
```
## Status
Puddle is stable and feature complete.
* Bug reports and fixes are welcome.
* New features will usually not be accepted if they can be feasibly implemented in a wrapper.
* Performance optimizations will usually not be accepted unless the performance issue rises to the level of a bug.
## Supported Go Versions
puddle supports the same versions of Go that are supported by the Go project. For [Go](https://golang.org/doc/devel/release.html#policy) that is the two most recent major releases. This means puddle supports Go 1.19 and higher.
## License
MIT
+24
View File
@@ -0,0 +1,24 @@
package puddle
import (
"context"
"time"
)
// valueCancelCtx combines two contexts into one. One context is used for values and the other is used for cancellation.
type valueCancelCtx struct {
valueCtx context.Context
cancelCtx context.Context
}
func (ctx *valueCancelCtx) Deadline() (time.Time, bool) { return ctx.cancelCtx.Deadline() }
func (ctx *valueCancelCtx) Done() <-chan struct{} { return ctx.cancelCtx.Done() }
func (ctx *valueCancelCtx) Err() error { return ctx.cancelCtx.Err() }
func (ctx *valueCancelCtx) Value(key any) any { return ctx.valueCtx.Value(key) }
func newValueCancelCtx(valueCtx, cancelContext context.Context) context.Context {
return &valueCancelCtx{
valueCtx: valueCtx,
cancelCtx: cancelContext,
}
}
+11
View File
@@ -0,0 +1,11 @@
// Package puddle is a generic resource pool with type-parametrized api.
/*
Puddle is a tiny generic resource pool library for Go that uses the standard
context library to signal cancellation of acquires. It is designed to contain
the minimum functionality a resource pool needs that cannot be implemented
without concurrency concerns. For example, a database connection pool may use
puddle internally and implement health checks and keep-alive behavior without
needing to implement any concurrent code of its own.
*/
package puddle
+85
View File
@@ -0,0 +1,85 @@
package genstack
// GenStack implements a generational stack.
//
// GenStack works as common stack except for the fact that all elements in the
// older generation are guaranteed to be popped before any element in the newer
// generation. New elements are always pushed to the current (newest)
// generation.
//
// We could also say that GenStack behaves as a stack in case of a single
// generation, but it behaves as a queue of individual generation stacks.
type GenStack[T any] struct {
// We can represent arbitrary number of generations using 2 stacks. The
// new stack stores all new pushes and the old stack serves all reads.
// Old stack can represent multiple generations. If old == new, then all
// elements pushed in previous (not current) generations have already
// been popped.
old *stack[T]
new *stack[T]
}
// NewGenStack creates a new empty GenStack.
func NewGenStack[T any]() *GenStack[T] {
s := &stack[T]{}
return &GenStack[T]{
old: s,
new: s,
}
}
func (s *GenStack[T]) Pop() (T, bool) {
// Pushes always append to the new stack, so if the old once becomes
// empty, it will remail empty forever.
if s.old.len() == 0 && s.old != s.new {
s.old = s.new
}
if s.old.len() == 0 {
var zero T
return zero, false
}
return s.old.pop(), true
}
// Push pushes a new element at the top of the stack.
func (s *GenStack[T]) Push(v T) { s.new.push(v) }
// NextGen starts a new stack generation.
func (s *GenStack[T]) NextGen() {
if s.old == s.new {
s.new = &stack[T]{}
return
}
// We need to pop from the old stack to the top of the new stack. Let's
// have an example:
//
// Old: <bottom> 4 3 2 1
// New: <bottom> 8 7 6 5
// PopOrder: 1 2 3 4 5 6 7 8
//
//
// To preserve pop order, we have to take all elements from the old
// stack and push them to the top of new stack:
//
// New: 8 7 6 5 4 3 2 1
//
s.new.push(s.old.takeAll()...)
// We have the old stack allocated and empty, so why not to reuse it as
// new new stack.
s.old, s.new = s.new, s.old
}
// Len returns number of elements in the stack.
func (s *GenStack[T]) Len() int {
l := s.old.len()
if s.old != s.new {
l += s.new.len()
}
return l
}
+39
View File
@@ -0,0 +1,39 @@
package genstack
// stack is a wrapper around an array implementing a stack.
//
// We cannot use slice to represent the stack because append might change the
// pointer value of the slice. That would be an issue in GenStack
// implementation.
type stack[T any] struct {
arr []T
}
// push pushes a new element at the top of a stack.
func (s *stack[T]) push(vs ...T) { s.arr = append(s.arr, vs...) }
// pop pops the stack top-most element.
//
// If stack length is zero, this method panics.
func (s *stack[T]) pop() T {
idx := s.len() - 1
val := s.arr[idx]
// Avoid memory leak
var zero T
s.arr[idx] = zero
s.arr = s.arr[:idx]
return val
}
// takeAll returns all elements in the stack in order as they are stored - i.e.
// the top-most stack element is the last one.
func (s *stack[T]) takeAll() []T {
arr := s.arr
s.arr = nil
return arr
}
// len returns number of elements in the stack.
func (s *stack[T]) len() int { return len(s.arr) }
+32
View File
@@ -0,0 +1,32 @@
package puddle
import "unsafe"
type ints interface {
int | int8 | int16 | int32 | int64 | uint | uint8 | uint16 | uint32 | uint64
}
// log2Int returns log2 of an integer. This function panics if val < 0. For val
// == 0, returns 0.
func log2Int[T ints](val T) uint8 {
if val <= 0 {
panic("log2 of non-positive number does not exist")
}
return log2IntRange(val, 0, uint8(8*unsafe.Sizeof(val)))
}
func log2IntRange[T ints](val T, begin, end uint8) uint8 {
length := end - begin
if length == 1 {
return begin
}
delim := begin + length/2
mask := T(1) << delim
if mask > val {
return log2IntRange(val, begin, delim)
} else {
return log2IntRange(val, delim, end)
}
}
+16
View File
@@ -0,0 +1,16 @@
package puddle
import "time"
// nanotime returns the time in nanoseconds since process start.
//
// This approach, described at
// https://github.com/golang/go/issues/61765#issuecomment-1672090302,
// is fast, monotonic, and portable, and avoids the previous
// dependence on runtime.nanotime using the (unsafe) linkname hack.
// In particular, time.Since does less work than time.Now.
func nanotime() int64 {
return time.Since(globalStart).Nanoseconds()
}
var globalStart = time.Now()
+710
View File
@@ -0,0 +1,710 @@
package puddle
import (
"context"
"errors"
"sync"
"sync/atomic"
"time"
"github.com/jackc/puddle/v2/internal/genstack"
"golang.org/x/sync/semaphore"
)
const (
resourceStatusConstructing = 0
resourceStatusIdle = iota
resourceStatusAcquired = iota
resourceStatusHijacked = iota
)
// ErrClosedPool occurs on an attempt to acquire a connection from a closed pool
// or a pool that is closed while the acquire is waiting.
var ErrClosedPool = errors.New("closed pool")
// ErrNotAvailable occurs on an attempt to acquire a resource from a pool
// that is at maximum capacity and has no available resources.
var ErrNotAvailable = errors.New("resource not available")
// Constructor is a function called by the pool to construct a resource.
type Constructor[T any] func(ctx context.Context) (res T, err error)
// Destructor is a function called by the pool to destroy a resource.
type Destructor[T any] func(res T)
// Resource is the resource handle returned by acquiring from the pool.
type Resource[T any] struct {
value T
pool *Pool[T]
creationTime time.Time
lastUsedNano int64
poolResetCount int
status byte
}
// Value returns the resource value.
func (res *Resource[T]) Value() T {
if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
panic("tried to access resource that is not acquired or hijacked")
}
return res.value
}
// Release returns the resource to the pool. res must not be subsequently used.
func (res *Resource[T]) Release() {
if res.status != resourceStatusAcquired {
panic("tried to release resource that is not acquired")
}
res.pool.releaseAcquiredResource(res, nanotime())
}
// ReleaseUnused returns the resource to the pool without updating when it was last used used. i.e. LastUsedNanotime
// will not change. res must not be subsequently used.
func (res *Resource[T]) ReleaseUnused() {
if res.status != resourceStatusAcquired {
panic("tried to release resource that is not acquired")
}
res.pool.releaseAcquiredResource(res, res.lastUsedNano)
}
// Destroy returns the resource to the pool for destruction. res must not be
// subsequently used.
func (res *Resource[T]) Destroy() {
if res.status != resourceStatusAcquired {
panic("tried to destroy resource that is not acquired")
}
go res.pool.destroyAcquiredResource(res)
}
// Hijack assumes ownership of the resource from the pool. Caller is responsible
// for cleanup of resource value.
func (res *Resource[T]) Hijack() {
if res.status != resourceStatusAcquired {
panic("tried to hijack resource that is not acquired")
}
res.pool.hijackAcquiredResource(res)
}
// CreationTime returns when the resource was created by the pool.
func (res *Resource[T]) CreationTime() time.Time {
if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
panic("tried to access resource that is not acquired or hijacked")
}
return res.creationTime
}
// LastUsedNanotime returns when Release was last called on the resource measured in nanoseconds from an arbitrary time
// (a monotonic time). Returns creation time if Release has never been called. This is only useful to compare with
// other calls to LastUsedNanotime. In almost all cases, IdleDuration should be used instead.
func (res *Resource[T]) LastUsedNanotime() int64 {
if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
panic("tried to access resource that is not acquired or hijacked")
}
return res.lastUsedNano
}
// IdleDuration returns the duration since Release was last called on the resource. This is equivalent to subtracting
// LastUsedNanotime to the current nanotime.
func (res *Resource[T]) IdleDuration() time.Duration {
if !(res.status == resourceStatusAcquired || res.status == resourceStatusHijacked) {
panic("tried to access resource that is not acquired or hijacked")
}
return time.Duration(nanotime() - res.lastUsedNano)
}
// Pool is a concurrency-safe resource pool.
type Pool[T any] struct {
// mux is the pool internal lock. Any modification of shared state of
// the pool (but Acquires of acquireSem) must be performed only by
// holder of the lock. Long running operations are not allowed when mux
// is held.
mux sync.Mutex
// acquireSem provides an allowance to acquire a resource.
//
// Releases are allowed only when caller holds mux. Acquires have to
// happen before mux is locked (doesn't apply to semaphore.TryAcquire in
// AcquireAllIdle).
acquireSem *semaphore.Weighted
destructWG sync.WaitGroup
allResources resList[T]
idleResources *genstack.GenStack[*Resource[T]]
constructor Constructor[T]
destructor Destructor[T]
maxSize int32
acquireCount int64
acquireDuration time.Duration
emptyAcquireCount int64
emptyAcquireWaitTime time.Duration
canceledAcquireCount atomic.Int64
resetCount int
baseAcquireCtx context.Context
cancelBaseAcquireCtx context.CancelFunc
closed bool
}
type Config[T any] struct {
Constructor Constructor[T]
Destructor Destructor[T]
MaxSize int32
}
// NewPool creates a new pool. Returns an error iff MaxSize is less than 1.
func NewPool[T any](config *Config[T]) (*Pool[T], error) {
if config.MaxSize < 1 {
return nil, errors.New("MaxSize must be >= 1")
}
baseAcquireCtx, cancelBaseAcquireCtx := context.WithCancel(context.Background())
return &Pool[T]{
acquireSem: semaphore.NewWeighted(int64(config.MaxSize)),
idleResources: genstack.NewGenStack[*Resource[T]](),
maxSize: config.MaxSize,
constructor: config.Constructor,
destructor: config.Destructor,
baseAcquireCtx: baseAcquireCtx,
cancelBaseAcquireCtx: cancelBaseAcquireCtx,
}, nil
}
// Close destroys all resources in the pool and rejects future Acquire calls.
// Blocks until all resources are returned to pool and destroyed.
func (p *Pool[T]) Close() {
defer p.destructWG.Wait()
p.mux.Lock()
defer p.mux.Unlock()
if p.closed {
return
}
p.closed = true
p.cancelBaseAcquireCtx()
for res, ok := p.idleResources.Pop(); ok; res, ok = p.idleResources.Pop() {
p.allResources.remove(res)
go p.destructResourceValue(res.value)
}
}
// Stat is a snapshot of Pool statistics.
type Stat struct {
constructingResources int32
acquiredResources int32
idleResources int32
maxResources int32
acquireCount int64
acquireDuration time.Duration
emptyAcquireCount int64
emptyAcquireWaitTime time.Duration
canceledAcquireCount int64
}
// TotalResources returns the total number of resources currently in the pool.
// The value is the sum of ConstructingResources, AcquiredResources, and
// IdleResources.
func (s *Stat) TotalResources() int32 {
return s.constructingResources + s.acquiredResources + s.idleResources
}
// ConstructingResources returns the number of resources with construction in progress in
// the pool.
func (s *Stat) ConstructingResources() int32 {
return s.constructingResources
}
// AcquiredResources returns the number of currently acquired resources in the pool.
func (s *Stat) AcquiredResources() int32 {
return s.acquiredResources
}
// IdleResources returns the number of currently idle resources in the pool.
func (s *Stat) IdleResources() int32 {
return s.idleResources
}
// MaxResources returns the maximum size of the pool.
func (s *Stat) MaxResources() int32 {
return s.maxResources
}
// AcquireCount returns the cumulative count of successful acquires from the pool.
func (s *Stat) AcquireCount() int64 {
return s.acquireCount
}
// AcquireDuration returns the total duration of all successful acquires from
// the pool.
func (s *Stat) AcquireDuration() time.Duration {
return s.acquireDuration
}
// EmptyAcquireCount returns the cumulative count of successful acquires from the pool
// that waited for a resource to be released or constructed because the pool was
// empty.
func (s *Stat) EmptyAcquireCount() int64 {
return s.emptyAcquireCount
}
// EmptyAcquireWaitTime returns the cumulative time waited for successful acquires
// from the pool for a resource to be released or constructed because the pool was
// empty.
func (s *Stat) EmptyAcquireWaitTime() time.Duration {
return s.emptyAcquireWaitTime
}
// CanceledAcquireCount returns the cumulative count of acquires from the pool
// that were canceled by a context.
func (s *Stat) CanceledAcquireCount() int64 {
return s.canceledAcquireCount
}
// Stat returns the current pool statistics.
func (p *Pool[T]) Stat() *Stat {
p.mux.Lock()
defer p.mux.Unlock()
s := &Stat{
maxResources: p.maxSize,
acquireCount: p.acquireCount,
emptyAcquireCount: p.emptyAcquireCount,
emptyAcquireWaitTime: p.emptyAcquireWaitTime,
canceledAcquireCount: p.canceledAcquireCount.Load(),
acquireDuration: p.acquireDuration,
}
for _, res := range p.allResources {
switch res.status {
case resourceStatusConstructing:
s.constructingResources += 1
case resourceStatusIdle:
s.idleResources += 1
case resourceStatusAcquired:
s.acquiredResources += 1
}
}
return s
}
// tryAcquireIdleResource checks if there is any idle resource. If there is
// some, this method removes it from idle list and returns it. If the idle pool
// is empty, this method returns nil and doesn't modify the idleResources slice.
//
// WARNING: Caller of this method must hold the pool mutex!
func (p *Pool[T]) tryAcquireIdleResource() *Resource[T] {
res, ok := p.idleResources.Pop()
if !ok {
return nil
}
res.status = resourceStatusAcquired
return res
}
// createNewResource creates a new resource and inserts it into list of pool
// resources.
//
// WARNING: Caller of this method must hold the pool mutex!
func (p *Pool[T]) createNewResource() *Resource[T] {
res := &Resource[T]{
pool: p,
creationTime: time.Now(),
lastUsedNano: nanotime(),
poolResetCount: p.resetCount,
status: resourceStatusConstructing,
}
p.allResources.append(res)
p.destructWG.Add(1)
return res
}
// Acquire gets a resource from the pool. If no resources are available and the pool is not at maximum capacity it will
// create a new resource. If the pool is at maximum capacity it will block until a resource is available. ctx can be
// used to cancel the Acquire.
//
// If Acquire creates a new resource the resource constructor function will receive a context that delegates Value() to
// ctx. Canceling ctx will cause Acquire to return immediately but it will not cancel the resource creation. This avoids
// the problem of it being impossible to create resources when the time to create a resource is greater than any one
// caller of Acquire is willing to wait.
func (p *Pool[T]) Acquire(ctx context.Context) (_ *Resource[T], err error) {
select {
case <-ctx.Done():
p.canceledAcquireCount.Add(1)
return nil, ctx.Err()
default:
}
return p.acquire(ctx)
}
// acquire is a continuation of Acquire function that doesn't check context
// validity.
//
// This function exists solely only for benchmarking purposes.
func (p *Pool[T]) acquire(ctx context.Context) (*Resource[T], error) {
startNano := nanotime()
var waitedForLock bool
if !p.acquireSem.TryAcquire(1) {
waitedForLock = true
err := p.acquireSem.Acquire(ctx, 1)
if err != nil {
p.canceledAcquireCount.Add(1)
return nil, err
}
}
p.mux.Lock()
if p.closed {
p.acquireSem.Release(1)
p.mux.Unlock()
return nil, ErrClosedPool
}
// If a resource is available in the pool.
if res := p.tryAcquireIdleResource(); res != nil {
waitTime := time.Duration(nanotime() - startNano)
if waitedForLock {
p.emptyAcquireCount += 1
p.emptyAcquireWaitTime += waitTime
}
p.acquireCount += 1
p.acquireDuration += waitTime
p.mux.Unlock()
return res, nil
}
if len(p.allResources) >= int(p.maxSize) {
// Unreachable code.
panic("bug: semaphore allowed more acquires than pool allows")
}
// The resource is not idle, but there is enough space to create one.
res := p.createNewResource()
p.mux.Unlock()
res, err := p.initResourceValue(ctx, res)
if err != nil {
return nil, err
}
p.mux.Lock()
defer p.mux.Unlock()
p.emptyAcquireCount += 1
p.acquireCount += 1
waitTime := time.Duration(nanotime() - startNano)
p.acquireDuration += waitTime
p.emptyAcquireWaitTime += waitTime
return res, nil
}
func (p *Pool[T]) initResourceValue(ctx context.Context, res *Resource[T]) (*Resource[T], error) {
// Create the resource in a goroutine to immediately return from Acquire
// if ctx is canceled without also canceling the constructor.
//
// See:
// - https://github.com/jackc/pgx/issues/1287
// - https://github.com/jackc/pgx/issues/1259
constructErrChan := make(chan error)
go func() {
constructorCtx := newValueCancelCtx(ctx, p.baseAcquireCtx)
value, err := p.constructor(constructorCtx)
if err != nil {
p.mux.Lock()
p.allResources.remove(res)
p.destructWG.Done()
// The resource won't be acquired because its
// construction failed. We have to allow someone else to
// take that resouce.
p.acquireSem.Release(1)
p.mux.Unlock()
select {
case constructErrChan <- err:
case <-ctx.Done():
// The caller is cancelled, so no-one awaits the
// error. This branch avoid goroutine leak.
}
return
}
// The resource is already in p.allResources where it might be read. So we need to acquire the lock to update its
// status.
p.mux.Lock()
res.value = value
res.status = resourceStatusAcquired
p.mux.Unlock()
// This select works because the channel is unbuffered.
select {
case constructErrChan <- nil:
case <-ctx.Done():
p.releaseAcquiredResource(res, res.lastUsedNano)
}
}()
select {
case <-ctx.Done():
p.canceledAcquireCount.Add(1)
return nil, ctx.Err()
case err := <-constructErrChan:
if err != nil {
return nil, err
}
return res, nil
}
}
// TryAcquire gets a resource from the pool if one is immediately available. If not, it returns ErrNotAvailable. If no
// resources are available but the pool has room to grow, a resource will be created in the background. ctx is only
// used to cancel the background creation.
func (p *Pool[T]) TryAcquire(ctx context.Context) (*Resource[T], error) {
if !p.acquireSem.TryAcquire(1) {
return nil, ErrNotAvailable
}
p.mux.Lock()
defer p.mux.Unlock()
if p.closed {
p.acquireSem.Release(1)
return nil, ErrClosedPool
}
// If a resource is available now
if res := p.tryAcquireIdleResource(); res != nil {
p.acquireCount += 1
return res, nil
}
if len(p.allResources) >= int(p.maxSize) {
// Unreachable code.
panic("bug: semaphore allowed more acquires than pool allows")
}
res := p.createNewResource()
go func() {
value, err := p.constructor(ctx)
p.mux.Lock()
defer p.mux.Unlock()
// We have to create the resource and only then release the
// semaphore - For the time being there is no resource that
// someone could acquire.
defer p.acquireSem.Release(1)
if err != nil {
p.allResources.remove(res)
p.destructWG.Done()
return
}
res.value = value
res.status = resourceStatusIdle
p.idleResources.Push(res)
}()
return nil, ErrNotAvailable
}
// acquireSemAll tries to acquire num free tokens from sem. This function is
// guaranteed to acquire at least the lowest number of tokens that has been
// available in the semaphore during runtime of this function.
//
// For the time being, semaphore doesn't allow to acquire all tokens atomically
// (see https://github.com/golang/sync/pull/19). We simulate this by trying all
// powers of 2 that are less or equal to num.
//
// For example, let's immagine we have 19 free tokens in the semaphore which in
// total has 24 tokens (i.e. the maxSize of the pool is 24 resources). Then if
// num is 24, the log2Uint(24) is 4 and we try to acquire 16, 8, 4, 2 and 1
// tokens. Out of those, the acquire of 16, 2 and 1 tokens will succeed.
//
// Naturally, Acquires and Releases of the semaphore might take place
// concurrently. For this reason, it's not guaranteed that absolutely all free
// tokens in the semaphore will be acquired. But it's guaranteed that at least
// the minimal number of tokens that has been present over the whole process
// will be acquired. This is sufficient for the use-case we have in this
// package.
//
// TODO: Replace this with acquireSem.TryAcquireAll() if it gets to
// upstream. https://github.com/golang/sync/pull/19
func acquireSemAll(sem *semaphore.Weighted, num int) int {
if sem.TryAcquire(int64(num)) {
return num
}
var acquired int
for i := int(log2Int(num)); i >= 0; i-- {
val := 1 << i
if sem.TryAcquire(int64(val)) {
acquired += val
}
}
return acquired
}
// AcquireAllIdle acquires all currently idle resources. Its intended use is for
// health check and keep-alive functionality. It does not update pool
// statistics.
func (p *Pool[T]) AcquireAllIdle() []*Resource[T] {
p.mux.Lock()
defer p.mux.Unlock()
if p.closed {
return nil
}
numIdle := p.idleResources.Len()
if numIdle == 0 {
return nil
}
// In acquireSemAll we use only TryAcquire and not Acquire. Because
// TryAcquire cannot block, the fact that we hold mutex locked and try
// to acquire semaphore cannot result in dead-lock.
//
// Because the mutex is locked, no parallel Release can run. This
// implies that the number of tokens can only decrease because some
// Acquire/TryAcquire call can consume the semaphore token. Consequently
// acquired is always less or equal to numIdle. Moreover if acquired <
// numIdle, then there are some parallel Acquire/TryAcquire calls that
// will take the remaining idle connections.
acquired := acquireSemAll(p.acquireSem, numIdle)
idle := make([]*Resource[T], acquired)
for i := range idle {
res, _ := p.idleResources.Pop()
res.status = resourceStatusAcquired
idle[i] = res
}
// We have to bump the generation to ensure that Acquire/TryAcquire
// calls running in parallel (those which caused acquired < numIdle)
// will consume old connections and not freshly released connections
// instead.
p.idleResources.NextGen()
return idle
}
// CreateResource constructs a new resource without acquiring it. It goes straight in the IdlePool. If the pool is full
// it returns an error. It can be useful to maintain warm resources under little load.
func (p *Pool[T]) CreateResource(ctx context.Context) error {
if !p.acquireSem.TryAcquire(1) {
return ErrNotAvailable
}
p.mux.Lock()
if p.closed {
p.acquireSem.Release(1)
p.mux.Unlock()
return ErrClosedPool
}
if len(p.allResources) >= int(p.maxSize) {
p.acquireSem.Release(1)
p.mux.Unlock()
return ErrNotAvailable
}
res := p.createNewResource()
p.mux.Unlock()
value, err := p.constructor(ctx)
p.mux.Lock()
defer p.mux.Unlock()
defer p.acquireSem.Release(1)
if err != nil {
p.allResources.remove(res)
p.destructWG.Done()
return err
}
res.value = value
res.status = resourceStatusIdle
// If closed while constructing resource then destroy it and return an error
if p.closed {
go p.destructResourceValue(res.value)
return ErrClosedPool
}
p.idleResources.Push(res)
return nil
}
// Reset destroys all resources, but leaves the pool open. It is intended for use when an error is detected that would
// disrupt all resources (such as a network interruption or a server state change).
//
// It is safe to reset a pool while resources are checked out. Those resources will be destroyed when they are returned
// to the pool.
func (p *Pool[T]) Reset() {
p.mux.Lock()
defer p.mux.Unlock()
p.resetCount++
for res, ok := p.idleResources.Pop(); ok; res, ok = p.idleResources.Pop() {
p.allResources.remove(res)
go p.destructResourceValue(res.value)
}
}
// releaseAcquiredResource returns res to the the pool.
func (p *Pool[T]) releaseAcquiredResource(res *Resource[T], lastUsedNano int64) {
p.mux.Lock()
defer p.mux.Unlock()
defer p.acquireSem.Release(1)
if p.closed || res.poolResetCount != p.resetCount {
p.allResources.remove(res)
go p.destructResourceValue(res.value)
} else {
res.lastUsedNano = lastUsedNano
res.status = resourceStatusIdle
p.idleResources.Push(res)
}
}
// Remove removes res from the pool and closes it. If res is not part of the
// pool Remove will panic.
func (p *Pool[T]) destroyAcquiredResource(res *Resource[T]) {
p.destructResourceValue(res.value)
p.mux.Lock()
defer p.mux.Unlock()
defer p.acquireSem.Release(1)
p.allResources.remove(res)
}
func (p *Pool[T]) hijackAcquiredResource(res *Resource[T]) {
p.mux.Lock()
defer p.mux.Unlock()
defer p.acquireSem.Release(1)
p.allResources.remove(res)
res.status = resourceStatusHijacked
p.destructWG.Done() // not responsible for destructing hijacked resources
}
func (p *Pool[T]) destructResourceValue(value T) {
p.destructor(value)
p.destructWG.Done()
}
+28
View File
@@ -0,0 +1,28 @@
package puddle
type resList[T any] []*Resource[T]
func (l *resList[T]) append(val *Resource[T]) { *l = append(*l, val) }
func (l *resList[T]) popBack() *Resource[T] {
idx := len(*l) - 1
val := (*l)[idx]
(*l)[idx] = nil // Avoid memory leak
*l = (*l)[:idx]
return val
}
func (l *resList[T]) remove(val *Resource[T]) {
for i, elem := range *l {
if elem == val {
lastIdx := len(*l) - 1
(*l)[i] = (*l)[lastIdx]
(*l)[lastIdx] = nil // Avoid memory leak
(*l) = (*l)[:lastIdx]
return
}
}
panic("BUG: removeResource could not find res in slice")
}
+24
View File
@@ -0,0 +1,24 @@
Copyright (c) 2021 Vladimir Mihailenco. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+284
View File
@@ -0,0 +1,284 @@
package pgdialect
import (
"context"
"fmt"
"strings"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate"
"github.com/uptrace/bun/migrate/sqlschema"
"github.com/uptrace/bun/schema"
)
func (d *Dialect) NewMigrator(db *bun.DB, schemaName string) sqlschema.Migrator {
return &migrator{db: db, schemaName: schemaName, BaseMigrator: sqlschema.NewBaseMigrator(db)}
}
type migrator struct {
*sqlschema.BaseMigrator
db *bun.DB
schemaName string
}
var _ sqlschema.Migrator = (*migrator)(nil)
func (m *migrator) AppendSQL(b []byte, operation any) (_ []byte, err error) {
gen := m.db.QueryGen()
// Append ALTER TABLE statement to the enclosed query bytes []byte.
appendAlterTable := func(query []byte, tableName string) []byte {
query = append(query, "ALTER TABLE "...)
query = m.appendFQN(gen, query, tableName)
return append(query, " "...)
}
switch change := operation.(type) {
case *migrate.CreateTableOp:
return m.AppendCreateTable(b, change.Model)
case *migrate.DropTableOp:
return m.AppendDropTable(b, m.schemaName, change.TableName)
case *migrate.RenameTableOp:
b, err = m.renameTable(gen, appendAlterTable(b, change.TableName), change)
case *migrate.RenameColumnOp:
b, err = m.renameColumn(gen, appendAlterTable(b, change.TableName), change)
case *migrate.AddColumnOp:
b, err = m.addColumn(gen, appendAlterTable(b, change.TableName), change)
case *migrate.DropColumnOp:
b, err = m.dropColumn(gen, appendAlterTable(b, change.TableName), change)
case *migrate.AddPrimaryKeyOp:
b, err = m.addPrimaryKey(gen, appendAlterTable(b, change.TableName), change.PrimaryKey)
case *migrate.ChangePrimaryKeyOp:
b, err = m.changePrimaryKey(gen, appendAlterTable(b, change.TableName), change)
case *migrate.DropPrimaryKeyOp:
b, err = m.dropConstraint(gen, appendAlterTable(b, change.TableName), change.PrimaryKey.Name)
case *migrate.AddUniqueConstraintOp:
b, err = m.addUnique(gen, appendAlterTable(b, change.TableName), change)
case *migrate.DropUniqueConstraintOp:
b, err = m.dropConstraint(gen, appendAlterTable(b, change.TableName), change.Unique.Name)
case *migrate.ChangeColumnTypeOp:
// If column changes to SERIAL, create sequence first.
// https://gist.github.com/oleglomako/185df689706c5499612a0d54d3ffe856
if !change.From.GetIsAutoIncrement() && change.To.GetIsAutoIncrement() {
change.To, b, err = m.createDefaultSequence(gen, b, change)
}
b, err = m.changeColumnType(gen, appendAlterTable(b, change.TableName), change)
case *migrate.AddForeignKeyOp:
b, err = m.addForeignKey(gen, appendAlterTable(b, change.TableName()), change)
case *migrate.DropForeignKeyOp:
b, err = m.dropConstraint(gen, appendAlterTable(b, change.TableName()), change.ConstraintName)
default:
return nil, fmt.Errorf("append sql: unknown operation %T", change)
}
if err != nil {
return nil, fmt.Errorf("append sql: %w", err)
}
return b, nil
}
func (m *migrator) appendFQN(gen schema.QueryGen, b []byte, tableName string) []byte {
return gen.AppendQuery(b, "?.?", bun.Ident(m.schemaName), bun.Ident(tableName))
}
func (m *migrator) renameTable(gen schema.QueryGen, b []byte, rename *migrate.RenameTableOp) (_ []byte, err error) {
b = append(b, "RENAME TO "...)
b = gen.AppendName(b, rename.NewName)
return b, nil
}
func (m *migrator) renameColumn(gen schema.QueryGen, b []byte, rename *migrate.RenameColumnOp) (_ []byte, err error) {
b = append(b, "RENAME COLUMN "...)
b = gen.AppendName(b, rename.OldName)
b = append(b, " TO "...)
b = gen.AppendName(b, rename.NewName)
return b, nil
}
func (m *migrator) addColumn(gen schema.QueryGen, b []byte, add *migrate.AddColumnOp) (_ []byte, err error) {
b = append(b, "ADD COLUMN "...)
b = gen.AppendName(b, add.ColumnName)
b = append(b, " "...)
b, err = add.Column.AppendQuery(gen, b)
if err != nil {
return nil, err
}
if add.Column.GetDefaultValue() != "" {
b = append(b, " DEFAULT "...)
b = append(b, add.Column.GetDefaultValue()...)
b = append(b, " "...)
}
if add.Column.GetIsIdentity() {
b = appendGeneratedAsIdentity(b)
}
return b, nil
}
func (m *migrator) dropColumn(gen schema.QueryGen, b []byte, drop *migrate.DropColumnOp) (_ []byte, err error) {
b = append(b, "DROP COLUMN "...)
b = gen.AppendName(b, drop.ColumnName)
return b, nil
}
func (m *migrator) addPrimaryKey(gen schema.QueryGen, b []byte, pk sqlschema.PrimaryKey) (_ []byte, err error) {
b = append(b, "ADD PRIMARY KEY ("...)
b, _ = pk.Columns.AppendQuery(gen, b)
b = append(b, ")"...)
return b, nil
}
func (m *migrator) changePrimaryKey(gen schema.QueryGen, b []byte, change *migrate.ChangePrimaryKeyOp) (_ []byte, err error) {
b, _ = m.dropConstraint(gen, b, change.Old.Name)
b = append(b, ", "...)
b, _ = m.addPrimaryKey(gen, b, change.New)
return b, nil
}
func (m *migrator) addUnique(gen schema.QueryGen, b []byte, change *migrate.AddUniqueConstraintOp) (_ []byte, err error) {
b = append(b, "ADD CONSTRAINT "...)
if change.Unique.Name != "" {
b = gen.AppendName(b, change.Unique.Name)
} else {
// Default naming scheme for unique constraints in Postgres is <table>_<column>_key
b = gen.AppendName(b, fmt.Sprintf("%s_%s_key", change.TableName, change.Unique.Columns))
}
b = append(b, " UNIQUE ("...)
b, _ = change.Unique.Columns.AppendQuery(gen, b)
b = append(b, ")"...)
return b, nil
}
func (m *migrator) dropConstraint(gen schema.QueryGen, b []byte, name string) (_ []byte, err error) {
b = append(b, "DROP CONSTRAINT "...)
b = gen.AppendName(b, name)
return b, nil
}
func (m *migrator) addForeignKey(gen schema.QueryGen, b []byte, add *migrate.AddForeignKeyOp) (_ []byte, err error) {
b = append(b, "ADD CONSTRAINT "...)
name := add.ConstraintName
if name == "" {
colRef := add.ForeignKey.From
columns := strings.Join(colRef.Column.Split(), "_")
name = fmt.Sprintf("%s_%s_fkey", colRef.TableName, columns)
}
b = gen.AppendName(b, name)
b = append(b, " FOREIGN KEY ("...)
if b, err = add.ForeignKey.From.Column.AppendQuery(gen, b); err != nil {
return b, err
}
b = append(b, ")"...)
b = append(b, " REFERENCES "...)
b = m.appendFQN(gen, b, add.ForeignKey.To.TableName)
b = append(b, " ("...)
if b, err = add.ForeignKey.To.Column.AppendQuery(gen, b); err != nil {
return b, err
}
b = append(b, ")"...)
return b, nil
}
// createDefaultSequence creates a SEQUENCE to back a serial column.
// Having a backing sequence is necessary to change column type to SERIAL.
// The updated Column's default is set to "nextval" of the new sequence.
func (m *migrator) createDefaultSequence(_ schema.QueryGen, b []byte, op *migrate.ChangeColumnTypeOp) (_ sqlschema.Column, _ []byte, err error) {
var last int
if err = m.db.NewSelect().Table(op.TableName).
ColumnExpr("MAX(?)", op.Column).Scan(context.TODO(), &last); err != nil {
return nil, b, err
}
seq := op.TableName + "_" + op.Column + "_seq"
fqn := op.TableName + "." + op.Column
// A sequence that is OWNED BY a table will be dropped
// if the table is dropped with CASCADE action.
b = append(b, "CREATE SEQUENCE "...)
b = append(b, seq...)
b = append(b, " START WITH "...)
b = append(b, fmt.Sprint(last+1)...) // start with next value
b = append(b, " OWNED BY "...)
b = append(b, fqn...)
b = append(b, ";\n"...)
return &Column{
Name: op.To.GetName(),
SQLType: op.To.GetSQLType(),
VarcharLen: op.To.GetVarcharLen(),
DefaultValue: fmt.Sprintf("nextval('%s'::regclass)", seq),
IsNullable: op.To.GetIsNullable(),
IsAutoIncrement: op.To.GetIsAutoIncrement(),
IsIdentity: op.To.GetIsIdentity(),
}, b, nil
}
func (m *migrator) changeColumnType(gen schema.QueryGen, b []byte, colDef *migrate.ChangeColumnTypeOp) (_ []byte, err error) {
// alterColumn never re-assigns err, so there is no need to check for err != nil after calling it
var i int
appendAlterColumn := func() {
if i > 0 {
b = append(b, ", "...)
}
b = append(b, "ALTER COLUMN "...)
b = gen.AppendName(b, colDef.Column)
i++
}
got, want := colDef.From, colDef.To
inspector := m.db.Dialect().(sqlschema.InspectorDialect)
if !inspector.CompareType(want, got) {
appendAlterColumn()
b = append(b, " SET DATA TYPE "...)
if b, err = want.AppendQuery(gen, b); err != nil {
return b, err
}
}
// Column must be declared NOT NULL before identity can be added.
// Although PG can resolve the order of operations itself, we make this explicit in the query.
if want.GetIsNullable() != got.GetIsNullable() {
appendAlterColumn()
if !want.GetIsNullable() {
b = append(b, " SET NOT NULL"...)
} else {
b = append(b, " DROP NOT NULL"...)
}
}
if want.GetIsIdentity() != got.GetIsIdentity() {
appendAlterColumn()
if !want.GetIsIdentity() {
b = append(b, " DROP IDENTITY"...)
} else {
b = append(b, " ADD"...)
b = appendGeneratedAsIdentity(b)
}
}
if want.GetDefaultValue() != got.GetDefaultValue() {
appendAlterColumn()
if want.GetDefaultValue() == "" {
b = append(b, " DROP DEFAULT"...)
} else {
b = append(b, " SET DEFAULT "...)
b = append(b, want.GetDefaultValue()...)
}
}
return b, nil
}
+87
View File
@@ -0,0 +1,87 @@
package pgdialect
import (
"database/sql/driver"
"fmt"
"reflect"
"time"
"github.com/uptrace/bun/dialect"
"github.com/uptrace/bun/schema"
)
var (
driverValuerType = reflect.TypeFor[driver.Valuer]()
stringType = reflect.TypeFor[string]()
sliceStringType = reflect.TypeFor[[]string]()
intType = reflect.TypeFor[int]()
sliceIntType = reflect.TypeFor[[]int]()
int64Type = reflect.TypeFor[int64]()
sliceInt64Type = reflect.TypeFor[[]int64]()
float64Type = reflect.TypeFor[float64]()
sliceFloat64Type = reflect.TypeFor[[]float64]()
timeType = reflect.TypeFor[time.Time]()
sliceTimeType = reflect.TypeFor[[]time.Time]()
)
func appendTime(buf []byte, tm time.Time) []byte {
return tm.UTC().AppendFormat(buf, "2006-01-02 15:04:05.999999-07:00")
}
var mapStringStringType = reflect.TypeOf(map[string]string(nil))
func (d *Dialect) hstoreAppender(typ reflect.Type) schema.AppenderFunc {
kind := typ.Kind()
switch kind {
case reflect.Ptr:
if fn := d.hstoreAppender(typ.Elem()); fn != nil {
return schema.PtrAppender(fn)
}
case reflect.Map:
// ok:
default:
return nil
}
if typ.Key() == stringType && typ.Elem() == stringType {
return appendMapStringStringValue
}
return func(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
err := fmt.Errorf("bun: Hstore(unsupported %s)", v.Type())
return dialect.AppendError(b, err)
}
}
func appendMapStringString(b []byte, m map[string]string) []byte {
if m == nil {
return dialect.AppendNull(b)
}
b = append(b, '\'')
for key, value := range m {
b = appendStringElem(b, key)
b = append(b, '=', '>')
b = appendStringElem(b, value)
b = append(b, ',')
}
if len(m) > 0 {
b = b[:len(b)-1] // Strip trailing comma.
}
b = append(b, '\'')
return b
}
func appendMapStringStringValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
m := v.Convert(mapStringStringType).Interface().(map[string]string)
return appendMapStringString(b, m)
}
+608
View File
@@ -0,0 +1,608 @@
package pgdialect
import (
"database/sql"
"database/sql/driver"
"fmt"
"math"
"reflect"
"strconv"
"time"
"github.com/uptrace/bun/dialect"
"github.com/uptrace/bun/internal"
"github.com/uptrace/bun/schema"
)
type ArrayValue struct {
v reflect.Value
append schema.AppenderFunc
scan schema.ScannerFunc
}
// Array accepts a slice and returns a wrapper for working with PostgreSQL
// array data type.
//
// For struct fields you can use array tag:
//
// Emails []string `bun:",array"`
func Array(vi any) *ArrayValue {
v := reflect.ValueOf(vi)
if !v.IsValid() {
panic(fmt.Errorf("bun: Array(nil)"))
}
return &ArrayValue{
v: v,
append: pgDialect.arrayAppender(v.Type()),
scan: arrayScanner(v.Type()),
}
}
var (
_ schema.QueryAppender = (*ArrayValue)(nil)
_ sql.Scanner = (*ArrayValue)(nil)
)
func (a *ArrayValue) AppendQuery(gen schema.QueryGen, b []byte) ([]byte, error) {
if a.append == nil {
panic(fmt.Errorf("bun: Array(unsupported %s)", a.v.Type()))
}
return a.append(gen, b, a.v), nil
}
func (a *ArrayValue) Scan(src any) error {
if a.scan == nil {
return fmt.Errorf("bun: Array(unsupported %s)", a.v.Type())
}
if a.v.Kind() != reflect.Ptr {
return fmt.Errorf("bun: Array(non-pointer %s)", a.v.Type())
}
return a.scan(a.v, src)
}
func (a *ArrayValue) Value() any {
if a.v.IsValid() {
return a.v.Interface()
}
return nil
}
//------------------------------------------------------------------------------
func (d *Dialect) arrayAppender(typ reflect.Type) schema.AppenderFunc {
kind := typ.Kind()
switch kind {
case reflect.Ptr:
if fn := d.arrayAppender(typ.Elem()); fn != nil {
return schema.PtrAppender(fn)
}
case reflect.Slice, reflect.Array:
// continue below
default:
return nil
}
elemType := typ.Elem()
if kind == reflect.Slice {
switch elemType {
case stringType:
return appendStringSliceValue
case intType:
return appendIntSliceValue
case int64Type:
return appendInt64SliceValue
case float64Type:
return appendFloat64SliceValue
case timeType:
return appendTimeSliceValue
}
}
appendElem := d.arrayElemAppender(elemType)
if appendElem == nil {
panic(fmt.Errorf("pgdialect: %s is not supported", typ))
}
return func(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
kind := v.Kind()
switch kind {
case reflect.Ptr, reflect.Slice:
if v.IsNil() {
return dialect.AppendNull(b)
}
}
if kind == reflect.Ptr {
v = v.Elem()
}
b = append(b, "'{"...)
ln := v.Len()
for i := 0; i < ln; i++ {
elem := v.Index(i)
if i > 0 {
b = append(b, ',')
}
b = appendElem(gen, b, elem)
}
b = append(b, "}'"...)
return b
}
}
func (d *Dialect) arrayElemAppender(typ reflect.Type) schema.AppenderFunc {
if typ.Implements(driverValuerType) {
return arrayAppendDriverValue
}
if typ == timeType {
return appendTimeElemValue
}
switch typ.Kind() {
case reflect.String:
return appendStringElemValue
case reflect.Slice:
if typ.Elem().Kind() == reflect.Uint8 {
return appendBytesElemValue
}
case reflect.Ptr:
return schema.PtrAppender(d.arrayElemAppender(typ.Elem()))
}
return schema.Appender(d, typ)
}
func appendTimeElemValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
ts := v.Convert(timeType).Interface().(time.Time)
b = append(b, '"')
b = appendTime(b, ts)
return append(b, '"')
}
func appendStringElemValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
return appendStringElem(b, v.String())
}
func appendBytesElemValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
return appendBytesElem(b, v.Bytes())
}
func arrayAppendDriverValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
iface, err := v.Interface().(driver.Valuer).Value()
if err != nil {
return dialect.AppendError(b, err)
}
return appendElem(b, iface)
}
func appendStringSliceValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
ss := v.Convert(sliceStringType).Interface().([]string)
return appendStringSlice(b, ss)
}
func appendStringSlice(b []byte, ss []string) []byte {
if ss == nil {
return dialect.AppendNull(b)
}
b = append(b, '\'')
b = append(b, '{')
for _, s := range ss {
b = appendStringElem(b, s)
b = append(b, ',')
}
if len(ss) > 0 {
b[len(b)-1] = '}' // Replace trailing comma.
} else {
b = append(b, '}')
}
b = append(b, '\'')
return b
}
func appendIntSliceValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
ints := v.Convert(sliceIntType).Interface().([]int)
return appendIntSlice(b, ints)
}
func appendIntSlice(b []byte, ints []int) []byte {
if ints == nil {
return dialect.AppendNull(b)
}
b = append(b, '\'')
b = append(b, '{')
for _, n := range ints {
b = strconv.AppendInt(b, int64(n), 10)
b = append(b, ',')
}
if len(ints) > 0 {
b[len(b)-1] = '}' // Replace trailing comma.
} else {
b = append(b, '}')
}
b = append(b, '\'')
return b
}
func appendInt64SliceValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
ints := v.Convert(sliceInt64Type).Interface().([]int64)
return appendInt64Slice(b, ints)
}
func appendInt64Slice(b []byte, ints []int64) []byte {
if ints == nil {
return dialect.AppendNull(b)
}
b = append(b, '\'')
b = append(b, '{')
for _, n := range ints {
b = strconv.AppendInt(b, n, 10)
b = append(b, ',')
}
if len(ints) > 0 {
b[len(b)-1] = '}' // Replace trailing comma.
} else {
b = append(b, '}')
}
b = append(b, '\'')
return b
}
func appendFloat64SliceValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
floats := v.Convert(sliceFloat64Type).Interface().([]float64)
return appendFloat64Slice(b, floats)
}
func appendFloat64Slice(b []byte, floats []float64) []byte {
if floats == nil {
return dialect.AppendNull(b)
}
b = append(b, '\'')
b = append(b, '{')
for _, n := range floats {
b = arrayAppendFloat64(b, n)
b = append(b, ',')
}
if len(floats) > 0 {
b[len(b)-1] = '}' // Replace trailing comma.
} else {
b = append(b, '}')
}
b = append(b, '\'')
return b
}
func arrayAppendFloat64(b []byte, num float64) []byte {
switch {
case math.IsNaN(num):
return append(b, "NaN"...)
case math.IsInf(num, 1):
return append(b, "Infinity"...)
case math.IsInf(num, -1):
return append(b, "-Infinity"...)
default:
return strconv.AppendFloat(b, num, 'f', -1, 64)
}
}
func appendTimeSliceValue(gen schema.QueryGen, b []byte, v reflect.Value) []byte {
ts := v.Convert(sliceTimeType).Interface().([]time.Time)
return appendTimeSlice(gen, b, ts)
}
func appendTimeSlice(gen schema.QueryGen, b []byte, ts []time.Time) []byte {
if ts == nil {
return dialect.AppendNull(b)
}
b = append(b, '\'')
b = append(b, '{')
for _, t := range ts {
b = append(b, '"')
b = appendTime(b, t)
b = append(b, '"')
b = append(b, ',')
}
if len(ts) > 0 {
b[len(b)-1] = '}' // Replace trailing comma.
} else {
b = append(b, '}')
}
b = append(b, '\'')
return b
}
//------------------------------------------------------------------------------
func arrayScanner(typ reflect.Type) schema.ScannerFunc {
kind := typ.Kind()
switch kind {
case reflect.Ptr:
if fn := arrayScanner(typ.Elem()); fn != nil {
return schema.PtrScanner(fn)
}
case reflect.Slice, reflect.Array:
// ok:
default:
return nil
}
elemType := typ.Elem()
if kind == reflect.Slice {
switch elemType {
case stringType:
return scanStringSliceValue
case intType:
return scanIntSliceValue
case int64Type:
return scanInt64SliceValue
case float64Type:
return scanFloat64SliceValue
}
}
scanElem := schema.Scanner(elemType)
return func(dest reflect.Value, src any) error {
dest = reflect.Indirect(dest)
if !dest.CanSet() {
return fmt.Errorf("bun: Scan(non-settable %s)", dest.Type())
}
kind := dest.Kind()
if src == nil {
if kind != reflect.Slice || !dest.IsNil() {
dest.Set(reflect.Zero(dest.Type()))
}
return nil
}
if kind == reflect.Slice {
if dest.IsNil() {
dest.Set(reflect.MakeSlice(dest.Type(), 0, 0))
} else if dest.Len() > 0 {
dest.Set(dest.Slice(0, 0))
}
}
if src == nil {
return nil
}
b, err := toBytes(src)
if err != nil {
return err
}
p := newArrayParser(b)
nextValue := internal.MakeSliceNextElemFunc(dest)
for p.Next() {
elem := p.Elem()
elemValue := nextValue()
if err := scanElem(elemValue, elem); err != nil {
return fmt.Errorf("scanElem failed: %w", err)
}
}
return p.Err()
}
}
func scanStringSliceValue(dest reflect.Value, src any) error {
dest = reflect.Indirect(dest)
if !dest.CanSet() {
return fmt.Errorf("bun: Scan(non-settable %s)", dest.Type())
}
slice, err := decodeStringSlice(src)
if err != nil {
return err
}
dest.Set(reflect.ValueOf(slice))
return nil
}
func decodeStringSlice(src any) ([]string, error) {
if src == nil {
return nil, nil
}
b, err := toBytes(src)
if err != nil {
return nil, err
}
slice := make([]string, 0)
p := newArrayParser(b)
for p.Next() {
elem := p.Elem()
slice = append(slice, string(elem))
}
if err := p.Err(); err != nil {
return nil, err
}
return slice, nil
}
func scanIntSliceValue(dest reflect.Value, src any) error {
dest = reflect.Indirect(dest)
if !dest.CanSet() {
return fmt.Errorf("bun: Scan(non-settable %s)", dest.Type())
}
slice, err := decodeIntSlice(src)
if err != nil {
return err
}
dest.Set(reflect.ValueOf(slice))
return nil
}
func decodeIntSlice(src any) ([]int, error) {
if src == nil {
return nil, nil
}
b, err := toBytes(src)
if err != nil {
return nil, err
}
slice := make([]int, 0)
p := newArrayParser(b)
for p.Next() {
elem := p.Elem()
if elem == nil {
slice = append(slice, 0)
continue
}
n, err := strconv.Atoi(internal.String(elem))
if err != nil {
return nil, err
}
slice = append(slice, n)
}
if err := p.Err(); err != nil {
return nil, err
}
return slice, nil
}
func scanInt64SliceValue(dest reflect.Value, src any) error {
dest = reflect.Indirect(dest)
if !dest.CanSet() {
return fmt.Errorf("bun: Scan(non-settable %s)", dest.Type())
}
slice, err := decodeInt64Slice(src)
if err != nil {
return err
}
dest.Set(reflect.ValueOf(slice))
return nil
}
func decodeInt64Slice(src any) ([]int64, error) {
if src == nil {
return nil, nil
}
b, err := toBytes(src)
if err != nil {
return nil, err
}
slice := make([]int64, 0)
p := newArrayParser(b)
for p.Next() {
elem := p.Elem()
if elem == nil {
slice = append(slice, 0)
continue
}
n, err := strconv.ParseInt(internal.String(elem), 10, 64)
if err != nil {
return nil, err
}
slice = append(slice, n)
}
if err := p.Err(); err != nil {
return nil, err
}
return slice, nil
}
func scanFloat64SliceValue(dest reflect.Value, src any) error {
dest = reflect.Indirect(dest)
if !dest.CanSet() {
return fmt.Errorf("bun: Scan(non-settable %s)", dest.Type())
}
slice, err := scanFloat64Slice(src)
if err != nil {
return err
}
dest.Set(reflect.ValueOf(slice))
return nil
}
func scanFloat64Slice(src any) ([]float64, error) {
if src == nil {
return nil, nil
}
b, err := toBytes(src)
if err != nil {
return nil, err
}
slice := make([]float64, 0)
p := newArrayParser(b)
for p.Next() {
elem := p.Elem()
if elem == nil {
slice = append(slice, 0)
continue
}
n, err := strconv.ParseFloat(internal.String(elem), 64)
if err != nil {
return nil, err
}
slice = append(slice, n)
}
if err := p.Err(); err != nil {
return nil, err
}
return slice, nil
}
func toBytes(src any) ([]byte, error) {
switch src := src.(type) {
case string:
return internal.Bytes(src), nil
case []byte:
return src, nil
default:
return nil, fmt.Errorf("pgdialect: got %T, wanted []byte or string", src)
}
}
+119
View File
@@ -0,0 +1,119 @@
package pgdialect
import (
"bytes"
"fmt"
"io"
)
type arrayParser struct {
p pgparser
elem []byte
err error
isJson bool
}
func newArrayParser(b []byte) *arrayParser {
p := new(arrayParser)
if b[0] == 'n' {
p.p.Reset(nil)
return p
}
if len(b) < 2 || (b[0] != '{' && b[0] != '[') || (b[len(b)-1] != '}' && b[len(b)-1] != ']') {
p.err = fmt.Errorf("pgdialect: can't parse array: %q", b)
return p
}
p.isJson = b[0] == '['
p.p.Reset(b[1 : len(b)-1])
return p
}
func (p *arrayParser) Next() bool {
if p.err != nil {
return false
}
p.err = p.readNext()
return p.err == nil
}
func (p *arrayParser) Err() error {
if p.err != io.EOF {
return p.err
}
return nil
}
func (p *arrayParser) Elem() []byte {
return p.elem
}
func (p *arrayParser) readNext() error {
ch := p.p.Read()
if ch == 0 {
return io.EOF
}
switch ch {
case '}', ']':
return io.EOF
case '"':
b, err := p.p.ReadSubstring(ch)
if err != nil {
return err
}
if p.p.Peek() == ',' {
p.p.Advance()
}
p.elem = b
return nil
case '[', '(':
rng, err := p.p.ReadRange(ch)
if err != nil {
return err
}
if p.p.Peek() == ',' {
p.p.Advance()
}
p.elem = rng
return nil
default:
if ch == '{' && p.isJson {
json, err := p.p.ReadJSON()
if err != nil {
return err
}
for {
if p.p.Peek() == ',' || p.p.Peek() == ' ' {
p.p.Advance()
} else {
break
}
}
p.elem = json
return nil
} else {
lit := p.p.ReadLiteral(ch)
if bytes.Equal(lit, []byte("NULL")) {
lit = nil
}
if p.p.Peek() == ',' {
p.p.Advance()
}
p.elem = lit
return nil
}
}
}
+163
View File
@@ -0,0 +1,163 @@
package pgdialect
import (
"database/sql"
"fmt"
"strconv"
"strings"
"github.com/uptrace/bun"
"github.com/uptrace/bun/dialect"
"github.com/uptrace/bun/dialect/feature"
"github.com/uptrace/bun/dialect/sqltype"
"github.com/uptrace/bun/migrate/sqlschema"
"github.com/uptrace/bun/schema"
)
var pgDialect = New()
func init() {
if Version() != bun.Version() {
panic(fmt.Errorf("pgdialect and Bun must have the same version: v%s != v%s",
Version(), bun.Version()))
}
}
type Dialect struct {
schema.BaseDialect
tables *schema.Tables
features feature.Feature
uintAsInt bool
}
var _ schema.Dialect = (*Dialect)(nil)
var _ sqlschema.InspectorDialect = (*Dialect)(nil)
var _ sqlschema.MigratorDialect = (*Dialect)(nil)
func New(opts ...DialectOption) *Dialect {
d := new(Dialect)
d.tables = schema.NewTables(d)
d.features = feature.CTE |
feature.WithValues |
feature.Returning |
feature.InsertReturning |
feature.DefaultPlaceholder |
feature.DoubleColonCast |
feature.InsertTableAlias |
feature.UpdateTableAlias |
feature.DeleteTableAlias |
feature.TableCascade |
feature.TableIdentity |
feature.TableTruncate |
feature.TableNotExists |
feature.InsertOnConflict |
feature.SelectExists |
feature.GeneratedIdentity |
feature.CompositeIn |
feature.FKDefaultOnAction |
feature.DeleteReturning |
feature.Merge |
feature.MergeReturning |
feature.AlterColumnExists |
feature.CreateIndexIfNotExists
for _, opt := range opts {
opt(d)
}
return d
}
type DialectOption func(d *Dialect)
func WithoutFeature(other feature.Feature) DialectOption {
return func(d *Dialect) {
d.features = d.features.Remove(other)
}
}
func WithAppendUintAsInt(on bool) DialectOption {
return func(d *Dialect) {
d.uintAsInt = on
}
}
func (d *Dialect) Init(*sql.DB) {}
func (d *Dialect) Name() dialect.Name {
return dialect.PG
}
func (d *Dialect) Features() feature.Feature {
return d.features
}
func (d *Dialect) Tables() *schema.Tables {
return d.tables
}
func (d *Dialect) OnTable(table *schema.Table) {
for _, field := range table.FieldMap {
d.onField(field)
}
}
func (d *Dialect) onField(field *schema.Field) {
field.DiscoveredSQLType = fieldSQLType(field)
if field.AutoIncrement && !field.Identity {
switch field.DiscoveredSQLType {
case sqltype.SmallInt:
field.CreateTableSQLType = pgTypeSmallSerial
case sqltype.Integer:
field.CreateTableSQLType = pgTypeSerial
case sqltype.BigInt:
field.CreateTableSQLType = pgTypeBigSerial
}
}
if field.Tag.HasOption("array") || strings.HasSuffix(field.UserSQLType, "[]") {
field.Append = d.arrayAppender(field.StructField.Type)
field.Scan = arrayScanner(field.StructField.Type)
return
}
if field.Tag.HasOption("multirange") || strings.HasSuffix(field.UserSQLType, "multirange") {
field.Scan = arrayScanner(field.StructField.Type)
return
}
switch field.DiscoveredSQLType {
case sqltype.HSTORE:
field.Append = d.hstoreAppender(field.StructField.Type)
field.Scan = hstoreScanner(field.StructField.Type)
}
}
func (d *Dialect) IdentQuote() byte {
return '"'
}
func (d *Dialect) AppendUint32(b []byte, n uint32) []byte {
if d.uintAsInt {
return strconv.AppendInt(b, int64(int32(n)), 10)
}
return strconv.AppendUint(b, uint64(n), 10)
}
func (d *Dialect) AppendUint64(b []byte, n uint64) []byte {
if d.uintAsInt {
return strconv.AppendInt(b, int64(n), 10)
}
return strconv.AppendUint(b, n, 10)
}
func (d *Dialect) AppendSequence(b []byte, _ *schema.Table, _ *schema.Field) []byte {
return appendGeneratedAsIdentity(b)
}
// appendGeneratedAsIdentity appends GENERATED BY DEFAULT AS IDENTITY to the column definition.
func appendGeneratedAsIdentity(b []byte) []byte {
return append(b, " GENERATED BY DEFAULT AS IDENTITY"...)
}
+87
View File
@@ -0,0 +1,87 @@
package pgdialect
import (
"database/sql/driver"
"encoding/hex"
"fmt"
"strconv"
"time"
"unicode/utf8"
"github.com/uptrace/bun/dialect"
)
func appendElem(buf []byte, val any) []byte {
switch val := val.(type) {
case int64:
return strconv.AppendInt(buf, val, 10)
case float64:
return arrayAppendFloat64(buf, val)
case bool:
return dialect.AppendBool(buf, val)
case []byte:
return appendBytesElem(buf, val)
case string:
return appendStringElem(buf, val)
case time.Time:
buf = append(buf, '"')
buf = appendTime(buf, val)
buf = append(buf, '"')
return buf
case driver.Valuer:
val2, err := val.Value()
if err != nil {
err := fmt.Errorf("pgdialect: can't append elem value: %w", err)
return dialect.AppendError(buf, err)
}
return appendElem(buf, val2)
default:
err := fmt.Errorf("pgdialect: can't append elem %T", val)
return dialect.AppendError(buf, err)
}
}
func appendBytesElem(b []byte, bs []byte) []byte {
if bs == nil {
return dialect.AppendNull(b)
}
b = append(b, `"\\x`...)
s := len(b)
b = append(b, make([]byte, hex.EncodedLen(len(bs)))...)
hex.Encode(b[s:], bs)
b = append(b, '"')
return b
}
func appendStringElem(b []byte, s string) []byte {
b = append(b, '"')
for _, r := range s {
switch r {
case 0:
// ignore
case '\'':
b = append(b, "''"...)
case '"':
b = append(b, '\\', '"')
case '\\':
b = append(b, '\\', '\\')
default:
if r < utf8.RuneSelf {
b = append(b, byte(r))
break
}
l := len(b)
if cap(b)-l < utf8.UTFMax {
b = append(b, make([]byte, utf8.UTFMax)...)
}
n := utf8.EncodeRune(b[l:l+utf8.UTFMax], r)
b = b[:l+n]
}
}
b = append(b, '"')
return b
}
+73
View File
@@ -0,0 +1,73 @@
package pgdialect
import (
"database/sql"
"fmt"
"reflect"
"github.com/uptrace/bun/schema"
)
type HStoreValue struct {
v reflect.Value
append schema.AppenderFunc
scan schema.ScannerFunc
}
// HStore accepts a map[string]string and returns a wrapper for working with PostgreSQL
// hstore data type.
//
// For struct fields you can use hstore tag:
//
// Attrs map[string]string `bun:",hstore"`
func HStore(vi any) *HStoreValue {
v := reflect.ValueOf(vi)
if !v.IsValid() {
panic(fmt.Errorf("bun: HStore(nil)"))
}
typ := v.Type()
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
if typ.Kind() != reflect.Map {
panic(fmt.Errorf("bun: Hstore(unsupported %s)", typ))
}
return &HStoreValue{
v: v,
append: pgDialect.hstoreAppender(v.Type()),
scan: hstoreScanner(v.Type()),
}
}
var (
_ schema.QueryAppender = (*HStoreValue)(nil)
_ sql.Scanner = (*HStoreValue)(nil)
)
func (h *HStoreValue) AppendQuery(gen schema.QueryGen, b []byte) ([]byte, error) {
if h.append == nil {
panic(fmt.Errorf("bun: HStore(unsupported %s)", h.v.Type()))
}
return h.append(gen, b, h.v), nil
}
func (h *HStoreValue) Scan(src any) error {
if h.scan == nil {
return fmt.Errorf("bun: HStore(unsupported %s)", h.v.Type())
}
if h.v.Kind() != reflect.Ptr {
return fmt.Errorf("bun: HStore(non-pointer %s)", h.v.Type())
}
return h.scan(h.v.Elem(), src)
}
func (h *HStoreValue) Value() any {
if h.v.IsValid() {
return h.v.Interface()
}
return nil
}
+100
View File
@@ -0,0 +1,100 @@
package pgdialect
import (
"bytes"
"fmt"
"io"
)
type hstoreParser struct {
p pgparser
key string
value string
err error
}
func newHStoreParser(b []byte) *hstoreParser {
p := new(hstoreParser)
if len(b) != 0 && (len(b) < 6 || b[0] != '"') {
p.err = fmt.Errorf("pgdialect: can't parse hstore: %q", b)
return p
}
p.p.Reset(b)
return p
}
func (p *hstoreParser) Next() bool {
if p.err != nil {
return false
}
p.err = p.readNext()
return p.err == nil
}
func (p *hstoreParser) Err() error {
if p.err != io.EOF {
return p.err
}
return nil
}
func (p *hstoreParser) Key() string {
return p.key
}
func (p *hstoreParser) Value() string {
return p.value
}
func (p *hstoreParser) readNext() error {
if !p.p.Valid() {
return io.EOF
}
if err := p.p.Skip('"'); err != nil {
return err
}
key, err := p.p.ReadUnescapedSubstring('"')
if err != nil {
return err
}
p.key = string(key)
if err := p.p.SkipPrefix([]byte("=>")); err != nil {
return err
}
ch, err := p.p.ReadByte()
if err != nil {
return err
}
switch ch {
case '"':
value, err := p.p.ReadUnescapedSubstring(ch)
if err != nil {
return err
}
p.skipComma()
p.value = string(value)
return nil
default:
value := p.p.ReadLiteral(ch)
if bytes.Equal(value, []byte("NULL")) {
p.value = ""
}
p.skipComma()
return nil
}
}
func (p *hstoreParser) skipComma() {
if p.p.Peek() == ',' {
p.p.Advance()
}
if p.p.Peek() == ' ' {
p.p.Advance()
}
}
+67
View File
@@ -0,0 +1,67 @@
package pgdialect
import (
"fmt"
"reflect"
"github.com/uptrace/bun/schema"
)
func hstoreScanner(typ reflect.Type) schema.ScannerFunc {
kind := typ.Kind()
switch kind {
case reflect.Ptr:
if fn := hstoreScanner(typ.Elem()); fn != nil {
return schema.PtrScanner(fn)
}
case reflect.Map:
// ok:
default:
return nil
}
if typ.Key() == stringType && typ.Elem() == stringType {
return scanMapStringStringValue
}
return func(dest reflect.Value, src any) error {
return fmt.Errorf("bun: Hstore(unsupported %s)", dest.Type())
}
}
func scanMapStringStringValue(dest reflect.Value, src any) error {
dest = reflect.Indirect(dest)
if !dest.CanSet() {
return fmt.Errorf("bun: Scan(non-settable %s)", dest.Type())
}
m, err := decodeMapStringString(src)
if err != nil {
return err
}
dest.Set(reflect.ValueOf(m))
return nil
}
func decodeMapStringString(src any) (map[string]string, error) {
if src == nil {
return nil, nil
}
b, err := toBytes(src)
if err != nil {
return nil, err
}
m := make(map[string]string)
p := newHStoreParser(b)
for p.Next() {
m[p.Key()] = p.Value()
}
if err := p.Err(); err != nil {
return nil, err
}
return m, nil
}
+300
View File
@@ -0,0 +1,300 @@
package pgdialect
import (
"context"
"strings"
"github.com/uptrace/bun"
"github.com/uptrace/bun/migrate/sqlschema"
)
type (
Schema = sqlschema.BaseDatabase
Table = sqlschema.BaseTable
Column = sqlschema.BaseColumn
)
func (d *Dialect) NewInspector(db *bun.DB, options ...sqlschema.InspectorOption) sqlschema.Inspector {
return newInspector(db, options...)
}
type Inspector struct {
sqlschema.InspectorConfig
db *bun.DB
}
var _ sqlschema.Inspector = (*Inspector)(nil)
func newInspector(db *bun.DB, options ...sqlschema.InspectorOption) *Inspector {
i := &Inspector{db: db}
sqlschema.ApplyInspectorOptions(&i.InspectorConfig, options...)
return i
}
func (in *Inspector) Inspect(ctx context.Context) (sqlschema.Database, error) {
dbSchema := Schema{
ForeignKeys: make(map[sqlschema.ForeignKey]string),
}
exclude := in.ExcludeTables
if len(exclude) == 0 {
// Avoid getting NOT LIKE ALL (ARRAY[NULL]) if bun.List() is called with an empty slice.
exclude = []string{""}
}
var tables []*InformationSchemaTable
if err := in.db.NewRaw(sqlInspectTables, in.SchemaName, bun.List(exclude)).Scan(ctx, &tables); err != nil {
return dbSchema, err
}
var fks []*ForeignKey
if err := in.db.NewRaw(sqlInspectForeignKeys, in.SchemaName, bun.List(exclude), bun.List(exclude)).Scan(ctx, &fks); err != nil {
return dbSchema, err
}
dbSchema.ForeignKeys = make(map[sqlschema.ForeignKey]string, len(fks))
for _, table := range tables {
var columns []*InformationSchemaColumn
if err := in.db.NewRaw(sqlInspectColumnsQuery, table.Schema, table.Name).Scan(ctx, &columns); err != nil {
return dbSchema, err
}
var colDefs []sqlschema.Column
uniqueGroups := make(map[string][]string)
for _, c := range columns {
def := c.Default
if c.IsSerial || c.IsIdentity {
def = ""
} else if !c.IsDefaultLiteral {
def = strings.ToLower(def)
}
colDefs = append(colDefs, &Column{
Name: c.Name,
SQLType: c.DataType,
VarcharLen: c.VarcharLen,
DefaultValue: def,
IsNullable: c.IsNullable,
IsAutoIncrement: c.IsSerial,
IsIdentity: c.IsIdentity,
})
for _, group := range c.UniqueGroups {
uniqueGroups[group] = append(uniqueGroups[group], c.Name)
}
}
var unique []sqlschema.Unique
for name, columns := range uniqueGroups {
unique = append(unique, sqlschema.Unique{
Name: name,
Columns: sqlschema.NewColumns(columns...),
})
}
var pk *sqlschema.PrimaryKey
if len(table.PrimaryKey.Columns) > 0 {
pk = &sqlschema.PrimaryKey{
Name: table.PrimaryKey.ConstraintName,
Columns: sqlschema.NewColumns(table.PrimaryKey.Columns...),
}
}
dbSchema.Tables = append(dbSchema.Tables, &Table{
Schema: table.Schema,
Name: table.Name,
Columns: colDefs,
PrimaryKey: pk,
UniqueConstraints: unique,
})
}
for _, fk := range fks {
dbFK := sqlschema.ForeignKey{
From: sqlschema.NewColumnReference(fk.SourceTable, fk.SourceColumns...),
To: sqlschema.NewColumnReference(fk.TargetTable, fk.TargetColumns...),
}
if _, exclude := in.ExcludeForeignKeys[dbFK]; exclude {
continue
}
dbSchema.ForeignKeys[dbFK] = fk.ConstraintName
}
return dbSchema, nil
}
type InformationSchemaTable struct {
Schema string `bun:"table_schema,pk"`
Name string `bun:"table_name,pk"`
PrimaryKey PrimaryKey `bun:"embed:primary_key_"`
Columns []*InformationSchemaColumn `bun:"rel:has-many,join:table_schema=table_schema,join:table_name=table_name"`
}
type InformationSchemaColumn struct {
Schema string `bun:"table_schema"`
Table string `bun:"table_name"`
Name string `bun:"column_name"`
DataType string `bun:"data_type"`
VarcharLen int `bun:"varchar_len"`
IsArray bool `bun:"is_array"`
ArrayDims int `bun:"array_dims"`
Default string `bun:"default"`
IsDefaultLiteral bool `bun:"default_is_literal_expr"`
IsIdentity bool `bun:"is_identity"`
IndentityType string `bun:"identity_type"`
IsSerial bool `bun:"is_serial"`
IsNullable bool `bun:"is_nullable"`
UniqueGroups []string `bun:"unique_groups,array"`
}
type ForeignKey struct {
ConstraintName string `bun:"constraint_name"`
SourceSchema string `bun:"schema_name"`
SourceTable string `bun:"table_name"`
SourceColumns []string `bun:"columns,array"`
TargetSchema string `bun:"target_schema"`
TargetTable string `bun:"target_table"`
TargetColumns []string `bun:"target_columns,array"`
}
type PrimaryKey struct {
ConstraintName string `bun:"name"`
Columns []string `bun:"columns,array"`
}
const (
// sqlInspectTables retrieves all user-defined tables in the selected schema.
// Pass bun.List([]string{...}) to exclude tables from this inspection or bun.List([]string{''}) to include all results.
sqlInspectTables = `
SELECT
"t".table_schema,
"t".table_name,
pk.name AS primary_key_name,
pk.columns AS primary_key_columns
FROM information_schema.tables "t"
LEFT JOIN (
SELECT i.indrelid, "idx".relname AS "name", ARRAY_AGG("a".attname) AS "columns"
FROM pg_index i
JOIN pg_attribute "a"
ON "a".attrelid = i.indrelid
AND "a".attnum = ANY("i".indkey)
AND i.indisprimary
JOIN pg_class "idx" ON i.indexrelid = "idx".oid
GROUP BY 1, 2
) pk
ON ("t".table_schema || '.' || "t".table_name)::regclass = pk.indrelid
WHERE table_type = 'BASE TABLE'
AND "t".table_schema = ?
AND "t".table_schema NOT LIKE 'pg_%'
AND "table_name" NOT LIKE ALL (ARRAY[?])
ORDER BY "t".table_schema, "t".table_name
`
// sqlInspectColumnsQuery retrieves column definitions for the specified table.
// Unlike sqlInspectTables and sqlInspectSchema, it should be passed to bun.NewRaw
// with additional args for table_schema and table_name.
sqlInspectColumnsQuery = `
SELECT
"c".table_schema,
"c".table_name,
"c".column_name,
"c".data_type,
"c".character_maximum_length::integer AS varchar_len,
"c".data_type = 'ARRAY' AS is_array,
COALESCE("c".array_dims, 0) AS array_dims,
CASE
WHEN "c".column_default ~ '^''.*''::.*$' THEN substring("c".column_default FROM '^''(.*)''::.*$')
ELSE "c".column_default
END AS "default",
"c".column_default ~ '^''.*''::.*$' OR "c".column_default ~ '^[0-9\.]+$' AS default_is_literal_expr,
"c".is_identity = 'YES' AS is_identity,
"c".column_default = format('nextval(''%s_%s_seq''::regclass)', "c".table_name, "c".column_name) AS is_serial,
COALESCE("c".identity_type, '') AS identity_type,
"c".is_nullable = 'YES' AS is_nullable,
"c"."unique_groups" AS unique_groups
FROM (
SELECT
"table_schema",
"table_name",
"column_name",
"c".data_type,
"c".character_maximum_length,
"c".column_default,
"c".is_identity,
"c".is_nullable,
att.array_dims,
att.identity_type,
att."unique_groups",
att."constraint_type"
FROM information_schema.columns "c"
LEFT JOIN (
SELECT
s.nspname AS "table_schema",
"t".relname AS "table_name",
"c".attname AS "column_name",
"c".attndims AS array_dims,
"c".attidentity AS identity_type,
ARRAY_AGG(con.conname) FILTER (WHERE con.contype = 'u') AS "unique_groups",
ARRAY_AGG(con.contype) AS "constraint_type"
FROM (
SELECT
conname,
contype,
connamespace,
conrelid,
conrelid AS attrelid,
UNNEST(conkey) AS attnum
FROM pg_constraint
) con
LEFT JOIN pg_attribute "c" USING (attrelid, attnum)
LEFT JOIN pg_namespace s ON s.oid = con.connamespace
LEFT JOIN pg_class "t" ON "t".oid = con.conrelid
GROUP BY 1, 2, 3, 4, 5
) att USING ("table_schema", "table_name", "column_name")
) "c"
WHERE "table_schema" = ? AND "table_name" = ?
ORDER BY "table_schema", "table_name", "column_name"
`
// sqlInspectForeignKeys get FK definitions for user-defined tables.
// Pass bun.List([]string{...}) to exclude tables from this inspection or bun.List([]string{''}) to include all results.
sqlInspectForeignKeys = `
WITH
"schemas" AS (
SELECT oid, nspname
FROM pg_namespace
),
"tables" AS (
SELECT oid, relnamespace, relname, relkind
FROM pg_class
),
"columns" AS (
SELECT attrelid, attname, attnum
FROM pg_attribute
WHERE attisdropped = false
)
SELECT DISTINCT
co.conname AS "constraint_name",
ss.nspname AS schema_name,
s.relname AS "table_name",
ARRAY_AGG(sc.attname) AS "columns",
ts.nspname AS target_schema,
"t".relname AS target_table,
ARRAY_AGG(tc.attname) AS target_columns
FROM pg_constraint co
LEFT JOIN "tables" s ON s.oid = co.conrelid
LEFT JOIN "schemas" ss ON ss.oid = s.relnamespace
LEFT JOIN "columns" sc ON sc.attrelid = s.oid AND sc.attnum = ANY(co.conkey)
LEFT JOIN "tables" t ON t.oid = co.confrelid
LEFT JOIN "schemas" ts ON ts.oid = "t".relnamespace
LEFT JOIN "columns" tc ON tc.attrelid = "t".oid AND tc.attnum = ANY(co.confkey)
WHERE co.contype = 'f'
AND co.conrelid IN (SELECT oid FROM pg_class WHERE relkind = 'r')
AND ARRAY_POSITION(co.conkey, sc.attnum) = ARRAY_POSITION(co.confkey, tc.attnum)
AND ss.nspname = ?
AND s.relname NOT LIKE ALL (ARRAY[?])
AND "t".relname NOT LIKE ALL (ARRAY[?])
GROUP BY "constraint_name", "schema_name", "table_name", target_schema, target_table
`
)
+143
View File
@@ -0,0 +1,143 @@
package pgdialect
import (
"bytes"
"encoding/hex"
"github.com/uptrace/bun/internal/parser"
)
type pgparser struct {
parser.Parser
buf []byte
}
func newParser(b []byte) *pgparser {
p := new(pgparser)
p.Reset(b)
return p
}
func (p *pgparser) ReadLiteral(ch byte) []byte {
p.Unread()
lit, _ := p.ReadSep(',')
return lit
}
func (p *pgparser) ReadUnescapedSubstring(ch byte) ([]byte, error) {
return p.readSubstring(ch, false)
}
func (p *pgparser) ReadSubstring(ch byte) ([]byte, error) {
return p.readSubstring(ch, true)
}
func (p *pgparser) readSubstring(ch byte, escaped bool) ([]byte, error) {
ch, err := p.ReadByte()
if err != nil {
return nil, err
}
p.buf = p.buf[:0]
for {
if ch == '"' {
break
}
next, err := p.ReadByte()
if err != nil {
return nil, err
}
if ch == '\\' {
switch next {
case '\\', '"':
p.buf = append(p.buf, next)
ch, err = p.ReadByte()
if err != nil {
return nil, err
}
default:
p.buf = append(p.buf, '\\')
ch = next
}
continue
}
if escaped && ch == '\'' && next == '\'' {
p.buf = append(p.buf, next)
ch, err = p.ReadByte()
if err != nil {
return nil, err
}
continue
}
p.buf = append(p.buf, ch)
ch = next
}
if bytes.HasPrefix(p.buf, []byte("\\x")) && len(p.buf)%2 == 0 {
data := p.buf[2:]
buf := make([]byte, hex.DecodedLen(len(data)))
n, err := hex.Decode(buf, data)
if err != nil {
return nil, err
}
return buf[:n], nil
}
return p.buf, nil
}
func (p *pgparser) ReadRange(ch byte) ([]byte, error) {
p.buf = p.buf[:0]
p.buf = append(p.buf, ch)
for p.Valid() {
ch = p.Read()
p.buf = append(p.buf, ch)
if ch == ']' || ch == ')' {
break
}
}
return p.buf, nil
}
func (p *pgparser) ReadJSON() ([]byte, error) {
p.Unread()
c, err := p.ReadByte()
if err != nil {
return nil, err
}
p.buf = p.buf[:0]
depth := 0
for {
switch c {
case '{':
depth++
case '}':
depth--
}
p.buf = append(p.buf, c)
if depth == 0 {
break
}
next, err := p.ReadByte()
if err != nil {
return nil, err
}
c = next
}
return p.buf, nil
}
+217
View File
@@ -0,0 +1,217 @@
package pgdialect
import (
"bytes"
"database/sql"
"fmt"
"time"
"github.com/uptrace/bun/internal"
"github.com/uptrace/bun/schema"
)
type Range[T any] struct {
Lower, Upper T
LowerBound, UpperBound RangeBound
}
type MultiRange[T any] []Range[T]
type RangeBound byte
const (
// RangeBoundUnset indicates that no bound is set.
// This usually means the range is uninitialized or unspecified.
RangeBoundUnset RangeBound = 0x0
// RangeBoundEmpty is a special marker for an empty range.
// This is NOT a valid PostgreSQL bound character, but is used internally
// to represent a range that contains no values.
RangeBoundEmpty RangeBound = 'E'
RangeBoundInclusiveLeft RangeBound = '['
RangeBoundInclusiveRight RangeBound = ']'
RangeBoundExclusiveLeft RangeBound = '('
RangeBoundExclusiveRight RangeBound = ')'
)
type RangeOption[T any] func(*Range[T])
func NewRange[T any](lower, upper T) Range[T] {
r := Range[T]{
Lower: lower,
Upper: upper,
LowerBound: RangeBoundInclusiveLeft,
UpperBound: RangeBoundExclusiveRight,
}
return r
}
func NewEmptyRange[T any]() Range[T] {
return Range[T]{LowerBound: RangeBoundEmpty, UpperBound: RangeBoundEmpty}
}
func (r *Range[T]) IsZero() bool {
// NOTE: r.LowerBound represent
return r == nil || r.LowerBound == 0
}
func (r Range[T]) IsEmpty() bool {
return r.LowerBound == RangeBoundEmpty
}
var _ sql.Scanner = (*Range[any])(nil)
func (r *Range[T]) Scan(raw any) (err error) {
var src []byte
switch v := raw.(type) {
case []byte:
src = v
case string:
src = []byte(v)
case nil:
return nil
default:
return fmt.Errorf("pgdialect: Range can't scan %T", raw)
}
src = bytes.TrimSpace(src)
if len(src) == 0 {
return nil
}
if string(src) == "empty" {
r.LowerBound, r.UpperBound = RangeBoundEmpty, RangeBoundEmpty
return nil
}
switch src[0] {
case byte(RangeBoundInclusiveLeft), byte(RangeBoundExclusiveLeft):
r.LowerBound = RangeBound(src[0])
default:
return fmt.Errorf("unexpected lower bound: %s", string(src[:1]))
}
switch src[len(src)-1] {
case byte(RangeBoundInclusiveRight), byte(RangeBoundExclusiveRight):
r.UpperBound = RangeBound(src[len(src)-1])
default:
return fmt.Errorf("unexpected upper bound: %s", string(src[len(src)-1:]))
}
src = src[1 : len(src)-1]
ind := bytes.IndexByte(src, ',')
if ind == -1 {
return fmt.Errorf("invalid range: wanted comma, got %s", string(src))
}
left, right := src[:ind], src[ind+1:]
if len(left) > 0 {
_, err := scanElem(&r.Lower, left)
if err != nil {
return err
}
} else {
r.LowerBound = RangeBoundUnset
}
if len(right) > 0 {
_, err = scanElem(&r.Upper, right)
if err != nil {
return err
}
} else {
r.UpperBound = RangeBoundUnset
}
return nil
}
var _ schema.QueryAppender = (*Range[any])(nil)
func (r Range[T]) AppendQuery(_ schema.QueryGen, buf []byte) ([]byte, error) {
buf = append(buf, '\'')
buf = appendRange(buf, r)
buf = append(buf, '\'')
return buf, nil
}
func appendRange[T any](buf []byte, r Range[T]) []byte {
if r.IsEmpty() {
buf = append(buf, []byte("empty")...)
return buf
}
if r.LowerBound == RangeBoundUnset {
// NOTE from pg's document:
// > Specifying a missing bound as inclusive is automatically converted to exclusive, e.g., [,] is converted to (,).
buf = append(buf, byte(RangeBoundExclusiveLeft))
} else {
buf = append(buf, byte(r.LowerBound))
buf = appendElem(buf, r.Lower)
}
buf = append(buf, ',')
if r.UpperBound == RangeBoundUnset {
buf = append(buf, byte(RangeBoundExclusiveRight))
} else {
buf = appendElem(buf, r.Upper)
buf = append(buf, byte(r.UpperBound))
}
return buf
}
func (m *MultiRange[T]) Len() int {
if m == nil {
return 0
}
return len(([]Range[T])(*m))
}
func (m *MultiRange[T]) IsZero() bool {
return m.Len() == 0
}
func (m MultiRange[T]) AppendQuery(_ schema.QueryGen, buf []byte) ([]byte, error) {
if m == nil {
return append(buf, []byte("'{}'")...), nil
}
rs := ([]Range[T])(m)
buf = append(buf, '\'', '{')
for _, r := range rs {
buf = appendRange(buf, r)
buf = append(buf, ',')
}
if len(rs) > 0 {
buf[len(buf)-1] = '}'
} else {
buf = append(buf, '}')
}
buf = append(buf, '\'')
return buf, nil
}
func scanElem(ptr any, src []byte) ([]byte, error) {
// NOTE: for daterange, pg return 2024-12-01, for tzrange, pg return "2024-12-01 12:00:00"
if len(src) >= 2 && src[0] == '"' {
src = src[1 : len(src)-1]
}
switch ptr := ptr.(type) {
case *time.Time:
tm, err := internal.ParseTime(internal.String(src))
if err != nil {
return nil, err
}
*ptr = tm
return src, nil
case sql.Scanner:
if err := ptr.Scan(src); err != nil {
return nil, err
}
return src, nil
default:
panic(fmt.Errorf("unsupported range type: %T", ptr))
}
}
+190
View File
@@ -0,0 +1,190 @@
package pgdialect
import (
"database/sql"
"encoding/json"
"net"
"reflect"
"strings"
"github.com/uptrace/bun/dialect/sqltype"
"github.com/uptrace/bun/migrate/sqlschema"
"github.com/uptrace/bun/schema"
)
const (
// Date / Time
pgTypeTimestamp = "TIMESTAMP" // Timestamp
pgTypeTimestampWithTz = "TIMESTAMP WITH TIME ZONE" // Timestamp with a time zone
pgTypeTimestampTz = "TIMESTAMPTZ" // Timestamp with a time zone (alias)
pgTypeDate = "DATE" // Date
pgTypeTime = "TIME" // Time without a time zone
pgTypeTimeTz = "TIME WITH TIME ZONE" // Time with a time zone
pgTypeInterval = "INTERVAL" // Time interval
// Network Addresses
pgTypeInet = "INET" // IPv4 or IPv6 hosts and networks
pgTypeCidr = "CIDR" // IPv4 or IPv6 networks
pgTypeMacaddr = "MACADDR" // MAC addresses
// Serial Types
pgTypeSmallSerial = "SMALLSERIAL" // 2 byte autoincrementing integer
pgTypeSerial = "SERIAL" // 4 byte autoincrementing integer
pgTypeBigSerial = "BIGSERIAL" // 8 byte autoincrementing integer
// Character Types
pgTypeChar = "CHAR" // fixed length string (blank padded)
pgTypeCharacter = "CHARACTER" // alias for CHAR
pgTypeText = "TEXT" // variable length string without limit
pgTypeVarchar = "VARCHAR" // variable length string with optional limit
pgTypeCharacterVarying = "CHARACTER VARYING" // alias for VARCHAR
// Binary Data Types
pgTypeBytea = "BYTEA" // binary string
)
var (
ipType = reflect.TypeFor[net.IP]()
ipNetType = reflect.TypeFor[net.IPNet]()
jsonRawMessageType = reflect.TypeFor[json.RawMessage]()
nullStringType = reflect.TypeFor[sql.NullString]()
)
func (d *Dialect) DefaultVarcharLen() int {
return 0
}
func (d *Dialect) DefaultSchema() string {
return "public"
}
func fieldSQLType(field *schema.Field) string {
if field.UserSQLType != "" {
return field.UserSQLType
}
if v, ok := field.Tag.Option("composite"); ok {
return v
}
if field.Tag.HasOption("hstore") {
return sqltype.HSTORE
}
if field.Tag.HasOption("array") {
switch field.IndirectType.Kind() {
case reflect.Slice, reflect.Array:
sqlType := sqlType(field.IndirectType.Elem())
return sqlType + "[]"
}
}
if field.DiscoveredSQLType == sqltype.Blob {
return pgTypeBytea
}
return sqlType(field.IndirectType)
}
func sqlType(typ reflect.Type) string {
if typ.Kind() == reflect.Ptr {
typ = typ.Elem()
}
switch typ {
case nullStringType: // typ.Kind() == reflect.Struct, test for exact match
return sqltype.VarChar
case ipType:
return pgTypeInet
case ipNetType:
return pgTypeCidr
case jsonRawMessageType:
return sqltype.JSONB
}
sqlType := schema.DiscoverSQLType(typ)
switch sqlType {
case sqltype.Timestamp:
sqlType = pgTypeTimestampTz
}
switch typ.Kind() {
case reflect.Map, reflect.Struct: // except typ == nullStringType, see above
if sqlType == sqltype.VarChar {
return sqltype.JSONB
}
return sqlType
case reflect.Array, reflect.Slice:
if typ.Elem().Kind() == reflect.Uint8 {
return pgTypeBytea
}
return sqltype.JSONB
}
return sqlType
}
var (
char = newAliases(pgTypeChar, pgTypeCharacter)
varchar = newAliases(pgTypeVarchar, pgTypeCharacterVarying)
timestampTz = newAliases(sqltype.Timestamp, pgTypeTimestampTz, pgTypeTimestampWithTz)
bigint = newAliases(sqltype.BigInt, pgTypeBigSerial)
integer = newAliases(sqltype.Integer, pgTypeSerial)
smallint = newAliases(sqltype.SmallInt, pgTypeSmallSerial)
)
func (d *Dialect) CompareType(col1, col2 sqlschema.Column) bool {
typ1, typ2 := strings.ToUpper(col1.GetSQLType()), strings.ToUpper(col2.GetSQLType())
if typ1 == typ2 {
return checkVarcharLen(col1, col2, d.DefaultVarcharLen())
}
switch {
case char.IsAlias(typ1) && char.IsAlias(typ2):
return checkVarcharLen(col1, col2, d.DefaultVarcharLen())
case varchar.IsAlias(typ1) && varchar.IsAlias(typ2):
return checkVarcharLen(col1, col2, d.DefaultVarcharLen())
case timestampTz.IsAlias(typ1) && timestampTz.IsAlias(typ2):
return true
case bigint.IsAlias(typ1) && bigint.IsAlias(typ2),
integer.IsAlias(typ1) && integer.IsAlias(typ2),
smallint.IsAlias(typ1) && smallint.IsAlias(typ2):
return true
}
return false
}
// checkVarcharLen returns true if columns have the same VarcharLen, or,
// if one specifies no VarcharLen and the other one has the default lenght for pgdialect.
// We assume that the types are otherwise equivalent and that any non-character column
// would have VarcharLen == 0;
func checkVarcharLen(col1, col2 sqlschema.Column, defaultLen int) bool {
vl1, vl2 := col1.GetVarcharLen(), col2.GetVarcharLen()
if vl1 == vl2 {
return true
}
if (vl1 == 0 && vl2 == defaultLen) || (vl1 == defaultLen && vl2 == 0) {
return true
}
return false
}
// typeAlias defines aliases for common data types. It is a lightweight string set implementation.
type typeAlias map[string]struct{}
// IsAlias checks if typ1 and typ2 are aliases of the same data type.
func (t typeAlias) IsAlias(typ string) bool {
_, ok := t[typ]
return ok
}
// newAliases creates a set of aliases.
func newAliases(aliases ...string) typeAlias {
types := make(typeAlias)
for _, a := range aliases {
types[a] = struct{}{}
}
return types
}
+6
View File
@@ -0,0 +1,6 @@
package pgdialect
// Version is the current release version.
func Version() string {
return "1.2.18"
}
+125
View File
@@ -0,0 +1,125 @@
package ordered
// Pair represents a key-value pair in the ordered map.
type Pair[K comparable, V any] struct {
Key K
Value V
next, prev *Pair[K, V] // Pointers to the next and previous pairs in the linked list.
}
// Map represents an ordered map.
type Map[K comparable, V any] struct {
root *Pair[K, V] // Sentinel node for the circular doubly linked list.
zero V // Zero value for the value type.
pairs map[K]*Pair[K, V] // Map from keys to pairs.
}
// NewMap creates a new ordered map with optional initial data.
func NewMap[K comparable, V any](initialData ...Pair[K, V]) *Map[K, V] {
m := &Map[K, V]{}
m.Clear()
for _, pair := range initialData {
m.Store(pair.Key, pair.Value)
}
return m
}
// Clear removes all pairs from the map.
func (m *Map[K, V]) Clear() {
if m.root != nil {
m.root.next, m.root.prev = nil, nil // avoid memory leaks
}
for _, pair := range m.pairs {
pair.next, pair.prev = nil, nil // avoid memory leaks
}
m.root = &Pair[K, V]{}
m.root.next, m.root.prev = m.root, m.root
m.pairs = make(map[K]*Pair[K, V])
}
// Len returns the number of pairs in the map.
func (m *Map[K, V]) Len() int {
return len(m.pairs)
}
// Load returns the value associated with the key, and a boolean indicating if the key was found.
func (m *Map[K, V]) Load(key K) (V, bool) {
if pair, present := m.pairs[key]; present {
return pair.Value, true
}
return m.zero, false
}
// Value returns the value associated with the key, or the zero value if the key is not found.
func (m *Map[K, V]) Value(key K) V {
if pair, present := m.pairs[key]; present {
return pair.Value
}
return m.zero
}
// Store adds or updates a key-value pair in the map.
func (m *Map[K, V]) Store(key K, value V) {
if pair, present := m.pairs[key]; present {
pair.Value = value
return
}
pair := &Pair[K, V]{Key: key, Value: value}
pair.prev = m.root.prev
m.root.prev.next = pair
m.root.prev = pair
pair.next = m.root
m.pairs[key] = pair
}
// Delete removes a key-value pair from the map.
func (m *Map[K, V]) Delete(key K) {
if pair, present := m.pairs[key]; present {
pair.prev.next = pair.next
pair.next.prev = pair.prev
pair.next, pair.prev = nil, nil // avoid memory leaks
delete(m.pairs, key)
}
}
// Range calls the given function for each key-value pair in the map in order.
func (m *Map[K, V]) Range(yield func(key K, value V) bool) {
for pair := m.root.next; pair != m.root; pair = pair.next {
if !yield(pair.Key, pair.Value) {
break
}
}
}
// Keys returns a slice of all keys in the map in order.
func (m *Map[K, V]) Keys() []K {
keys := make([]K, 0, len(m.pairs))
m.Range(func(key K, _ V) bool {
keys = append(keys, key)
return true
})
return keys
}
// Values returns a slice of all values in the map in order.
func (m *Map[K, V]) Values() []V {
values := make([]V, 0, len(m.pairs))
m.Range(func(_ K, value V) bool {
values = append(values, value)
return true
})
return values
}
// Pairs returns a slice of all key-value pairs in the map in order.
func (m *Map[K, V]) Pairs() []Pair[K, V] {
pairs := make([]Pair[K, V], 0, len(m.pairs))
m.Range(func(key K, value V) bool {
pairs = append(pairs, Pair[K, V]{Key: key, Value: value})
return true
})
return pairs
}
+475
View File
@@ -0,0 +1,475 @@
package migrate
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"github.com/uptrace/bun"
"github.com/uptrace/bun/internal"
"github.com/uptrace/bun/migrate/sqlschema"
"github.com/uptrace/bun/schema"
)
// AutoMigratorOption configures an AutoMigrator.
type AutoMigratorOption func(m *AutoMigrator)
// WithModel adds a bun.Model to the migration scope.
func WithModel(models ...any) AutoMigratorOption {
return func(m *AutoMigrator) {
m.includeModels = append(m.includeModels, models...)
}
}
// WithExcludeTable tells AutoMigrator to exclude database tables from the migration scope.
// This prevents AutoMigrator from dropping tables which may exist in the schema
// but which are not used by the application.
//
// Expressions may make use of the wildcards supported by the SQL LIKE operator:
// - % as a wildcard
// - _ as a single character
//
// Do not exclude tables included via WithModel, as BunModelInspector ignores this setting.
func WithExcludeTable(tables ...string) AutoMigratorOption {
return func(m *AutoMigrator) {
m.excludeTables = append(m.excludeTables, tables...)
}
}
// WithExcludeForeignKeys tells AutoMigrator to exclude a foreign key constaint
// from the migration scope. This prevents AutoMigrator from dropping foreign keys
// that are defined manually via CreateTableQuery.ForeignKey().
func WithExcludeForeignKeys(fks ...sqlschema.ForeignKey) AutoMigratorOption {
return func(m *AutoMigrator) {
m.excludeForeignKeys = append(m.excludeForeignKeys, fks...)
}
}
// WithSchemaName sets the database schema to migrate objects in.
// By default, dialects' default schema is used.
func WithSchemaName(schemaName string) AutoMigratorOption {
return func(m *AutoMigrator) {
m.schemaName = schemaName
}
}
// WithTableNameAuto overrides default migrations table name.
func WithTableNameAuto(table string) AutoMigratorOption {
return func(m *AutoMigrator) {
m.table = table
m.migratorOpts = append(m.migratorOpts, WithTableName(table))
}
}
// WithLocksTableNameAuto overrides default migration locks table name.
func WithLocksTableNameAuto(table string) AutoMigratorOption {
return func(m *AutoMigrator) {
m.locksTable = table
m.migratorOpts = append(m.migratorOpts, WithLocksTableName(table))
}
}
// WithMarkAppliedOnSuccessAuto sets the migrator to only mark migrations as applied/unapplied
// when their up/down is successful.
func WithMarkAppliedOnSuccessAuto(enabled bool) AutoMigratorOption {
return func(m *AutoMigrator) {
m.migratorOpts = append(m.migratorOpts, WithMarkAppliedOnSuccess(enabled))
}
}
// WithMigrationsDirectoryAuto overrides the default directory for migration files.
func WithMigrationsDirectoryAuto(directory string) AutoMigratorOption {
return func(m *AutoMigrator) {
m.migrationsOpts = append(m.migrationsOpts, WithMigrationsDirectory(directory))
}
}
// AutoMigrator performs automated schema migrations.
//
// It is designed to be a drop-in replacement for some Migrator functionality and supports all existing
// configuration options.
// Similarly to Migrator, it has methods to create SQL migrations, write them to a file, and apply them.
// Unlike Migrator, it detects the differences between the state defined by bun models and the current
// database schema automatically.
//
// Usage:
// 1. Generate migrations and apply them at once with AutoMigrator.Migrate().
// 2. Create up- and down-SQL migration files and apply migrations using Migrator.Migrate().
//
// While both methods produce complete, reversible migrations (with entries in the database
// and SQL migration files), prefer creating migrations and applying them separately for
// any non-trivial cases to ensure AutoMigrator detects expected changes correctly.
//
// Limitations:
// - AutoMigrator only supports a subset of the possible ALTER TABLE modifications.
// - Some changes are not automatically reversible. For example, you would need to manually
// add a CREATE TABLE query to the .down migration file to revert a DROP TABLE migration.
// - Does not validate most dialect-specific constraints. For example, when changing column
// data type, make sure the data con be auto-casted to the new type.
// - Due to how the schema-state diff is calculated, it is not possible to rename a table and
// modify any of its columns' _data type_ in a single run. This will cause the AutoMigrator
// to drop and re-create the table under a different name; it is better to apply this change in 2 steps.
// Renaming a table and renaming its columns at the same time is possible.
// - Renaming table/column to an existing name, i.e. like this [A->B] [B->C], is not possible due to how
// AutoMigrator distinguishes "rename" and "unchanged" columns.
//
// Dialect must implement both sqlschema.Inspector and sqlschema.Migrator to be used with AutoMigrator.
type AutoMigrator struct {
db *bun.DB
// dbInspector creates the current state for the target database.
dbInspector sqlschema.Inspector
// modelInspector creates the desired state based on the model definitions.
modelInspector sqlschema.Inspector
// dbMigrator executes ALTER TABLE queries.
dbMigrator sqlschema.Migrator
table string // Migrations table (excluded from database inspection)
locksTable string // Migration locks table (excluded from database inspection)
// schemaName is the database schema considered for migration.
schemaName string
// includeModels define the migration scope.
includeModels []any
excludeTables []string // excludeTables are excluded from database inspection.
excludeForeignKeys []sqlschema.ForeignKey // excludeForeignKeys are excluded from database inspection.
// diffOpts are passed to detector constructor.
diffOpts []diffOption
// migratorOpts are passed to Migrator constructor.
migratorOpts []MigratorOption
// migrationsOpts are passed to Migrations constructor.
migrationsOpts []MigrationsOption
}
// NewAutoMigrator creates an AutoMigrator that detects schema differences and generates migrations.
func NewAutoMigrator(db *bun.DB, opts ...AutoMigratorOption) (*AutoMigrator, error) {
am := &AutoMigrator{
db: db,
table: defaultTable,
locksTable: defaultLocksTable,
schemaName: db.Dialect().DefaultSchema(),
}
for _, opt := range opts {
opt(am)
}
am.excludeTables = append(am.excludeTables, am.table, am.locksTable)
dbInspector, err := sqlschema.NewInspector(db,
sqlschema.WithSchemaName(am.schemaName),
sqlschema.WithExcludeTables(am.excludeTables...),
sqlschema.WithExcludeForeignKeys(am.excludeForeignKeys...),
)
if err != nil {
return nil, err
}
am.dbInspector = dbInspector
am.diffOpts = append(am.diffOpts, withCompareTypeFunc(db.Dialect().(sqlschema.InspectorDialect).CompareType))
dbMigrator, err := sqlschema.NewMigrator(db, am.schemaName)
if err != nil {
return nil, err
}
am.dbMigrator = dbMigrator
tables := schema.NewTables(db.Dialect())
tables.Register(am.includeModels...)
am.modelInspector = sqlschema.NewBunModelInspector(tables, sqlschema.WithSchemaName(am.schemaName))
return am, nil
}
func (am *AutoMigrator) plan(ctx context.Context) (*changeset, error) {
var err error
got, err := am.dbInspector.Inspect(ctx)
if err != nil {
return nil, err
}
want, err := am.modelInspector.Inspect(ctx)
if err != nil {
return nil, err
}
changes := diff(got, want, am.diffOpts...)
if err := changes.ResolveDependencies(); err != nil {
return nil, fmt.Errorf("plan migrations: %w", err)
}
return changes, nil
}
// Migrate writes required changes to a new migration file and runs the migration.
// This will create an entry in the migrations table, making it possible to revert
// the changes with Migrator.Rollback(). MigrationOptions are passed on to Migrator.Migrate().
func (am *AutoMigrator) Migrate(ctx context.Context, opts ...MigrationOption) (*MigrationGroup, error) {
migrations, _, err := am.createSQLMigrations(ctx, false)
if err != nil {
if err == errNothingToMigrate {
return new(MigrationGroup), nil
}
return nil, fmt.Errorf("auto migrate: %w", err)
}
migrator := NewMigrator(am.db, migrations, am.migratorOpts...)
if err := migrator.Init(ctx); err != nil {
return nil, fmt.Errorf("auto migrate: %w", err)
}
group, err := migrator.Migrate(ctx, opts...)
if err != nil {
return nil, fmt.Errorf("auto migrate: %w", err)
}
return group, nil
}
// CreateSQLMigration writes required changes to a new migration file.
// Use migrate.Migrator to apply the generated migrations.
func (am *AutoMigrator) CreateSQLMigrations(ctx context.Context) ([]*MigrationFile, error) {
_, files, err := am.createSQLMigrations(ctx, false)
if err == errNothingToMigrate {
return files, nil
}
return files, err
}
// CreateTxSQLMigration writes required changes to a new migration file making sure they will be executed
// in a transaction when applied. Use migrate.Migrator to apply the generated migrations.
func (am *AutoMigrator) CreateTxSQLMigrations(ctx context.Context) ([]*MigrationFile, error) {
_, files, err := am.createSQLMigrations(ctx, true)
if err == errNothingToMigrate {
return files, nil
}
return files, err
}
// errNothingToMigrate is a sentinel error which means the database is already in a desired state.
// Should not be returned to the user -- return a nil-error instead.
var errNothingToMigrate = errors.New("nothing to migrate")
func (am *AutoMigrator) createSQLMigrations(ctx context.Context, transactional bool) (*Migrations, []*MigrationFile, error) {
changes, err := am.plan(ctx)
if err != nil {
return nil, nil, fmt.Errorf("create sql migrations: %w", err)
}
if changes.Len() == 0 {
return nil, nil, errNothingToMigrate
}
name, _ := genMigrationName(am.schemaName + "_auto")
migrations := NewMigrations(am.migrationsOpts...)
migrations.Add(Migration{
Name: name,
Up: wrapGoMigrationFunc(changes.Up(am.dbMigrator)),
Down: wrapGoMigrationFunc(changes.Down(am.dbMigrator)),
Comment: "Changes detected by bun.AutoMigrator",
})
// Append .tx.up.sql or .up.sql to migration name, depending if it should be transactional.
fname := func(direction string) string {
return name + map[bool]string{true: ".tx.", false: "."}[transactional] + direction + ".sql"
}
up, err := am.createSQL(ctx, migrations, fname("up"), changes, transactional)
if err != nil {
return nil, nil, fmt.Errorf("create sql migration up: %w", err)
}
down, err := am.createSQL(ctx, migrations, fname("down"), changes.GetReverse(), transactional)
if err != nil {
return nil, nil, fmt.Errorf("create sql migration down: %w", err)
}
return migrations, []*MigrationFile{up, down}, nil
}
func (am *AutoMigrator) createSQL(_ context.Context, migrations *Migrations, fname string, changes *changeset, transactional bool) (*MigrationFile, error) {
var buf bytes.Buffer
if transactional {
buf.WriteString("SET statement_timeout = 0;")
}
if err := changes.WriteTo(&buf, am.dbMigrator); err != nil {
return nil, err
}
content := buf.Bytes()
fpath := filepath.Join(migrations.getDirectory(), fname)
if err := os.WriteFile(fpath, content, 0o644); err != nil {
return nil, err
}
mf := &MigrationFile{
Name: fname,
Path: fpath,
Content: string(content),
}
return mf, nil
}
func (c *changeset) Len() int {
return len(c.operations)
}
// Func creates a MigrationFunc that applies all operations all the changeset.
func (c *changeset) Func(m sqlschema.Migrator) MigrationFunc {
return func(ctx context.Context, db *bun.DB) error {
return c.apply(ctx, db, m)
}
}
// GetReverse returns a new changeset with each operation in it "reversed" and in reverse order.
func (c *changeset) GetReverse() *changeset {
var reverse changeset
for i := len(c.operations) - 1; i >= 0; i-- {
reverse.Add(c.operations[i].GetReverse())
}
return &reverse
}
// Up is syntactic sugar.
func (c *changeset) Up(m sqlschema.Migrator) MigrationFunc {
return c.Func(m)
}
// Down is syntactic sugar.
func (c *changeset) Down(m sqlschema.Migrator) MigrationFunc {
return c.GetReverse().Func(m)
}
// apply generates SQL for each operation and executes it.
func (c *changeset) apply(ctx context.Context, db *bun.DB, m sqlschema.Migrator) error {
if len(c.operations) == 0 {
return nil
}
for _, op := range c.operations {
if _, skip := op.(*Unimplemented); skip {
continue
}
b := internal.MakeQueryBytes()
b, err := m.AppendSQL(b, op)
if err != nil {
return fmt.Errorf("apply changes: %w", err)
}
query := internal.String(b)
if _, err = db.ExecContext(ctx, query); err != nil {
return fmt.Errorf("apply changes: %w", err)
}
}
return nil
}
func (c *changeset) WriteTo(w io.Writer, m sqlschema.Migrator) error {
var err error
b := internal.MakeQueryBytes()
for _, op := range c.operations {
if comment, isComment := op.(*Unimplemented); isComment {
b = append(b, "/*\n"...)
b = append(b, *comment...)
b = append(b, "\n*/"...)
continue
}
// Append each query separately, merge later.
// Dialects assume that the []byte only holds
// the contents of a single query and may be misled.
queryBytes := internal.MakeQueryBytes()
queryBytes, err = m.AppendSQL(queryBytes, op)
if err != nil {
return fmt.Errorf("write changeset: %w", err)
}
b = append(b, queryBytes...)
b = append(b, ";\n"...)
}
if _, err := w.Write(b); err != nil {
return fmt.Errorf("write changeset: %w", err)
}
return nil
}
func (c *changeset) ResolveDependencies() error {
if len(c.operations) <= 1 {
return nil
}
const (
unvisited = iota
current
visited
)
status := make(map[Operation]int, len(c.operations))
for _, op := range c.operations {
status[op] = unvisited
}
var resolved []Operation
var nextOp Operation
var visit func(op Operation) error
next := func() bool {
for op, s := range status {
if s == unvisited {
nextOp = op
return true
}
}
return false
}
// visit iterates over c.operations until it finds all operations that depend on the current one
// or runs into circular dependency, in which case it will return an error.
visit = func(op Operation) error {
switch status[op] {
case visited:
return nil
case current:
// TODO: add details (circle) to the error message
return errors.New("detected circular dependency")
}
status[op] = current
for _, another := range c.operations {
if dop, hasDeps := another.(interface {
DependsOn(Operation) bool
}); another == op || !hasDeps || !dop.DependsOn(op) {
continue
}
if err := visit(another); err != nil {
return err
}
}
status[op] = visited
// Any dependent nodes would've already been added to the list by now, so we prepend.
resolved = append([]Operation{op}, resolved...)
return nil
}
for next() {
if err := visit(nextOp); err != nil {
return err
}
}
c.operations = resolved
return nil
}
+426
View File
@@ -0,0 +1,426 @@
package migrate
import (
"github.com/uptrace/bun/internal/ordered"
"github.com/uptrace/bun/migrate/sqlschema"
)
// changeset is a set of changes to the database schema definition.
type changeset struct {
operations []Operation
}
// Add new operations to the changeset.
func (c *changeset) Add(op ...Operation) {
c.operations = append(c.operations, op...)
}
// diff calculates the diff between the current database schema and the target state.
// The changeset is not sorted -- the caller should resolve dependencies before applying the changes.
func diff(got, want sqlschema.Database, opts ...diffOption) *changeset {
d := newDetector(got, want, opts...)
return d.detectChanges()
}
func (d *detector) detectChanges() *changeset {
currentTables := toOrderedMap(d.current.GetTables())
targetTables := toOrderedMap(d.target.GetTables())
RenameCreate:
for _, wantPair := range targetTables.Pairs() {
wantName, wantTable := wantPair.Key, wantPair.Value
// A table with this name exists in the database. We assume that schema objects won't
// be renamed to an already existing name, nor do we support such cases.
// Simply check if the table definition has changed.
if haveTable, ok := currentTables.Load(wantName); ok {
d.detectColumnChanges(haveTable, wantTable, true)
d.detectConstraintChanges(haveTable, wantTable)
continue
}
// Find all renamed tables. We assume that renamed tables have the same signature.
for _, havePair := range currentTables.Pairs() {
haveName, haveTable := havePair.Key, havePair.Value
if _, exists := targetTables.Load(haveName); !exists && d.canRename(haveTable, wantTable) {
d.changes.Add(&RenameTableOp{
TableName: haveTable.GetName(),
NewName: wantName,
})
d.refMap.RenameTable(haveTable.GetName(), wantName)
// Find renamed columns, if any, and check if constraints (PK, UNIQUE) have been updated.
// We need not check wantTable any further.
d.detectColumnChanges(haveTable, wantTable, false)
d.detectConstraintChanges(haveTable, wantTable)
currentTables.Delete(haveName)
continue RenameCreate
}
}
// If wantTable does not exist in the database and was not renamed
// then we need to create this table in the database.
additional := wantTable.(*sqlschema.BunTable)
d.changes.Add(&CreateTableOp{
TableName: wantTable.GetName(),
Model: additional.Model,
})
}
// Drop any remaining "current" tables which do not have a model.
for _, tPair := range currentTables.Pairs() {
name, table := tPair.Key, tPair.Value
if _, keep := targetTables.Load(name); !keep {
d.changes.Add(&DropTableOp{
TableName: table.GetName(),
})
}
}
targetFKs := d.target.GetForeignKeys()
currentFKs := d.refMap.Deref()
for fk := range targetFKs {
if _, ok := currentFKs[fk]; !ok {
d.changes.Add(&AddForeignKeyOp{
ForeignKey: fk,
ConstraintName: "", // leave empty to let each dialect apply their convention
})
}
}
for fk, name := range currentFKs {
if _, ok := targetFKs[fk]; !ok {
d.changes.Add(&DropForeignKeyOp{
ConstraintName: name,
ForeignKey: fk,
})
}
}
return &d.changes
}
// toOrderedMap transforms a slice of objects to an ordered map, using return of GetName() as key.
func toOrderedMap[V interface{ GetName() string }](named []V) *ordered.Map[string, V] {
m := ordered.NewMap[string, V]()
for _, v := range named {
m.Store(v.GetName(), v)
}
return m
}
// detectColumnChanges finds renamed columns and, if checkType == true, columns with changed type.
func (d *detector) detectColumnChanges(current, target sqlschema.Table, checkType bool) {
currentColumns := toOrderedMap(current.GetColumns())
targetColumns := toOrderedMap(target.GetColumns())
ChangeRename:
for _, tPair := range targetColumns.Pairs() {
tName, tCol := tPair.Key, tPair.Value
// This column exists in the database, so it hasn't been renamed, dropped, or added.
// Still, we should not delete(columns, thisColumn), because later we will need to
// check that we do not try to rename a column to an already a name that already exists.
if cCol, ok := currentColumns.Load(tName); ok {
if checkType && !d.equalColumns(cCol, tCol) {
d.changes.Add(&ChangeColumnTypeOp{
TableName: target.GetName(),
Column: tName,
From: cCol,
To: d.makeTargetColDef(cCol, tCol),
})
}
continue
}
// Column tName does not exist in the database -- it's been either renamed or added.
// Find renamed columns first.
for _, cPair := range currentColumns.Pairs() {
cName, cCol := cPair.Key, cPair.Value
// Cannot rename if a column with this name already exists or the types differ.
if _, exists := targetColumns.Load(cName); exists || !d.equalColumns(tCol, cCol) {
continue
}
d.changes.Add(&RenameColumnOp{
TableName: target.GetName(),
OldName: cName,
NewName: tName,
})
d.refMap.RenameColumn(target.GetName(), cName, tName)
currentColumns.Delete(cName) // no need to check this column again
// Update primary key definition to avoid superficially recreating the constraint.
current.GetPrimaryKey().Columns.Replace(cName, tName)
continue ChangeRename
}
d.changes.Add(&AddColumnOp{
TableName: target.GetName(),
ColumnName: tName,
Column: tCol,
})
}
// Drop columns which do not exist in the target schema and were not renamed.
for _, cPair := range currentColumns.Pairs() {
cName, cCol := cPair.Key, cPair.Value
if _, keep := targetColumns.Load(cName); !keep {
d.changes.Add(&DropColumnOp{
TableName: target.GetName(),
ColumnName: cName,
Column: cCol,
})
}
}
}
func (d *detector) detectConstraintChanges(current, target sqlschema.Table) {
Add:
for _, want := range target.GetUniqueConstraints() {
for _, got := range current.GetUniqueConstraints() {
if got.Equals(want) {
continue Add
}
}
d.changes.Add(&AddUniqueConstraintOp{
TableName: target.GetName(),
Unique: want,
})
}
Drop:
for _, got := range current.GetUniqueConstraints() {
for _, want := range target.GetUniqueConstraints() {
if got.Equals(want) {
continue Drop
}
}
d.changes.Add(&DropUniqueConstraintOp{
TableName: target.GetName(),
Unique: got,
})
}
targetPK := target.GetPrimaryKey()
currentPK := current.GetPrimaryKey()
// Detect primary key changes
if targetPK == nil && currentPK == nil {
return
}
switch {
case targetPK == nil && currentPK != nil:
d.changes.Add(&DropPrimaryKeyOp{
TableName: target.GetName(),
PrimaryKey: *currentPK,
})
case currentPK == nil && targetPK != nil:
d.changes.Add(&AddPrimaryKeyOp{
TableName: target.GetName(),
PrimaryKey: *targetPK,
})
case targetPK.Columns != currentPK.Columns:
d.changes.Add(&ChangePrimaryKeyOp{
TableName: target.GetName(),
Old: *currentPK,
New: *targetPK,
})
}
}
func newDetector(got, want sqlschema.Database, opts ...diffOption) *detector {
cfg := &detectorConfig{
cmpType: func(c1, c2 sqlschema.Column) bool {
return c1.GetSQLType() == c2.GetSQLType() && c1.GetVarcharLen() == c2.GetVarcharLen()
},
}
for _, opt := range opts {
opt(cfg)
}
return &detector{
current: got,
target: want,
refMap: newRefMap(got.GetForeignKeys()),
cmpType: cfg.cmpType,
}
}
type diffOption func(*detectorConfig)
func withCompareTypeFunc(f CompareTypeFunc) diffOption {
return func(cfg *detectorConfig) {
cfg.cmpType = f
}
}
// detectorConfig controls how differences in the model states are resolved.
type detectorConfig struct {
cmpType CompareTypeFunc
}
// detector may modify the passed database schemas, so it isn't safe to re-use them.
type detector struct {
// current state represents the existing database schema.
current sqlschema.Database
// target state represents the database schema defined in bun models.
target sqlschema.Database
changes changeset
refMap refMap
// cmpType determines column type equivalence.
// Default is direct comparison with '==' operator, which is inaccurate
// due to the existence of dialect-specific type aliases. The caller
// should pass a concrete InspectorDialect.EquivalentType for robust comparison.
cmpType CompareTypeFunc
}
// canRename checks if t1 can be renamed to t2.
func (d detector) canRename(t1, t2 sqlschema.Table) bool {
return t1.GetSchema() == t2.GetSchema() && equalSignatures(t1, t2, d.equalColumns)
}
func (d detector) equalColumns(col1, col2 sqlschema.Column) bool {
return d.cmpType(col1, col2) &&
col1.GetDefaultValue() == col2.GetDefaultValue() &&
col1.GetIsNullable() == col2.GetIsNullable() &&
col1.GetIsAutoIncrement() == col2.GetIsAutoIncrement() &&
col1.GetIsIdentity() == col2.GetIsIdentity()
}
func (d detector) makeTargetColDef(current, target sqlschema.Column) sqlschema.Column {
// Avoid unnecessary type-change migrations if the types are equivalent.
if d.cmpType(current, target) {
target = &sqlschema.BaseColumn{
Name: target.GetName(),
DefaultValue: target.GetDefaultValue(),
IsNullable: target.GetIsNullable(),
IsAutoIncrement: target.GetIsAutoIncrement(),
IsIdentity: target.GetIsIdentity(),
SQLType: current.GetSQLType(),
VarcharLen: current.GetVarcharLen(),
}
}
return target
}
// CompareTypeFunc compares two column definitions and reports whether they have the same SQL type.
type CompareTypeFunc func(sqlschema.Column, sqlschema.Column) bool
// equalSignatures determines if two tables have the same "signature".
func equalSignatures(t1, t2 sqlschema.Table, eq CompareTypeFunc) bool {
sig1 := newSignature(t1, eq)
sig2 := newSignature(t2, eq)
return sig1.Equals(sig2)
}
// signature is a set of column definitions, which allows "relation/name-agnostic" comparison between them;
// meaning that two columns are considered equal if their types are the same.
type signature struct {
// underlying stores the number of occurrences for each unique column type.
// It helps to account for the fact that a table might have multiple columns that have the same type.
underlying map[sqlschema.BaseColumn]int
eq CompareTypeFunc
}
func newSignature(t sqlschema.Table, eq CompareTypeFunc) signature {
s := signature{
underlying: make(map[sqlschema.BaseColumn]int),
eq: eq,
}
s.scan(t)
return s
}
// scan iterates over table's field and counts occurrences of each unique column definition.
func (s *signature) scan(t sqlschema.Table) {
for _, icol := range t.GetColumns() {
scanCol := icol.(*sqlschema.BaseColumn)
// This is slightly more expensive than if the columns could be compared directly
// and we always did s.underlying[col]++, but we get type-equivalence in return.
col, count := s.getCount(*scanCol)
if count == 0 {
s.underlying[*scanCol] = 1
} else {
s.underlying[col]++
}
}
}
// getCount uses CompareTypeFunc to find a column with the same (equivalent) SQL type
// and returns its count. Count 0 means there are no columns with of this type.
func (s *signature) getCount(keyCol sqlschema.BaseColumn) (key sqlschema.BaseColumn, count int) {
for col, cnt := range s.underlying {
if s.eq(&col, &keyCol) {
return col, cnt
}
}
return keyCol, 0
}
// Equals returns true if 2 signatures share an identical set of columns.
func (s *signature) Equals(other signature) bool {
if len(s.underlying) != len(other.underlying) {
return false
}
for col, count := range s.underlying {
if _, countOther := other.getCount(col); countOther != count {
return false
}
}
return true
}
// refMap is a utility for tracking superficial changes in foreign keys,
// which do not require any modification in the database.
// Modern SQL dialects automatically updated foreign key constraints whenever
// a column or a table is renamed. Detector can use refMap to ignore any
// differences in foreign keys which were caused by renamed column/table.
type refMap map[*sqlschema.ForeignKey]string
func newRefMap(fks map[sqlschema.ForeignKey]string) refMap {
rm := make(map[*sqlschema.ForeignKey]string)
for fk, name := range fks {
rm[&fk] = name
}
return rm
}
// RenameT updates table name in all foreign key definions which depend on it.
func (rm refMap) RenameTable(tableName string, newName string) {
for fk := range rm {
switch tableName {
case fk.From.TableName:
fk.From.TableName = newName
case fk.To.TableName:
fk.To.TableName = newName
}
}
}
// RenameColumn updates column name in all foreign key definions which depend on it.
func (rm refMap) RenameColumn(tableName string, column, newName string) {
for fk := range rm {
if tableName == fk.From.TableName {
fk.From.Column.Replace(column, newName)
}
if tableName == fk.To.TableName {
fk.To.Column.Replace(column, newName)
}
}
}
// Deref returns copies of ForeignKey values to a map.
func (rm refMap) Deref() map[sqlschema.ForeignKey]string {
out := make(map[sqlschema.ForeignKey]string)
for fk, name := range rm {
out[*fk] = name
}
return out
}
+357
View File
@@ -0,0 +1,357 @@
package migrate
import (
"bufio"
"bytes"
"context"
"fmt"
"io"
"io/fs"
"slices"
"strings"
"text/template"
"time"
"github.com/uptrace/bun"
)
// Migration represents a single database migration with up and down functions.
type Migration struct {
bun.BaseModel
ID int64 `bun:",pk,autoincrement"`
Name string
Comment string `bun:"-"`
GroupID int64
MigratedAt time.Time `bun:",notnull,nullzero,default:current_timestamp"`
Up internalMigrationFunc `bun:"-"`
Down internalMigrationFunc `bun:"-"`
}
// String returns the migration name and comment.
func (m Migration) String() string {
return fmt.Sprintf("%s_%s", m.Name, m.Comment)
}
// IsApplied reports whether the migration has been applied.
func (m Migration) IsApplied() bool {
return m.ID > 0
}
// MigrationFunc is a function that executes a migration against a database.
type MigrationFunc func(ctx context.Context, db *bun.DB) error
type internalMigrationFunc func(ctx context.Context, migrator *Migrator, migration *Migration) error
func wrapGoMigrationFunc(fn MigrationFunc) internalMigrationFunc {
return func(ctx context.Context, migrator *Migrator, migration *Migration) error {
if migrator.beforeMigrationHook != nil {
if err := migrator.beforeMigrationHook(ctx, migrator.db, migration); err != nil {
return err
}
}
if err := fn(ctx, migrator.db); err != nil {
return err
}
if migrator.afterMigrationHook != nil {
if err := migrator.afterMigrationHook(ctx, migrator.db, migration); err != nil {
return err
}
}
return nil
}
}
func newSQLMigrationFunc(fsys fs.FS, name string) internalMigrationFunc {
return func(ctx context.Context, migrator *Migrator, migration *Migration) error {
sqlFile, err := fsys.Open(name)
if err != nil {
return err
}
contents, err := io.ReadAll(sqlFile)
if err != nil {
return err
}
var reader io.Reader = bytes.NewReader(contents)
if migrator.templateData != nil {
buf, err := renderTemplate(contents, migrator.templateData)
if err != nil {
return err
}
reader = buf
}
scanner := bufio.NewScanner(reader)
var queries []string
var query []byte
for scanner.Scan() {
b := scanner.Bytes()
const prefix = "--bun:"
if bytes.HasPrefix(b, []byte(prefix)) {
b = b[len(prefix):]
if bytes.Equal(b, []byte("split")) {
queries = append(queries, string(query))
query = query[:0]
continue
}
return fmt.Errorf("bun: unknown directive: %q", b)
}
query = append(query, b...)
query = append(query, '\n')
}
if len(query) > 0 {
queries = append(queries, string(query))
}
if err := scanner.Err(); err != nil {
return err
}
var idb bun.IConn
isTx := strings.HasSuffix(name, ".tx.up.sql") || strings.HasSuffix(name, ".tx.down.sql")
if isTx {
tx, err := migrator.db.BeginTx(ctx, nil)
if err != nil {
return err
}
idb = tx
} else {
conn, err := migrator.db.Conn(ctx)
if err != nil {
return err
}
idb = conn
}
var retErr error
var execErr error
defer func() {
if tx, ok := idb.(bun.Tx); ok {
if execErr != nil {
retErr = tx.Rollback()
} else {
retErr = tx.Commit()
}
return
}
if conn, ok := idb.(bun.Conn); ok {
retErr = conn.Close()
return
}
panic("not reached")
}()
execErr = migrator.exec(ctx, idb, migration, queries)
if execErr != nil {
return execErr
}
return retErr
}
}
func renderTemplate(contents []byte, templateData any) (*bytes.Buffer, error) {
tmpl, err := template.New("migration").Parse(string(contents))
if err != nil {
return nil, fmt.Errorf("failed to parse template: %w", err)
}
var rendered bytes.Buffer
if err := tmpl.Execute(&rendered, templateData); err != nil {
return nil, fmt.Errorf("failed to execute template: %w", err)
}
return &rendered, nil
}
const goTemplate = `package %s
import (
"context"
"fmt"
"github.com/uptrace/bun"
)
func init() {
Migrations.MustRegister(func(ctx context.Context, db *bun.DB) error {
fmt.Print(" [up migration] ")
return nil
}, func(ctx context.Context, db *bun.DB) error {
fmt.Print(" [down migration] ")
return nil
})
}
`
const sqlTemplate = `SET statement_timeout = 0;
--bun:split
SELECT 1
--bun:split
SELECT 2
`
const transactionalSQLTemplate = `SET statement_timeout = 0;
SELECT 1;
`
//------------------------------------------------------------------------------
// MigrationSlice is a slice of migrations that provides helper methods for filtering and grouping.
type MigrationSlice []Migration
func (ms MigrationSlice) String() string {
if len(ms) == 0 {
return "empty"
}
if len(ms) > 5 {
return fmt.Sprintf("%d migrations (%s ... %s)", len(ms), ms[0].Name, ms[len(ms)-1].Name)
}
var sb strings.Builder
for i := range ms {
if i > 0 {
sb.WriteString(", ")
}
sb.WriteString(ms[i].String())
}
return sb.String()
}
// Applied returns applied migrations in descending order
// (the order is important and is used in Rollback).
func (ms MigrationSlice) Applied() MigrationSlice {
var applied MigrationSlice
for i := range ms {
if ms[i].IsApplied() {
applied = append(applied, ms[i])
}
}
sortDesc(applied)
return applied
}
// Unapplied returns unapplied migrations in ascending order
// (the order is important and is used in Migrate).
func (ms MigrationSlice) Unapplied() MigrationSlice {
var unapplied MigrationSlice
for i := range ms {
if !ms[i].IsApplied() {
unapplied = append(unapplied, ms[i])
}
}
sortAsc(unapplied)
return unapplied
}
// LastGroupID returns the last applied migration group id.
// The id is 0 when there are no migration groups.
func (ms MigrationSlice) LastGroupID() int64 {
var lastGroupID int64
for i := range ms {
groupID := ms[i].GroupID
if groupID > lastGroupID {
lastGroupID = groupID
}
}
return lastGroupID
}
// LastGroup returns the last applied migration group.
func (ms MigrationSlice) LastGroup() *MigrationGroup {
group := &MigrationGroup{
ID: ms.LastGroupID(),
}
if group.ID == 0 {
return group
}
for i := range ms {
if ms[i].GroupID == group.ID {
group.Migrations = append(group.Migrations, ms[i])
}
}
return group
}
// MigrationGroup is a group of migrations that were applied together in a single Migrate call.
type MigrationGroup struct {
ID int64
Migrations MigrationSlice
}
// IsZero reports whether the group is empty.
func (g MigrationGroup) IsZero() bool {
return g.ID == 0 && len(g.Migrations) == 0
}
func (g MigrationGroup) String() string {
if g.IsZero() {
return "nil"
}
return fmt.Sprintf("group #%d (%s)", g.ID, g.Migrations)
}
// MigrationFile represents a generated migration file on disk.
type MigrationFile struct {
Name string
Path string
Content string
}
//------------------------------------------------------------------------------
type migrationConfig struct {
nop bool
}
func newMigrationConfig(opts []MigrationOption) *migrationConfig {
cfg := new(migrationConfig)
for _, opt := range opts {
opt(cfg)
}
return cfg
}
// MigrationOption configures how a migration is executed.
type MigrationOption func(cfg *migrationConfig)
// WithNopMigration creates a no-op migration that marks itself as applied without running.
func WithNopMigration() MigrationOption {
return func(cfg *migrationConfig) {
cfg.nop = true
}
}
//------------------------------------------------------------------------------
func sortAsc(ms MigrationSlice) {
slices.SortFunc(ms, func(a, b Migration) int {
return strings.Compare(a.Name, b.Name)
})
}
func sortDesc(ms MigrationSlice) {
slices.SortFunc(ms, func(a, b Migration) int {
return strings.Compare(b.Name, a.Name)
})
}
+177
View File
@@ -0,0 +1,177 @@
package migrate
import (
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"regexp"
"runtime"
"strings"
)
// MigrationsOption configures a Migrations instance.
type MigrationsOption func(m *Migrations)
// WithMigrationsDirectory sets the directory where migration files are stored.
func WithMigrationsDirectory(directory string) MigrationsOption {
return func(m *Migrations) {
m.explicitDirectory = directory
}
}
// Migrations is a collection of registered migrations.
type Migrations struct {
ms MigrationSlice
explicitDirectory string
implicitDirectory string
}
// NewMigrations creates a new collection of migrations.
func NewMigrations(opts ...MigrationsOption) *Migrations {
m := new(Migrations)
for _, opt := range opts {
opt(m)
}
m.implicitDirectory = filepath.Dir(migrationFile())
return m
}
// Sorted returns a copy of the migrations sorted by name in ascending order.
func (m *Migrations) Sorted() MigrationSlice {
migrations := make(MigrationSlice, len(m.ms))
copy(migrations, m.ms)
sortAsc(migrations)
return migrations
}
// MustRegister is like Register but panics on error.
func (m *Migrations) MustRegister(up, down MigrationFunc) {
if err := m.Register(up, down); err != nil {
panic(err)
}
}
// Register registers up and down migration functions derived from the caller's file name.
func (m *Migrations) Register(up, down MigrationFunc) error {
fpath := migrationFile()
name, comment, err := extractMigrationName(fpath)
if err != nil {
return err
}
m.Add(Migration{
Name: name,
Comment: comment,
Up: wrapGoMigrationFunc(up),
Down: wrapGoMigrationFunc(down),
})
return nil
}
// Add appends a migration to the collection.
func (m *Migrations) Add(migration Migration) {
if migration.Name == "" {
panic("migration name is required")
}
m.ms = append(m.ms, migration)
}
// DiscoverCaller discovers SQL migration files in the caller's directory.
func (m *Migrations) DiscoverCaller() error {
dir := filepath.Dir(migrationFile())
return m.Discover(os.DirFS(dir))
}
// Discover discovers SQL migration files in the given filesystem.
func (m *Migrations) Discover(fsys fs.FS) error {
return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return nil
}
if !strings.HasSuffix(path, ".up.sql") && !strings.HasSuffix(path, ".down.sql") {
return nil
}
name, comment, err := extractMigrationName(path)
if err != nil {
return err
}
migration := m.getOrCreateMigration(name)
migration.Comment = comment
migrationFunc := newSQLMigrationFunc(fsys, path)
if strings.HasSuffix(path, ".up.sql") {
migration.Up = migrationFunc
return nil
}
if strings.HasSuffix(path, ".down.sql") {
migration.Down = migrationFunc
return nil
}
return errors.New("migrate: not reached")
})
}
func (m *Migrations) getOrCreateMigration(name string) *Migration {
for i := range m.ms {
mig := &m.ms[i]
if mig.Name == name {
return mig
}
}
m.ms = append(m.ms, Migration{Name: name})
return &m.ms[len(m.ms)-1]
}
func (m *Migrations) getDirectory() string {
if m.explicitDirectory != "" {
return m.explicitDirectory
}
if m.implicitDirectory != "" {
return m.implicitDirectory
}
return filepath.Dir(migrationFile())
}
func migrationFile() string {
const depth = 32
var pcs [depth]uintptr
n := runtime.Callers(1, pcs[:])
frames := runtime.CallersFrames(pcs[:n])
for {
f, ok := frames.Next()
if !ok {
break
}
if !strings.Contains(f.Function, "/bun/migrate.") {
return f.File
}
}
return ""
}
var fnameRE = regexp.MustCompile(`^(\d{1,14})_([0-9a-z_\-]+)\.`)
func extractMigrationName(fpath string) (string, string, error) {
fname := filepath.Base(fpath)
matches := fnameRE.FindStringSubmatch(fname)
if matches == nil {
return "", "", fmt.Errorf("migrate: unsupported migration name format: %q", fname)
}
return matches[1], matches[2], nil
}
+656
View File
@@ -0,0 +1,656 @@
package migrate
import (
"context"
"errors"
"fmt"
"os"
"path/filepath"
"regexp"
"strings"
"time"
"github.com/uptrace/bun"
"github.com/uptrace/bun/dialect/feature"
)
const (
defaultTable = "bun_migrations"
defaultLocksTable = "bun_migration_locks"
)
// MigratorOption configures a Migrator.
type MigratorOption func(m *Migrator)
// WithTableName overrides default migrations table name.
func WithTableName(table string) MigratorOption {
return func(m *Migrator) {
m.table = table
}
}
// WithLocksTableName overrides default migration locks table name.
func WithLocksTableName(table string) MigratorOption {
return func(m *Migrator) {
m.locksTable = table
}
}
// WithMarkAppliedOnSuccess sets the migrator to only mark migrations as applied/unapplied
// when their up/down is successful.
func WithMarkAppliedOnSuccess(enabled bool) MigratorOption {
return func(m *Migrator) {
m.markAppliedOnSuccess = enabled
}
}
// WithUpsert enables upsert (ON CONFLICT / ON DUPLICATE KEY / MERGE) in MarkApplied.
// This is required when re-running already-applied migrations via RunMigration.
// Init automatically creates a unique index on the name column.
func WithUpsert(enabled bool) MigratorOption {
return func(m *Migrator) {
m.useUpsert = enabled
}
}
// WithTemplateData sets data passed to SQL migration templates during rendering.
func WithTemplateData(data any) MigratorOption {
return func(m *Migrator) {
m.templateData = data
}
}
// MigrationHook is a callback invoked before or after each migration runs.
type MigrationHook func(ctx context.Context, db bun.IConn, migration *Migration) error
// BeforeMigration registers a hook that runs before each migration.
func BeforeMigration(hook MigrationHook) MigratorOption {
return func(m *Migrator) {
m.beforeMigrationHook = hook
}
}
// AfterMigration registers a hook that runs after each migration.
func AfterMigration(hook MigrationHook) MigratorOption {
return func(m *Migrator) {
m.afterMigrationHook = hook
}
}
// Migrator manages the lifecycle of database migrations.
type Migrator struct {
db *bun.DB
migrations *Migrations
table string
locksTable string
markAppliedOnSuccess bool
useUpsert bool
templateData any
beforeMigrationHook MigrationHook
afterMigrationHook MigrationHook
}
// NewMigrator creates a new Migrator for the given database and migrations.
func NewMigrator(db *bun.DB, migrations *Migrations, opts ...MigratorOption) *Migrator {
m := &Migrator{
db: db,
migrations: migrations,
table: defaultTable,
locksTable: defaultLocksTable,
}
for _, opt := range opts {
opt(m)
}
return m
}
// DB returns the underlying bun.DB.
func (m *Migrator) DB() *bun.DB {
return m.db
}
// MigrationsWithStatus returns migrations with status in ascending order.
func (m *Migrator) MigrationsWithStatus(ctx context.Context) (MigrationSlice, error) {
sorted, _, err := m.migrationsWithStatus(ctx)
return sorted, err
}
func (m *Migrator) migrationsWithStatus(ctx context.Context) (MigrationSlice, int64, error) {
sorted := m.migrations.Sorted()
applied, err := m.AppliedMigrations(ctx)
if err != nil {
return nil, 0, err
}
appliedMap := migrationMap(applied)
for i := range sorted {
m1 := &sorted[i]
if m2, ok := appliedMap[m1.Name]; ok {
m1.ID = m2.ID
m1.GroupID = m2.GroupID
m1.MigratedAt = m2.MigratedAt
}
}
return sorted, applied.LastGroupID(), nil
}
// Init creates the migration tables if they do not already exist.
func (m *Migrator) Init(ctx context.Context) error {
if _, err := m.db.NewCreateTable().
Model((*Migration)(nil)).
ModelTableExpr(m.table).
IfNotExists().
Exec(ctx); err != nil {
return err
}
if m.useUpsert {
if _, err := m.db.NewCreateIndex().
Unique().
TableExpr(m.table).
Index(m.table + "_name_unique").
Column("name").
IfNotExists().
Exec(ctx); err != nil && !isIndexAlreadyExistsError(err) {
return err
}
}
if _, err := m.db.NewCreateTable().
Model((*migrationLock)(nil)).
ModelTableExpr(m.locksTable).
IfNotExists().
Exec(ctx); err != nil {
return err
}
return nil
}
// Reset drops and re-creates the migration tables.
func (m *Migrator) Reset(ctx context.Context) error {
if _, err := m.db.NewDropTable().
Model((*Migration)(nil)).
ModelTableExpr(m.table).
IfExists().
Exec(ctx); err != nil {
return err
}
if _, err := m.db.NewDropTable().
Model((*migrationLock)(nil)).
ModelTableExpr(m.locksTable).
IfExists().
Exec(ctx); err != nil {
return err
}
return m.Init(ctx)
}
// Migrate runs unapplied migrations. If a migration fails, migrate immediately exits.
func (m *Migrator) Migrate(ctx context.Context, opts ...MigrationOption) (*MigrationGroup, error) {
cfg := newMigrationConfig(opts)
group := new(MigrationGroup)
if err := m.validate(); err != nil {
return group, err
}
migrations, lastGroupID, err := m.migrationsWithStatus(ctx)
if err != nil {
return group, err
}
migrations = migrations.Unapplied()
if len(migrations) == 0 {
return group, nil
}
group.ID = lastGroupID + 1
for i := range migrations {
migration := &migrations[i]
migration.GroupID = group.ID
if !m.markAppliedOnSuccess {
if err := m.MarkApplied(ctx, migration); err != nil {
return group, err
}
}
group.Migrations = migrations[:i+1]
if !cfg.nop && migration.Up != nil {
if err := migration.Up(ctx, m, migration); err != nil {
return group, fmt.Errorf("%s: up: %w", migration.Name, err)
}
}
if m.markAppliedOnSuccess {
if err := m.MarkApplied(ctx, migration); err != nil {
return group, err
}
}
}
return group, nil
}
// RunMigration runs the up migration with the given name and marks it as applied.
// It runs the migration even if it is already marked as applied.
// The migration is added as a new applied record, creating a separate migration group.
func (m *Migrator) RunMigration(
ctx context.Context, migrationName string, opts ...MigrationOption,
) error {
cfg := newMigrationConfig(opts)
if err := m.validate(); err != nil {
return err
}
if migrationName == "" {
return errors.New("migrate: migration name cannot be empty")
}
if !m.useUpsert {
return errors.New("migrate: RunMigration requires WithUpsert(true)")
}
migrations, lastGroupID, err := m.migrationsWithStatus(ctx)
if err != nil {
return err
}
var migration *Migration
for i := range migrations {
if migrations[i].Name == migrationName {
migration = &migrations[i]
break
}
}
if migration == nil {
return fmt.Errorf("migrate: migration with name %q not found", migrationName)
}
if migration.Up == nil {
return fmt.Errorf("migrate: migration %s does not have up migration", migration.Name)
}
if cfg.nop {
return nil
}
migration.GroupID = lastGroupID + 1
if !m.markAppliedOnSuccess {
if err := m.MarkApplied(ctx, migration); err != nil {
return err
}
}
if err := migration.Up(ctx, m, migration); err != nil {
return fmt.Errorf("%s: up: %w", migration.Name, err)
}
if m.markAppliedOnSuccess {
if err := m.MarkApplied(ctx, migration); err != nil {
return err
}
}
return nil
}
// Rollback rolls back the last migration group.
func (m *Migrator) Rollback(ctx context.Context, opts ...MigrationOption) (*MigrationGroup, error) {
cfg := newMigrationConfig(opts)
lastGroup := new(MigrationGroup)
if err := m.validate(); err != nil {
return lastGroup, err
}
migrations, err := m.MigrationsWithStatus(ctx)
if err != nil {
return lastGroup, err
}
lastGroup = migrations.LastGroup()
for i := len(lastGroup.Migrations) - 1; i >= 0; i-- {
migration := &lastGroup.Migrations[i]
if !m.markAppliedOnSuccess {
if err := m.MarkUnapplied(ctx, migration); err != nil {
return lastGroup, err
}
}
if !cfg.nop && migration.Down != nil {
if err := migration.Down(ctx, m, migration); err != nil {
return lastGroup, fmt.Errorf("%s: down: %w", migration.Name, err)
}
}
if m.markAppliedOnSuccess {
if err := m.MarkUnapplied(ctx, migration); err != nil {
return lastGroup, err
}
}
}
return lastGroup, nil
}
type goMigrationConfig struct {
packageName string
goTemplate string
}
// GoMigrationOption configures Go migration file generation.
type GoMigrationOption func(cfg *goMigrationConfig)
// WithPackageName sets the Go package name used in generated migration files.
func WithPackageName(name string) GoMigrationOption {
return func(cfg *goMigrationConfig) {
cfg.packageName = name
}
}
// WithGoTemplate sets the Go template string used for generated migration files.
func WithGoTemplate(template string) GoMigrationOption {
return func(cfg *goMigrationConfig) {
cfg.goTemplate = template
}
}
// CreateGoMigration creates a Go migration file.
func (m *Migrator) CreateGoMigration(
ctx context.Context, name string, opts ...GoMigrationOption,
) (*MigrationFile, error) {
cfg := &goMigrationConfig{
packageName: "migrations",
goTemplate: goTemplate,
}
for _, opt := range opts {
opt(cfg)
}
name, err := genMigrationName(name)
if err != nil {
return nil, err
}
fname := name + ".go"
fpath := filepath.Join(m.migrations.getDirectory(), fname)
content := fmt.Sprintf(cfg.goTemplate, cfg.packageName)
if err := os.WriteFile(fpath, []byte(content), 0o644); err != nil {
return nil, err
}
mf := &MigrationFile{
Name: fname,
Path: fpath,
Content: content,
}
return mf, nil
}
// CreateTxSQLMigration creates transactional up and down SQL migration files.
func (m *Migrator) CreateTxSQLMigrations(ctx context.Context, name string) ([]*MigrationFile, error) {
name, err := genMigrationName(name)
if err != nil {
return nil, err
}
up, err := m.createSQL(ctx, name+".tx.up.sql", true)
if err != nil {
return nil, err
}
down, err := m.createSQL(ctx, name+".tx.down.sql", true)
if err != nil {
return nil, err
}
return []*MigrationFile{up, down}, nil
}
// CreateSQLMigrations creates up and down SQL migration files.
func (m *Migrator) CreateSQLMigrations(ctx context.Context, name string) ([]*MigrationFile, error) {
name, err := genMigrationName(name)
if err != nil {
return nil, err
}
up, err := m.createSQL(ctx, name+".up.sql", false)
if err != nil {
return nil, err
}
down, err := m.createSQL(ctx, name+".down.sql", false)
if err != nil {
return nil, err
}
return []*MigrationFile{up, down}, nil
}
func (m *Migrator) createSQL(_ context.Context, fname string, transactional bool) (*MigrationFile, error) {
fpath := filepath.Join(m.migrations.getDirectory(), fname)
template := sqlTemplate
if transactional {
template = transactionalSQLTemplate
}
if err := os.WriteFile(fpath, []byte(template), 0o644); err != nil {
return nil, err
}
mf := &MigrationFile{
Name: fname,
Path: fpath,
Content: goTemplate,
}
return mf, nil
}
var nameRE = regexp.MustCompile(`^[0-9a-z_\-]+$`)
func genMigrationName(name string) (string, error) {
const timeFormat = "20060102150405"
if name == "" {
return "", errors.New("migrate: migration name can't be empty")
}
if !nameRE.MatchString(name) {
return "", fmt.Errorf("migrate: invalid migration name: %q", name)
}
version := time.Now().UTC().Format(timeFormat)
return fmt.Sprintf("%s_%s", version, name), nil
}
// MarkApplied marks the migration as applied (completed).
func (m *Migrator) MarkApplied(ctx context.Context, migration *Migration) error {
q := m.db.NewInsert().Model(migration).
ModelTableExpr(m.table)
if m.useUpsert {
switch {
case m.db.HasFeature(feature.InsertOnConflict):
q = q.On("CONFLICT (name) DO UPDATE").
Set("group_id = EXCLUDED.group_id").
Set("migrated_at = EXCLUDED.migrated_at")
case m.db.HasFeature(feature.InsertOnDuplicateKey):
q = q.On("DUPLICATE KEY UPDATE").
Set("group_id = VALUES(group_id)").
Set("migrated_at = VALUES(migrated_at)")
case m.db.HasFeature(feature.Merge):
source := MigrationSlice{*migration}
_, err := m.db.NewMerge().
Model(migration).
ModelTableExpr("? AS migration", bun.Name(m.table)).
With("_data", m.db.NewValues(&source)).
Using("_data").
On("migration.name = _data.name").
WhenUpdate("MATCHED", func(q *bun.UpdateQuery) *bun.UpdateQuery {
return q.
Set("group_id = _data.group_id").
Set("migrated_at = _data.migrated_at")
}).
WhenInsert("NOT MATCHED", func(q *bun.InsertQuery) *bun.InsertQuery {
return q.
Value("name", "_data.name").
Value("group_id", "_data.group_id").
Value("migrated_at", "_data.migrated_at")
}).
Exec(ctx)
return err
default:
return errors.New("migrate: dialect does not support upsert or merge")
}
}
_, err := q.Exec(ctx)
return err
}
// MarkUnapplied marks the migration as unapplied (new).
func (m *Migrator) MarkUnapplied(ctx context.Context, migration *Migration) error {
_, err := m.db.NewDelete().
Model(migration).
ModelTableExpr(m.table).
Where("id = ?", migration.ID).
Exec(ctx)
return err
}
// TruncateTable removes all rows from the migrations table.
func (m *Migrator) TruncateTable(ctx context.Context) error {
_, err := m.db.NewTruncateTable().
Model((*Migration)(nil)).
ModelTableExpr(m.table).
Exec(ctx)
return err
}
// MissingMigrations returns applied migrations that can no longer be found.
func (m *Migrator) MissingMigrations(ctx context.Context) (MigrationSlice, error) {
applied, err := m.AppliedMigrations(ctx)
if err != nil {
return nil, err
}
existing := migrationMap(m.migrations.ms)
for i := len(applied) - 1; i >= 0; i-- {
m := &applied[i]
if _, ok := existing[m.Name]; ok {
applied = append(applied[:i], applied[i+1:]...)
}
}
return applied, nil
}
// AppliedMigrations returns applied (applied) migrations in descending order.
func (m *Migrator) AppliedMigrations(ctx context.Context) (MigrationSlice, error) {
var ms MigrationSlice
if err := m.db.NewSelect().
ColumnExpr("*").
Model(&ms).
ModelTableExpr(m.table).
Scan(ctx); err != nil {
return nil, err
}
return ms, nil
}
func (m *Migrator) formattedTableName(db *bun.DB) string {
return db.QueryGen().FormatQuery(m.table)
}
func (m *Migrator) validate() error {
if len(m.migrations.ms) == 0 {
return errors.New("migrate: there are no migrations")
}
return nil
}
func (m *Migrator) exec(
ctx context.Context, db bun.IConn, migration *Migration, queries []string,
) error {
if m.beforeMigrationHook != nil {
if err := m.beforeMigrationHook(ctx, db, migration); err != nil {
return err
}
}
for _, query := range queries {
if strings.TrimSpace(query) == "" {
continue
}
if _, err := db.ExecContext(ctx, query); err != nil {
return err
}
}
if m.afterMigrationHook != nil {
if err := m.afterMigrationHook(ctx, db, migration); err != nil {
return err
}
}
return nil
}
//------------------------------------------------------------------------------
type migrationLock struct {
ID int64 `bun:",pk,autoincrement"`
TableName string `bun:",unique"`
}
// Lock acquires an advisory lock on the migration table to prevent concurrent migrations.
func (m *Migrator) Lock(ctx context.Context) error {
lock := &migrationLock{
TableName: m.formattedTableName(m.db),
}
if _, err := m.db.NewInsert().
Model(lock).
ModelTableExpr(m.locksTable).
Exec(ctx); err != nil {
return fmt.Errorf("migrate: migrations table is already locked (%w)", err)
}
return nil
}
// Unlock releases the advisory lock on the migration table.
func (m *Migrator) Unlock(ctx context.Context) error {
tableName := m.formattedTableName(m.db)
_, err := m.db.NewDelete().
Model((*migrationLock)(nil)).
ModelTableExpr(m.locksTable).
Where("? = ?", bun.Ident("table_name"), tableName).
Exec(ctx)
return err
}
// isIndexAlreadyExistsError checks whether err indicates the index already exists.
// This is needed for dialects that do not support CREATE INDEX IF NOT EXISTS
// (e.g. MySQL, MSSQL), where a duplicate-index error is expected on repeated Init calls.
func isIndexAlreadyExistsError(err error) bool {
s := strings.ToLower(err.Error())
// MySQL: Error 1061: Duplicate key name '...'
// MSSQL: The index '...' already exists on table '...'
// Oracle: ORA-00955: name is already used by an existing object
return strings.Contains(s, "duplicate key name") || strings.Contains(s, "already exist")
}
func migrationMap(ms MigrationSlice) map[string]*Migration {
mp := make(map[string]*Migration)
for i := range ms {
m := &ms[i]
mp[m.Name] = m
}
return mp
}
+339
View File
@@ -0,0 +1,339 @@
package migrate
import (
"fmt"
"github.com/uptrace/bun/migrate/sqlschema"
)
// Operation encapsulates the request to change a database definition
// and knowns which operation can revert it.
//
// It is useful to define "monolith" Operations whenever possible,
// even though they a dialect may require several distinct steps to apply them.
// For example, changing a primary key involves first dropping the old constraint
// before generating the new one. Yet, this is only an implementation detail and
// passing a higher-level ChangePrimaryKeyOp will give the dialect more information
// about the applied change.
//
// Some operations might be irreversible due to technical limitations. Returning
// a *comment from GetReverse() will add an explanatory note to the generate migration file.
//
// To declare dependency on another Operation, operations should implement
// { DependsOn(Operation) bool } interface, which Changeset will use to resolve dependencies.
type Operation interface {
GetReverse() Operation
}
// CreateTableOp creates a new table in the schema.
//
// It does not report dependency on any other migration and may be executed first.
// Make sure the dialect does not include FOREIGN KEY constraints in the CREATE TABLE
// statement, as those may potentially reference not-yet-existing columns/tables.
type CreateTableOp struct {
TableName string
Model any
}
var _ Operation = (*CreateTableOp)(nil)
func (op *CreateTableOp) GetReverse() Operation {
return &DropTableOp{TableName: op.TableName}
}
// DropTableOp drops a database table. This operation is not reversible.
type DropTableOp struct {
TableName string
}
var _ Operation = (*DropTableOp)(nil)
func (op *DropTableOp) DependsOn(another Operation) bool {
drop, ok := another.(*DropForeignKeyOp)
return ok && drop.ForeignKey.DependsOnTable(op.TableName)
}
// GetReverse for a DropTable returns a no-op migration. Logically, CreateTable is the reverse,
// but DropTable does not have the table's definition to create one.
func (op *DropTableOp) GetReverse() Operation {
c := Unimplemented(fmt.Sprintf("WARNING: \"DROP TABLE %s\" cannot be reversed automatically because table definition is not available", op.TableName))
return &c
}
// RenameTableOp renames the table. Changing the "schema" part of the table's FQN (moving tables between schemas) is not allowed.
type RenameTableOp struct {
TableName string
NewName string
}
var _ Operation = (*RenameTableOp)(nil)
func (op *RenameTableOp) GetReverse() Operation {
return &RenameTableOp{
TableName: op.NewName,
NewName: op.TableName,
}
}
// RenameColumnOp renames a column in the table. If the changeset includes a rename operation
// for the column's table, it should be executed first.
type RenameColumnOp struct {
TableName string
OldName string
NewName string
}
var _ Operation = (*RenameColumnOp)(nil)
func (op *RenameColumnOp) GetReverse() Operation {
return &RenameColumnOp{
TableName: op.TableName,
OldName: op.NewName,
NewName: op.OldName,
}
}
func (op *RenameColumnOp) DependsOn(another Operation) bool {
rename, ok := another.(*RenameTableOp)
return ok && op.TableName == rename.NewName
}
// AddColumnOp adds a new column to the table.
type AddColumnOp struct {
TableName string
ColumnName string
Column sqlschema.Column
}
var _ Operation = (*AddColumnOp)(nil)
func (op *AddColumnOp) GetReverse() Operation {
return &DropColumnOp{
TableName: op.TableName,
ColumnName: op.ColumnName,
Column: op.Column,
}
}
// DropColumnOp drop a column from the table.
//
// While some dialects allow DROP CASCADE to drop dependent constraints,
// explicit handling on constraints is preferred for transparency and debugging.
// DropColumnOp depends on DropForeignKeyOp, DropPrimaryKeyOp, and ChangePrimaryKeyOp
// if any of the constraints is defined on this table.
type DropColumnOp struct {
TableName string
ColumnName string
Column sqlschema.Column
}
var _ Operation = (*DropColumnOp)(nil)
func (op *DropColumnOp) GetReverse() Operation {
return &AddColumnOp{
TableName: op.TableName,
ColumnName: op.ColumnName,
Column: op.Column,
}
}
func (op *DropColumnOp) DependsOn(another Operation) bool {
switch drop := another.(type) {
case *DropForeignKeyOp:
return drop.ForeignKey.DependsOnColumn(op.TableName, op.ColumnName)
case *DropPrimaryKeyOp:
return op.TableName == drop.TableName && drop.PrimaryKey.Columns.Contains(op.ColumnName)
case *ChangePrimaryKeyOp:
return op.TableName == drop.TableName && drop.Old.Columns.Contains(op.ColumnName)
}
return false
}
// AddForeignKey adds a new FOREIGN KEY constraint.
type AddForeignKeyOp struct {
ForeignKey sqlschema.ForeignKey
ConstraintName string
}
var _ Operation = (*AddForeignKeyOp)(nil)
func (op *AddForeignKeyOp) TableName() string {
return op.ForeignKey.From.TableName
}
func (op *AddForeignKeyOp) DependsOn(another Operation) bool {
switch another := another.(type) {
case *RenameTableOp:
return op.ForeignKey.DependsOnTable(another.TableName) || op.ForeignKey.DependsOnTable(another.NewName)
case *CreateTableOp:
return op.ForeignKey.DependsOnTable(another.TableName)
}
return false
}
func (op *AddForeignKeyOp) GetReverse() Operation {
return &DropForeignKeyOp{
ForeignKey: op.ForeignKey,
ConstraintName: op.ConstraintName,
}
}
// DropForeignKeyOp drops a FOREIGN KEY constraint.
type DropForeignKeyOp struct {
ForeignKey sqlschema.ForeignKey
ConstraintName string
}
var _ Operation = (*DropForeignKeyOp)(nil)
func (op *DropForeignKeyOp) TableName() string {
return op.ForeignKey.From.TableName
}
func (op *DropForeignKeyOp) GetReverse() Operation {
return &AddForeignKeyOp{
ForeignKey: op.ForeignKey,
ConstraintName: op.ConstraintName,
}
}
// AddUniqueConstraintOp adds new UNIQUE constraint to the table.
type AddUniqueConstraintOp struct {
TableName string
Unique sqlschema.Unique
}
var _ Operation = (*AddUniqueConstraintOp)(nil)
func (op *AddUniqueConstraintOp) GetReverse() Operation {
return &DropUniqueConstraintOp{
TableName: op.TableName,
Unique: op.Unique,
}
}
func (op *AddUniqueConstraintOp) DependsOn(another Operation) bool {
switch another := another.(type) {
case *AddColumnOp:
return op.TableName == another.TableName && op.Unique.Columns.Contains(another.ColumnName)
case *RenameTableOp:
return op.TableName == another.NewName
case *DropUniqueConstraintOp:
// We want to drop the constraint with the same name before adding this one.
return op.TableName == another.TableName && op.Unique.Name == another.Unique.Name
default:
return false
}
}
// DropUniqueConstraintOp drops a UNIQUE constraint.
type DropUniqueConstraintOp struct {
TableName string
Unique sqlschema.Unique
}
var _ Operation = (*DropUniqueConstraintOp)(nil)
func (op *DropUniqueConstraintOp) DependsOn(another Operation) bool {
if rename, ok := another.(*RenameTableOp); ok {
return op.TableName == rename.NewName
}
return false
}
func (op *DropUniqueConstraintOp) GetReverse() Operation {
return &AddUniqueConstraintOp{
TableName: op.TableName,
Unique: op.Unique,
}
}
// ChangeColumnTypeOp set a new data type for the column.
// The two types should be such that the data can be auto-casted from one to another.
// E.g. reducing VARCHAR lenght is not possible in most dialects.
// AutoMigrator does not enforce or validate these rules.
type ChangeColumnTypeOp struct {
TableName string
Column string
From sqlschema.Column
To sqlschema.Column
}
var _ Operation = (*ChangeColumnTypeOp)(nil)
func (op *ChangeColumnTypeOp) GetReverse() Operation {
return &ChangeColumnTypeOp{
TableName: op.TableName,
Column: op.Column,
From: op.To,
To: op.From,
}
}
// DropPrimaryKeyOp drops the table's PRIMARY KEY.
type DropPrimaryKeyOp struct {
TableName string
PrimaryKey sqlschema.PrimaryKey
}
var _ Operation = (*DropPrimaryKeyOp)(nil)
func (op *DropPrimaryKeyOp) GetReverse() Operation {
return &AddPrimaryKeyOp{
TableName: op.TableName,
PrimaryKey: op.PrimaryKey,
}
}
// AddPrimaryKeyOp adds a new PRIMARY KEY to the table.
type AddPrimaryKeyOp struct {
TableName string
PrimaryKey sqlschema.PrimaryKey
}
var _ Operation = (*AddPrimaryKeyOp)(nil)
func (op *AddPrimaryKeyOp) GetReverse() Operation {
return &DropPrimaryKeyOp{
TableName: op.TableName,
PrimaryKey: op.PrimaryKey,
}
}
func (op *AddPrimaryKeyOp) DependsOn(another Operation) bool {
switch another := another.(type) {
case *AddColumnOp:
return op.TableName == another.TableName && op.PrimaryKey.Columns.Contains(another.ColumnName)
}
return false
}
// ChangePrimaryKeyOp changes the PRIMARY KEY of the table.
type ChangePrimaryKeyOp struct {
TableName string
Old sqlschema.PrimaryKey
New sqlschema.PrimaryKey
}
var _ Operation = (*AddPrimaryKeyOp)(nil)
func (op *ChangePrimaryKeyOp) GetReverse() Operation {
return &ChangePrimaryKeyOp{
TableName: op.TableName,
Old: op.New,
New: op.Old,
}
}
// Unimplemented denotes an Operation that cannot be executed.
//
// Operations, which cannot be reversed due to current technical limitations,
// may have their GetReverse() return &Unimplemented with a helpful message.
//
// When applying operations, changelog should skip it or output as a log message,
// and write it as an SQL Unimplemented when creating migration files.
type Unimplemented string
var _ Operation = (*Unimplemented)(nil)
func (reason *Unimplemented) GetReverse() Operation { return reason }
+75
View File
@@ -0,0 +1,75 @@
package sqlschema
import (
"fmt"
"github.com/uptrace/bun/schema"
)
type Column interface {
GetName() string
GetSQLType() string
GetVarcharLen() int
GetDefaultValue() string
GetIsNullable() bool
GetIsAutoIncrement() bool
GetIsIdentity() bool
AppendQuery(schema.QueryGen, []byte) ([]byte, error)
}
var _ Column = (*BaseColumn)(nil)
// BaseColumn is a base column definition that stores various attributes of a column.
//
// Dialects and only dialects can use it to implement the Column interface.
// Other packages must use the Column interface.
type BaseColumn struct {
Name string
SQLType string
VarcharLen int
DefaultValue string
IsNullable bool
IsAutoIncrement bool
IsIdentity bool
// TODO: add Precision and Cardinality for timestamps/bit-strings/floats and arrays respectively.
}
func (cd BaseColumn) GetName() string {
return cd.Name
}
func (cd BaseColumn) GetSQLType() string {
return cd.SQLType
}
func (cd BaseColumn) GetVarcharLen() int {
return cd.VarcharLen
}
func (cd BaseColumn) GetDefaultValue() string {
return cd.DefaultValue
}
func (cd BaseColumn) GetIsNullable() bool {
return cd.IsNullable
}
func (cd BaseColumn) GetIsAutoIncrement() bool {
return cd.IsAutoIncrement
}
func (cd BaseColumn) GetIsIdentity() bool {
return cd.IsIdentity
}
// AppendQuery appends full SQL data type.
func (c *BaseColumn) AppendQuery(gen schema.QueryGen, b []byte) (_ []byte, err error) {
b = append(b, c.SQLType...)
if c.VarcharLen == 0 {
return b, nil
}
b = append(b, "("...)
b = append(b, fmt.Sprint(c.VarcharLen)...)
b = append(b, ")"...)
return b, nil
}
+128
View File
@@ -0,0 +1,128 @@
package sqlschema
import (
"slices"
"strings"
"github.com/uptrace/bun/schema"
)
type Database interface {
GetTables() []Table
GetForeignKeys() map[ForeignKey]string
}
var _ Database = (*BaseDatabase)(nil)
// BaseDatabase is a base database definition.
//
// Dialects and only dialects can use it to implement the Database interface.
// Other packages must use the Database interface.
type BaseDatabase struct {
Tables []Table
ForeignKeys map[ForeignKey]string
}
func (ds BaseDatabase) GetTables() []Table {
return ds.Tables
}
func (ds BaseDatabase) GetForeignKeys() map[ForeignKey]string {
return ds.ForeignKeys
}
// ForeignKey represents a foreign key constraint between two tables.
type ForeignKey struct {
From ColumnReference
To ColumnReference
}
func NewColumnReference(tableName string, columns ...string) ColumnReference {
return ColumnReference{
TableName: tableName,
Column: NewColumns(columns...),
}
}
func (fk ForeignKey) DependsOnTable(tableName string) bool {
return fk.From.TableName == tableName || fk.To.TableName == tableName
}
func (fk ForeignKey) DependsOnColumn(tableName string, column string) bool {
return fk.DependsOnTable(tableName) &&
(fk.From.Column.Contains(column) || fk.To.Column.Contains(column))
}
// Columns is a hashable representation of []string used to define schema constraints that depend on multiple columns.
// Although having duplicated column references in these constraints is illegal, Columns neither validates nor enforces this constraint on the caller.
type Columns string
// NewColumns creates a composite column from a slice of column names.
func NewColumns(columns ...string) Columns {
slices.Sort(columns)
return Columns(strings.Join(columns, ","))
}
func (c *Columns) String() string {
return string(*c)
}
func (c *Columns) AppendQuery(gen schema.QueryGen, b []byte) ([]byte, error) {
return schema.Safe(*c).AppendQuery(gen, b)
}
// Split returns a slice of column names that make up the composite.
func (c *Columns) Split() []string {
return strings.Split(c.String(), ",")
}
// ContainsColumns checks that columns in "other" are a subset of current colums.
func (c *Columns) ContainsColumns(other Columns) bool {
columns := c.Split()
Outer:
for _, check := range other.Split() {
for _, column := range columns {
if check == column {
continue Outer
}
}
return false
}
return true
}
// Contains checks that a composite column contains the current column.
func (c *Columns) Contains(other string) bool {
return c.ContainsColumns(Columns(other))
}
// Replace renames a column if it is part of the composite.
// If a composite consists of multiple columns, only one column will be renamed.
func (c *Columns) Replace(oldColumn, newColumn string) bool {
columns := c.Split()
for i, column := range columns {
if column == oldColumn {
columns[i] = newColumn
*c = NewColumns(columns...)
return true
}
}
return false
}
// Unique represents a unique constraint defined on 1 or more columns.
type Unique struct {
Name string
Columns Columns
}
// Equals checks that two unique constraint are the same, assuming both are defined for the same table.
func (u Unique) Equals(other Unique) bool {
return u.Columns == other.Columns
}
// ColumnReference identifies a column or set of columns in a table.
type ColumnReference struct {
TableName string
Column Columns
}
+274
View File
@@ -0,0 +1,274 @@
package sqlschema
import (
"context"
"fmt"
"strconv"
"strings"
"github.com/uptrace/bun"
"github.com/uptrace/bun/schema"
)
type InspectorDialect interface {
schema.Dialect
// Inspector returns a new instance of Inspector for the dialect.
// Dialects MAY set their default InspectorConfig values in constructor
// but MUST apply InspectorOptions to ensure they can be overriden.
//
// Use ApplyInspectorOptions to reduce boilerplate.
NewInspector(db *bun.DB, options ...InspectorOption) Inspector
// CompareType returns true if col1 and co2 SQL types are equivalent,
// i.e. they might use dialect-specifc type aliases (SERIAL ~ SMALLINT)
// or specify the same VARCHAR length differently (VARCHAR(255) ~ VARCHAR).
CompareType(Column, Column) bool
}
// InspectorConfig controls the scope of migration by limiting the objects Inspector should return.
// Inspectors SHOULD use the configuration directly instead of copying it, or MAY choose to embed it,
// to make sure options are always applied correctly.
//
// ExcludeTables and ExcludeForeignKeys are intended for database inspectors,
// to compensate for the fact that model structs may not wholly reflect the
// state of the database schema.
// Database inspectors MUST respect these exclusions to prevent relations
// from being dropped unintentionally.
type InspectorConfig struct {
// SchemaName limits inspection to tables in a particular schema.
SchemaName string
// ExcludeTables from inspection. Patterns MAY make use of wildcards
// like % and _ and dialects MUST acknowledge that by using them
// with the SQL LIKE operator.
ExcludeTables []string
// ExcludeForeignKeys from inspection.
ExcludeForeignKeys map[ForeignKey]string
}
// Inspector reads schema state.
type Inspector interface {
Inspect(ctx context.Context) (Database, error)
}
func WithSchemaName(schemaName string) InspectorOption {
return func(cfg *InspectorConfig) {
cfg.SchemaName = schemaName
}
}
// WithExcludeTables forces inspector to exclude tables from the reported schema state.
// It works in append-only mode, i.e. tables cannot be re-included.
//
// Patterns MAY make use of % and _ wildcards, as if writing a LIKE clause in SQL.
func WithExcludeTables(tables ...string) InspectorOption {
return func(cfg *InspectorConfig) {
cfg.ExcludeTables = append(cfg.ExcludeTables, tables...)
}
}
// WithExcludeForeignKeys forces inspector to exclude foreign keys
// from the reported schema state.
func WithExcludeForeignKeys(fks ...ForeignKey) InspectorOption {
return func(cfg *InspectorConfig) {
for _, fk := range fks {
cfg.ExcludeForeignKeys[fk] = ""
}
}
}
// NewInspector creates a new database inspector, if the dialect supports it.
func NewInspector(db *bun.DB, options ...InspectorOption) (Inspector, error) {
dialect, ok := (db.Dialect()).(InspectorDialect)
if !ok {
return nil, fmt.Errorf("%s does not implement sqlschema.Inspector", db.Dialect().Name())
}
return &inspector{
Inspector: dialect.NewInspector(db, options...),
}, nil
}
func NewBunModelInspector(tables *schema.Tables, options ...InspectorOption) *BunModelInspector {
bmi := &BunModelInspector{
tables: tables,
}
ApplyInspectorOptions(&bmi.InspectorConfig, options...)
return bmi
}
// InspectorOption configures an Inspector.
type InspectorOption func(*InspectorConfig)
// ApplyInspectorOptions applies the given options to an InspectorConfig.
func ApplyInspectorOptions(cfg *InspectorConfig, options ...InspectorOption) {
if cfg.ExcludeForeignKeys == nil {
cfg.ExcludeForeignKeys = make(map[ForeignKey]string)
}
for _, opt := range options {
opt(cfg)
}
}
// inspector is opaque pointer to a database inspector.
type inspector struct {
Inspector
}
// BunModelInspector creates the current project state from the passed bun.Models.
// Do not recycle BunModelInspector for different sets of models, as older models will not be de-registerred before the next run.
//
// BunModelInspector does not know which the database's dialect, so it does not
// assume any default schema name. Always specify the target schema name via
// WithSchemaName option to receive meaningful results.
type BunModelInspector struct {
InspectorConfig
tables *schema.Tables
}
var _ Inspector = (*BunModelInspector)(nil)
func (bmi *BunModelInspector) Inspect(ctx context.Context) (Database, error) {
state := BunModelSchema{
BaseDatabase: BaseDatabase{
ForeignKeys: make(map[ForeignKey]string),
},
}
for _, t := range bmi.tables.All() {
if t.Schema != bmi.SchemaName {
continue
}
var columns []Column
for _, f := range t.Fields {
sqlType, length, err := parseLen(f.CreateTableSQLType)
if err != nil {
return nil, fmt.Errorf("parse length in %q: %w", f.CreateTableSQLType, err)
}
columns = append(columns, &BaseColumn{
Name: f.Name,
SQLType: strings.ToLower(sqlType), // TODO(dyma): maybe this is not necessary after Column.Eq()
VarcharLen: length,
DefaultValue: exprOrLiteral(f.SQLDefault),
IsNullable: !f.NotNull,
IsAutoIncrement: f.AutoIncrement,
IsIdentity: f.Identity,
})
}
var unique []Unique
for name, group := range t.Unique {
// Create a separate unique index for single-column unique constraints
// let each dialect apply the default naming convention.
if name == "" {
for _, f := range group {
unique = append(unique, Unique{Columns: NewColumns(f.Name)})
}
continue
}
// Set the name if it is a "unique group", in which case the user has provided the name.
var columns []string
for _, f := range group {
columns = append(columns, f.Name)
}
unique = append(unique, Unique{Name: name, Columns: NewColumns(columns...)})
}
var pk *PrimaryKey
if len(t.PKs) > 0 {
var columns []string
for _, f := range t.PKs {
columns = append(columns, f.Name)
}
pk = &PrimaryKey{Columns: NewColumns(columns...)}
}
// In cases where a table is defined in a non-default schema in the `bun:table` tag,
// schema.Table only extracts the name of the schema, but passes the entire tag value to t.Name
// for backwads-compatibility. For example, a bun model like this:
// type Model struct { bun.BaseModel `bun:"table:favourite.books` }
// produces
// schema.Table{ Schema: "favourite", Name: "favourite.books" }
tableName := strings.TrimPrefix(t.Name, t.Schema+".")
state.Tables = append(state.Tables, &BunTable{
BaseTable: BaseTable{
Schema: t.Schema,
Name: tableName,
Columns: columns,
UniqueConstraints: unique,
PrimaryKey: pk,
},
Model: t.ZeroIface,
})
for _, rel := range t.Relations {
// These relations are nominal and do not need a foreign key to be declared in the current table.
// They will be either expressed as N:1 relations in an m2m mapping table, or will be referenced by the other table if it's a 1:N.
if rel.Type == schema.ManyToManyRelation ||
rel.Type == schema.HasManyRelation {
continue
}
var fromCols, toCols []string
for _, f := range rel.BasePKs {
fromCols = append(fromCols, f.Name)
}
for _, f := range rel.JoinPKs {
toCols = append(toCols, f.Name)
}
target := rel.JoinTable
state.ForeignKeys[ForeignKey{
From: NewColumnReference(t.Name, fromCols...),
To: NewColumnReference(target.Name, toCols...),
}] = ""
}
}
return state, nil
}
func parseLen(typ string) (string, int, error) {
paren := strings.Index(typ, "(")
if paren == -1 {
return typ, 0, nil
}
length, err := strconv.Atoi(typ[paren+1 : len(typ)-1])
if err != nil {
return typ, 0, err
}
return typ[:paren], length, nil
}
// exprOrLiteral converts string to lowercase, if it does not contain a string literal 'lit'
// and trims the surrounding ” otherwise.
// Use it to ensure that user-defined default values in the models are always comparable
// to those returned by the database inspector, regardless of the case convention in individual drivers.
func exprOrLiteral(s string) string {
if strings.HasPrefix(s, "'") && strings.HasSuffix(s, "'") {
return strings.Trim(s, "'")
}
return strings.ToLower(s)
}
// BunModelSchema is the schema state derived from bun table models.
type BunModelSchema struct {
BaseDatabase
Tables []Table
}
func (ms BunModelSchema) GetTables() []Table {
return ms.Tables
}
// BunTable provides additional table metadata that is only accessible from scanning bun models.
type BunTable struct {
BaseTable
// Model stores the zero interface to the underlying Go struct.
Model any
}
+51
View File
@@ -0,0 +1,51 @@
package sqlschema
import (
"fmt"
"github.com/uptrace/bun"
"github.com/uptrace/bun/schema"
)
// MigratorDialect is a Dialect that can create a Migrator for executing schema changes.
type MigratorDialect interface {
schema.Dialect
NewMigrator(db *bun.DB, schemaName string) Migrator
}
// Migrator renders schema-change operations as SQL.
type Migrator interface {
AppendSQL(b []byte, operation any) ([]byte, error)
}
// migrator is a dialect-agnostic wrapper for sqlschema.MigratorDialect.
type migrator struct {
Migrator
}
func NewMigrator(db *bun.DB, schemaName string) (Migrator, error) {
md, ok := db.Dialect().(MigratorDialect)
if !ok {
return nil, fmt.Errorf("%q dialect does not implement sqlschema.Migrator", db.Dialect().Name())
}
return &migrator{
Migrator: md.NewMigrator(db, schemaName),
}, nil
}
// BaseMigrator can be embeded by dialect's Migrator implementations to re-use some of the existing bun queries.
type BaseMigrator struct {
db *bun.DB
}
func NewBaseMigrator(db *bun.DB) *BaseMigrator {
return &BaseMigrator{db: db}
}
func (m *BaseMigrator) AppendCreateTable(b []byte, model any) ([]byte, error) {
return m.db.NewCreateTable().Model(model).AppendQuery(m.db.QueryGen(), b)
}
func (m *BaseMigrator) AppendDropTable(b []byte, schemaName, tableName string) ([]byte, error) {
return m.db.NewDropTable().TableExpr("?.?", bun.Ident(schemaName), bun.Ident(tableName)).AppendQuery(m.db.QueryGen(), b)
}
+56
View File
@@ -0,0 +1,56 @@
package sqlschema
type Table interface {
GetSchema() string
GetName() string
GetColumns() []Column
GetPrimaryKey() *PrimaryKey
GetUniqueConstraints() []Unique
}
var _ Table = (*BaseTable)(nil)
// BaseTable is a base table definition.
//
// Dialects and only dialects can use it to implement the Table interface.
// Other packages must use the Table interface.
type BaseTable struct {
Schema string
Name string
// ColumnDefinitions map each column name to the column definition.
Columns []Column
// PrimaryKey holds the primary key definition.
// A nil value means that no primary key is defined for the table.
PrimaryKey *PrimaryKey
// UniqueConstraints defined on the table.
UniqueConstraints []Unique
}
// PrimaryKey represents a primary key constraint defined on 1 or more columns.
type PrimaryKey struct {
Name string
Columns Columns
}
func (td *BaseTable) GetSchema() string {
return td.Schema
}
func (td *BaseTable) GetName() string {
return td.Name
}
func (td *BaseTable) GetColumns() []Column {
return td.Columns
}
func (td *BaseTable) GetPrimaryKey() *PrimaryKey {
return td.PrimaryKey
}
func (td *BaseTable) GetUniqueConstraints() []Unique {
return td.UniqueConstraints
}
+27
View File
@@ -0,0 +1,27 @@
Copyright 2009 The Go Authors.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google LLC nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+22
View File
@@ -0,0 +1,22 @@
Additional IP Rights Grant (Patents)
"This implementation" means the copyrightable works distributed by
Google as part of the Go project.
Google hereby grants to You a perpetual, worldwide, non-exclusive,
no-charge, royalty-free, irrevocable (except as stated in this section)
patent license to make, have made, use, offer to sell, sell, import,
transfer and otherwise run, modify and propagate the contents of this
implementation of Go, where such license applies only to those patent
claims, both currently owned or controlled by Google and acquired in
the future, licensable by Google that are necessarily infringed by this
implementation of Go. This grant does not include claims that would be
infringed only as a consequence of further modification of this
implementation. If you or your agent or exclusive licensee institute or
order or agree to the institution of patent litigation against any
entity (including a cross-claim or counterclaim in a lawsuit) alleging
that this implementation of Go or any code incorporated within this
implementation of Go constitutes direct or contributory patent
infringement, or inducement of patent infringement, then any patent
rights granted to you under this License for this implementation of Go
shall terminate as of the date such litigation is filed.
+160
View File
@@ -0,0 +1,160 @@
// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package semaphore provides a weighted semaphore implementation.
package semaphore // import "golang.org/x/sync/semaphore"
import (
"container/list"
"context"
"sync"
)
type waiter struct {
n int64
ready chan<- struct{} // Closed when semaphore acquired.
}
// NewWeighted creates a new weighted semaphore with the given
// maximum combined weight for concurrent access.
func NewWeighted(n int64) *Weighted {
w := &Weighted{size: n}
return w
}
// Weighted provides a way to bound concurrent access to a resource.
// The callers can request access with a given weight.
type Weighted struct {
size int64
cur int64
mu sync.Mutex
waiters list.List
}
// Acquire acquires the semaphore with a weight of n, blocking until resources
// are available or ctx is done. On success, returns nil. On failure, returns
// ctx.Err() and leaves the semaphore unchanged.
func (s *Weighted) Acquire(ctx context.Context, n int64) error {
done := ctx.Done()
s.mu.Lock()
select {
case <-done:
// ctx becoming done has "happened before" acquiring the semaphore,
// whether it became done before the call began or while we were
// waiting for the mutex. We prefer to fail even if we could acquire
// the mutex without blocking.
s.mu.Unlock()
return ctx.Err()
default:
}
if s.size-s.cur >= n && s.waiters.Len() == 0 {
// Since we hold s.mu and haven't synchronized since checking done, if
// ctx becomes done before we return here, it becoming done must have
// "happened concurrently" with this call - it cannot "happen before"
// we return in this branch. So, we're ok to always acquire here.
s.cur += n
s.mu.Unlock()
return nil
}
if n > s.size {
// Don't make other Acquire calls block on one that's doomed to fail.
s.mu.Unlock()
<-done
return ctx.Err()
}
ready := make(chan struct{})
w := waiter{n: n, ready: ready}
elem := s.waiters.PushBack(w)
s.mu.Unlock()
select {
case <-done:
s.mu.Lock()
select {
case <-ready:
// Acquired the semaphore after we were canceled.
// Pretend we didn't and put the tokens back.
s.cur -= n
s.notifyWaiters()
default:
isFront := s.waiters.Front() == elem
s.waiters.Remove(elem)
// If we're at the front and there're extra tokens left, notify other waiters.
if isFront && s.size > s.cur {
s.notifyWaiters()
}
}
s.mu.Unlock()
return ctx.Err()
case <-ready:
// Acquired the semaphore. Check that ctx isn't already done.
// We check the done channel instead of calling ctx.Err because we
// already have the channel, and ctx.Err is O(n) with the nesting
// depth of ctx.
select {
case <-done:
s.Release(n)
return ctx.Err()
default:
}
return nil
}
}
// TryAcquire acquires the semaphore with a weight of n without blocking.
// On success, returns true. On failure, returns false and leaves the semaphore unchanged.
func (s *Weighted) TryAcquire(n int64) bool {
s.mu.Lock()
success := s.size-s.cur >= n && s.waiters.Len() == 0
if success {
s.cur += n
}
s.mu.Unlock()
return success
}
// Release releases the semaphore with a weight of n.
func (s *Weighted) Release(n int64) {
s.mu.Lock()
s.cur -= n
if s.cur < 0 {
s.mu.Unlock()
panic("semaphore: released more than held")
}
s.notifyWaiters()
s.mu.Unlock()
}
func (s *Weighted) notifyWaiters() {
for {
next := s.waiters.Front()
if next == nil {
break // No more waiters blocked.
}
w := next.Value.(waiter)
if s.size-s.cur < w.n {
// Not enough tokens for the next waiter. We could keep going (to try to
// find a waiter with a smaller request), but under load that could cause
// starvation for large requests; instead, we leave all remaining waiters
// blocked.
//
// Consider a semaphore used as a read-write lock, with N tokens, N
// readers, and one writer. Each reader can Acquire(1) to obtain a read
// lock. The writer can Acquire(N) to obtain a write lock, excluding all
// of the readers. If we allow the readers to jump ahead in the queue,
// the writer will starve — there is always one token available for every
// reader.
break
}
s.cur += w.n
s.waiters.Remove(next)
close(w.ready)
}
}
+15
View File
@@ -71,6 +71,12 @@ github.com/jackc/pgx/v5/pgconn/ctxwatch
github.com/jackc/pgx/v5/pgconn/internal/bgreader
github.com/jackc/pgx/v5/pgproto3
github.com/jackc/pgx/v5/pgtype
github.com/jackc/pgx/v5/pgxpool
github.com/jackc/pgx/v5/stdlib
# github.com/jackc/puddle/v2 v2.2.2
## explicit; go 1.19
github.com/jackc/puddle/v2
github.com/jackc/puddle/v2/internal/genstack
# github.com/jinzhu/inflection v1.0.0
## explicit
github.com/jinzhu/inflection
@@ -143,9 +149,15 @@ github.com/uptrace/bun/dialect/feature
github.com/uptrace/bun/dialect/sqltype
github.com/uptrace/bun/extra/bunjson
github.com/uptrace/bun/internal
github.com/uptrace/bun/internal/ordered
github.com/uptrace/bun/internal/parser
github.com/uptrace/bun/internal/tagparser
github.com/uptrace/bun/migrate
github.com/uptrace/bun/migrate/sqlschema
github.com/uptrace/bun/schema
# github.com/uptrace/bun/dialect/pgdialect v1.2.18
## explicit; go 1.24.0
github.com/uptrace/bun/dialect/pgdialect
# github.com/vmihailenco/msgpack/v5 v5.4.1
## explicit; go 1.19
github.com/vmihailenco/msgpack/v5
@@ -158,6 +170,9 @@ github.com/vmihailenco/tagparser/v2/internal/parser
# golang.org/x/crypto v0.51.0
## explicit; go 1.25.0
golang.org/x/crypto/md4
# golang.org/x/sync v0.20.0
## explicit; go 1.25.0
golang.org/x/sync/semaphore
# golang.org/x/sys v0.44.0
## explicit; go 1.25.0
golang.org/x/sys/cpu