database/sql query 超時設定
阿新 • • 發佈:2019-01-26
參考文章:Query
問題描述
接上文,mysql的同步訪問問題解決後,繼續壓測,發現訪問mysql的耗時逐漸增大,影響模組整體處理能力。我設定了readTimeout,沒有任何反應,請求耗時還是會超過readTimeout所設定的值。因為:
- readTimeout只能限制連線資料讀取時間,如果程式發生在獲取連線前等待時間過長,無法通過這個引數設定,且readTimeout*3才是真正的讀取超時時間,因為會重試3次。
- 當設定的MaxOpenConn消耗殆盡時,再次進入的請求會夯死在database/sql內部,直到有空閒連線可操作。
顯然Query方法是無法滿足我們的需求。
解決方法
翻原始碼,發現了一個新方法QueryContext
,通過傳入context,該context有超時時間,那麼就達到了該查詢請求有了超時設定,nice。
// conn returns a newly-opened or cached *driverConn.
func (db *DB) conn(ctx context.Context, strategy connReuseStrategy) (*driverConn, error) {
db.mu.Lock()
if db.closed {
db.mu.Unlock()
return nil, errDBClosed
}
// Check if the context is expired.
select {
default:
case <-ctx.Done():
db.mu.Unlock()
return nil, ctx.Err()
}
lifetime := db.maxLifetime
// Prefer a free connection, if possible.
numFree := len(db.freeConn)
if strategy == cachedOrNewConn && numFree > 0 {
conn := db.freeConn[0]
copy(db.freeConn, db.freeConn[1:])
db.freeConn = db.freeConn[:numFree-1]
conn.inUse = true
db.mu.Unlock()
if conn.expired(lifetime) {
conn.Close()
return nil, driver.ErrBadConn
}
// Lock around reading lastErr to ensure the session resetter finished.
conn.Lock()
err := conn.lastErr
conn.Unlock()
if err == driver.ErrBadConn {
conn.Close()
return nil, driver.ErrBadConn
}
return conn, nil
}
// Out of free connections or we were asked not to use one. If we're not
// allowed to open any more connections, make a request and wait.
if db.maxOpen > 0 && db.numOpen >= db.maxOpen {
// Make the connRequest channel. It's buffered so that the
// connectionOpener doesn't block while waiting for the req to be read.
req := make(chan connRequest, 1)
reqKey := db.nextRequestKeyLocked()
db.connRequests[reqKey] = req
db.waitCount++
db.mu.Unlock()
waitStart := time.Now()
// Timeout the connection request with the context.
select {
case <-ctx.Done():
// Remove the connection request and ensure no value has been sent
// on it after removing.
db.mu.Lock()
delete(db.connRequests, reqKey)
db.mu.Unlock()
atomic.AddInt64(&db.waitDuration, int64(time.Since(waitStart)))
select {
default:
case ret, ok := <-req:
if ok && ret.conn != nil {
db.putConn(ret.conn, ret.err, false)
}
}
return nil, ctx.Err()
case ret, ok := <-req:
atomic.AddInt64(&db.waitDuration, int64(time.Since(waitStart)))
if !ok {
return nil, errDBClosed
}
if ret.err == nil && ret.conn.expired(lifetime) {
ret.conn.Close()
return nil, driver.ErrBadConn
}
if ret.conn == nil {
return nil, ret.err
}
// Lock around reading lastErr to ensure the session resetter finished.
ret.conn.Lock()
err := ret.conn.lastErr
ret.conn.Unlock()
if err == driver.ErrBadConn {
ret.conn.Close()
return nil, driver.ErrBadConn
}
return ret.conn, ret.err
}
}
db.numOpen++ // optimistically
db.mu.Unlock()
ci, err := db.connector.Connect(ctx)
if err != nil {
db.mu.Lock()
db.numOpen-- // correct for earlier optimism
db.maybeOpenNewConnections()
db.mu.Unlock()
return nil, err
}
db.mu.Lock()
dc := &driverConn{
db: db,
createdAt: nowFunc(),
ci: ci,
inUse: true,
}
db.addDepLocked(dc, dc)
db.mu.Unlock()
return dc, nil
}
使用該訪問進行query查詢
sqlCtx, _ := context.WithTimeout(context.Background(), time.Duration(p.queryTimeout)*time.Millisecond)
//defer cancel()
rows, err := p.orderClient.QueryContext(sqlCtx, sqlStr)
壓測資料
時間單位都是ms
thread_num | mysql_query_timeout | max_conn_num | max_idle_conn | qps | mysql_err_num | mysql_err_max_time | mysql_succ_num | mysql_succ_max_time |
---|---|---|---|---|---|---|---|---|
1000 | 1000 | 0 | 50 | 4000 | 10 | 1500 | 6280 | 317 |
1000 | 1000 | 50 | 50 | 3500 | 135 | 1020 | 5370 | 833 |
1000 | 2000 | 50 | 50 | 3300 | 39 | 2020 | 5040 | 1172 |
400 | 2000 | 50 | 50 | 2800 | 3 | 2015 | 4620 | 640 |
400 | 1000 | 50 | 50 | 2800 | 9 | 1020 | 4646 | 484 |
資料顯示SetMaxOpenConns為0的情況下,設定context是沒有效果的。但是>0後,該timeout設定有效,原因也很好理解,因為maxOpenConn=0時,請求不會等待空閒連線。所以也不會有ctx.Done()邏輯存在。