include the current limits in memory allocation errors

This commit is contained in:
Marten Seemann 2022-06-06 13:30:03 +02:00
parent 069f2c2f6e
commit 6bdbf3adb5

View File

@ -34,6 +34,23 @@ func (e *errStreamOrConnLimitExceeded) AppendLogValues(v []interface{}) []interf
) )
} }
type errMemoryLimitExceeded struct {
current, attempted, limit int64
priority uint8
err error
}
func (e *errMemoryLimitExceeded) Error() string { return e.err.Error() }
func (e *errMemoryLimitExceeded) Unwrap() error { return e.err }
func (e *errMemoryLimitExceeded) AppendLogValues(v []interface{}) []interface{} {
return append(v,
"current", e.current,
"attempted", e.attempted,
"priority", e.priority,
"limit", e.limit,
)
}
// A resourceScope can be a DAG, where a downstream node is not allowed to outlive an upstream node // A resourceScope can be a DAG, where a downstream node is not allowed to outlive an upstream node
// (ie cannot call Done in the upstream node before the downstream node) and account for resources // (ie cannot call Done in the upstream node before the downstream node) and account for resources
// using a linearized parent set. // using a linearized parent set.
@ -95,9 +112,14 @@ func (rc *resources) checkMemory(rsvp int64, prio uint8) error {
threshold := (1 + int64(prio)) * limit / 256 threshold := (1 + int64(prio)) * limit / 256
if newmem > threshold { if newmem > threshold {
return network.ErrResourceLimitExceeded return &errMemoryLimitExceeded{
current: rc.memory,
attempted: rsvp,
limit: limit,
priority: prio,
err: network.ErrResourceLimitExceeded,
}
} }
return nil return nil
} }
@ -307,7 +329,14 @@ func (s *resourceScope) ReserveMemory(size int, prio uint8) error {
} }
if err := s.rc.reserveMemory(int64(size), prio); err != nil { if err := s.rc.reserveMemory(int64(size), prio); err != nil {
log.Debugw("blocked memory reservation", "scope", s.name, "size", size, "priority", prio, "stat", s.rc.stat(), "error", err) logValues := make([]interface{}, 0, 6)
logValues = append(logValues, "scope", s.name)
var limitErr *errMemoryLimitExceeded
if errors.As(err, &limitErr) {
logValues = limitErr.AppendLogValues(logValues)
}
logValues = append(logValues, "error", err)
log.Debugw("blocked memory reservation", logValues...)
s.trace.BlockReserveMemory(s.name, prio, int64(size), s.rc.memory) s.trace.BlockReserveMemory(s.name, prio, int64(size), s.rc.memory)
s.metrics.BlockMemory(size) s.metrics.BlockMemory(size)
return s.wrapError(err) return s.wrapError(err)
@ -333,7 +362,14 @@ func (s *resourceScope) reserveMemoryForEdges(size int, prio uint8) error {
var err error var err error
for _, e := range s.edges { for _, e := range s.edges {
if err = e.ReserveMemoryForChild(int64(size), prio); err != nil { if err = e.ReserveMemoryForChild(int64(size), prio); err != nil {
log.Debugw("blocked memory reservation from constraining edge", "scope", s.name, "edge", e.name, "size", size, "priority", prio, "stat", e.Stat(), "error", err) logValues := make([]interface{}, 0, 6)
logValues = append(logValues, "scope", s.name, "edge", e.name)
var limitErr *errMemoryLimitExceeded
if errors.As(err, &limitErr) {
logValues = limitErr.AppendLogValues(logValues)
}
logValues = append(logValues, "error", err)
log.Debugw("blocked memory reservation from constraining edge", logValues...)
break break
} }