platform: Add OOM Report & Crash Report
This commit is contained in:
世界
@@ -1,51 +0,0 @@
|
||||
package oomkiller
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/sagernet/sing-box/option"
|
||||
E "github.com/sagernet/sing/common/exceptions"
|
||||
)
|
||||
|
||||
func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64, useAvailable bool) (timerConfig, error) {
|
||||
safetyMargin := uint64(defaultSafetyMargin)
|
||||
if options.SafetyMargin != nil && options.SafetyMargin.Value() > 0 {
|
||||
safetyMargin = options.SafetyMargin.Value()
|
||||
}
|
||||
|
||||
minInterval := defaultMinInterval
|
||||
if options.MinInterval != 0 {
|
||||
minInterval = time.Duration(options.MinInterval.Build())
|
||||
if minInterval <= 0 {
|
||||
return timerConfig{}, E.New("min_interval must be greater than 0")
|
||||
}
|
||||
}
|
||||
|
||||
maxInterval := defaultMaxInterval
|
||||
if options.MaxInterval != 0 {
|
||||
maxInterval = time.Duration(options.MaxInterval.Build())
|
||||
if maxInterval <= 0 {
|
||||
return timerConfig{}, E.New("max_interval must be greater than 0")
|
||||
}
|
||||
}
|
||||
if maxInterval < minInterval {
|
||||
return timerConfig{}, E.New("max_interval must be greater than or equal to min_interval")
|
||||
}
|
||||
|
||||
checksBeforeLimit := defaultChecksBeforeLimit
|
||||
if options.ChecksBeforeLimit != 0 {
|
||||
checksBeforeLimit = options.ChecksBeforeLimit
|
||||
if checksBeforeLimit <= 0 {
|
||||
return timerConfig{}, E.New("checks_before_limit must be greater than 0")
|
||||
}
|
||||
}
|
||||
|
||||
return timerConfig{
|
||||
memoryLimit: memoryLimit,
|
||||
safetyMargin: safetyMargin,
|
||||
minInterval: minInterval,
|
||||
maxInterval: maxInterval,
|
||||
checksBeforeLimit: checksBeforeLimit,
|
||||
useAvailable: useAvailable,
|
||||
}, nil
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
package oomkiller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/sagernet/sing-box/adapter"
|
||||
C "github.com/sagernet/sing-box/constant"
|
||||
"github.com/sagernet/sing-box/option"
|
||||
"github.com/sagernet/sing/common/memory"
|
||||
"github.com/sagernet/sing/service"
|
||||
)
|
||||
|
||||
const DefaultAppleNetworkExtensionMemoryLimit = 50 * 1024 * 1024
|
||||
|
||||
type policyMode uint8
|
||||
|
||||
const (
|
||||
policyModeNone policyMode = iota
|
||||
policyModeMemoryLimit
|
||||
policyModeAvailable
|
||||
policyModeNetworkExtension
|
||||
)
|
||||
|
||||
func (m policyMode) hasTimerMode() bool {
|
||||
return m != policyModeNone
|
||||
}
|
||||
|
||||
func resolvePolicyMode(ctx context.Context, options option.OOMKillerServiceOptions) (uint64, policyMode) {
|
||||
platformInterface := service.FromContext[adapter.PlatformInterface](ctx)
|
||||
if C.IsIos && platformInterface != nil && platformInterface.UnderNetworkExtension() {
|
||||
return DefaultAppleNetworkExtensionMemoryLimit, policyModeNetworkExtension
|
||||
}
|
||||
if options.MemoryLimitOverride > 0 {
|
||||
return options.MemoryLimitOverride, policyModeMemoryLimit
|
||||
}
|
||||
if options.MemoryLimit != nil {
|
||||
memoryLimit := options.MemoryLimit.Value()
|
||||
if memoryLimit > 0 {
|
||||
return memoryLimit, policyModeMemoryLimit
|
||||
}
|
||||
}
|
||||
if memory.AvailableAvailable() {
|
||||
return 0, policyModeAvailable
|
||||
}
|
||||
return 0, policyModeNone
|
||||
}
|
||||
+43
-152
@@ -1,192 +1,83 @@
|
||||
//go:build darwin && cgo
|
||||
|
||||
package oomkiller
|
||||
|
||||
/*
|
||||
#include <dispatch/dispatch.h>
|
||||
|
||||
static dispatch_source_t memoryPressureSource;
|
||||
|
||||
extern void goMemoryPressureCallback(unsigned long status);
|
||||
|
||||
static void startMemoryPressureMonitor() {
|
||||
memoryPressureSource = dispatch_source_create(
|
||||
DISPATCH_SOURCE_TYPE_MEMORYPRESSURE,
|
||||
0,
|
||||
DISPATCH_MEMORYPRESSURE_WARN | DISPATCH_MEMORYPRESSURE_CRITICAL,
|
||||
dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0)
|
||||
);
|
||||
dispatch_source_set_event_handler(memoryPressureSource, ^{
|
||||
unsigned long status = dispatch_source_get_data(memoryPressureSource);
|
||||
goMemoryPressureCallback(status);
|
||||
});
|
||||
dispatch_activate(memoryPressureSource);
|
||||
}
|
||||
|
||||
static void stopMemoryPressureMonitor() {
|
||||
if (memoryPressureSource) {
|
||||
dispatch_source_cancel(memoryPressureSource);
|
||||
memoryPressureSource = NULL;
|
||||
}
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"context"
|
||||
runtimeDebug "runtime/debug"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/sagernet/sing-box/adapter"
|
||||
boxService "github.com/sagernet/sing-box/adapter/service"
|
||||
boxConstant "github.com/sagernet/sing-box/constant"
|
||||
"github.com/sagernet/sing-box/log"
|
||||
"github.com/sagernet/sing-box/option"
|
||||
"github.com/sagernet/sing/common/memory"
|
||||
"github.com/sagernet/sing/service"
|
||||
)
|
||||
|
||||
type OOMReporter interface {
|
||||
WriteReport(memoryUsage uint64) error
|
||||
}
|
||||
|
||||
func RegisterService(registry *boxService.Registry) {
|
||||
boxService.Register[option.OOMKillerServiceOptions](registry, boxConstant.TypeOOMKiller, NewService)
|
||||
}
|
||||
|
||||
var (
|
||||
globalAccess sync.Mutex
|
||||
globalServices []*Service
|
||||
)
|
||||
|
||||
type Service struct {
|
||||
boxService.Adapter
|
||||
logger log.ContextLogger
|
||||
router adapter.Router
|
||||
memoryLimit uint64
|
||||
hasTimerMode bool
|
||||
useAvailable bool
|
||||
timerConfig timerConfig
|
||||
adaptiveTimer *adaptiveTimer
|
||||
ctx context.Context
|
||||
logger log.ContextLogger
|
||||
router adapter.Router
|
||||
timerConfig timerConfig
|
||||
adaptiveTimer *adaptiveTimer
|
||||
lastReportTime atomic.Int64
|
||||
}
|
||||
|
||||
func NewService(ctx context.Context, logger log.ContextLogger, tag string, options option.OOMKillerServiceOptions) (adapter.Service, error) {
|
||||
s := &Service{
|
||||
Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag),
|
||||
logger: logger,
|
||||
router: service.FromContext[adapter.Router](ctx),
|
||||
}
|
||||
|
||||
if options.MemoryLimit != nil {
|
||||
s.memoryLimit = options.MemoryLimit.Value()
|
||||
if s.memoryLimit > 0 {
|
||||
s.hasTimerMode = true
|
||||
}
|
||||
}
|
||||
|
||||
config, err := buildTimerConfig(options, s.memoryLimit, s.useAvailable)
|
||||
memoryLimit, mode := resolvePolicyMode(ctx, options)
|
||||
config, err := buildTimerConfig(options, memoryLimit, mode, options.KillerDisabled)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.timerConfig = config
|
||||
|
||||
return s, nil
|
||||
return &Service{
|
||||
Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag),
|
||||
ctx: ctx,
|
||||
logger: logger,
|
||||
router: service.FromContext[adapter.Router](ctx),
|
||||
timerConfig: config,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (s *Service) Start(stage adapter.StartStage) error {
|
||||
if stage != adapter.StartStateStart {
|
||||
return nil
|
||||
}
|
||||
|
||||
if s.hasTimerMode {
|
||||
s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig)
|
||||
if s.memoryLimit > 0 {
|
||||
s.logger.Info("started memory monitor with limit: ", s.memoryLimit/(1024*1024), " MiB")
|
||||
} else {
|
||||
s.logger.Info("started memory monitor with available memory detection")
|
||||
}
|
||||
} else {
|
||||
s.logger.Info("started memory pressure monitor")
|
||||
}
|
||||
|
||||
globalAccess.Lock()
|
||||
isFirst := len(globalServices) == 0
|
||||
globalServices = append(globalServices, s)
|
||||
globalAccess.Unlock()
|
||||
|
||||
if isFirst {
|
||||
C.startMemoryPressureMonitor()
|
||||
}
|
||||
return nil
|
||||
func (s *Service) createTimer() {
|
||||
s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig, s.writeOOMReport)
|
||||
}
|
||||
|
||||
func (s *Service) Close() error {
|
||||
func (s *Service) startTimer() {
|
||||
s.createTimer()
|
||||
s.adaptiveTimer.start()
|
||||
}
|
||||
|
||||
func (s *Service) stopTimer() {
|
||||
if s.adaptiveTimer != nil {
|
||||
s.adaptiveTimer.stop()
|
||||
}
|
||||
globalAccess.Lock()
|
||||
for i, svc := range globalServices {
|
||||
if svc == s {
|
||||
globalServices = append(globalServices[:i], globalServices[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
isLast := len(globalServices) == 0
|
||||
globalAccess.Unlock()
|
||||
if isLast {
|
||||
C.stopMemoryPressureMonitor()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
//export goMemoryPressureCallback
|
||||
func goMemoryPressureCallback(status C.ulong) {
|
||||
globalAccess.Lock()
|
||||
services := make([]*Service, len(globalServices))
|
||||
copy(services, globalServices)
|
||||
globalAccess.Unlock()
|
||||
if len(services) == 0 {
|
||||
func (s *Service) writeOOMReport(memoryUsage uint64) {
|
||||
now := time.Now().Unix()
|
||||
lastReport := s.lastReportTime.Load()
|
||||
if now-lastReport < 3600 {
|
||||
return
|
||||
}
|
||||
criticalFlag := C.ulong(C.DISPATCH_MEMORYPRESSURE_CRITICAL)
|
||||
warnFlag := C.ulong(C.DISPATCH_MEMORYPRESSURE_WARN)
|
||||
isCritical := status&criticalFlag != 0
|
||||
isWarning := status&warnFlag != 0
|
||||
var level string
|
||||
switch {
|
||||
case isCritical:
|
||||
level = "critical"
|
||||
case isWarning:
|
||||
level = "warning"
|
||||
default:
|
||||
level = "normal"
|
||||
if !s.lastReportTime.CompareAndSwap(lastReport, now) {
|
||||
return
|
||||
}
|
||||
var freeOSMemory bool
|
||||
for _, s := range services {
|
||||
usage := memory.Total()
|
||||
if s.hasTimerMode {
|
||||
if isCritical {
|
||||
s.logger.Warn("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB")
|
||||
if s.adaptiveTimer != nil {
|
||||
s.adaptiveTimer.startNow()
|
||||
}
|
||||
} else if isWarning {
|
||||
s.logger.Warn("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB")
|
||||
} else {
|
||||
s.logger.Debug("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB")
|
||||
if s.adaptiveTimer != nil {
|
||||
s.adaptiveTimer.stop()
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if isCritical {
|
||||
s.logger.Error("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB, resetting network")
|
||||
s.router.ResetNetwork()
|
||||
freeOSMemory = true
|
||||
} else if isWarning {
|
||||
s.logger.Warn("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB")
|
||||
} else {
|
||||
s.logger.Debug("memory pressure: ", level, ", usage: ", usage/(1024*1024), " MiB")
|
||||
}
|
||||
}
|
||||
reporter := service.FromContext[OOMReporter](s.ctx)
|
||||
if reporter == nil {
|
||||
return
|
||||
}
|
||||
if freeOSMemory {
|
||||
runtimeDebug.FreeOSMemory()
|
||||
err := reporter.WriteReport(memoryUsage)
|
||||
if err != nil {
|
||||
s.logger.Warn("failed to write OOM report: ", err)
|
||||
} else {
|
||||
s.logger.Info("OOM report saved")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
//go:build darwin && cgo
|
||||
|
||||
package oomkiller
|
||||
|
||||
/*
|
||||
#include <dispatch/dispatch.h>
|
||||
|
||||
static dispatch_source_t memoryPressureSource;
|
||||
|
||||
extern void goMemoryPressureCallback(unsigned long status);
|
||||
|
||||
static void startMemoryPressureMonitor() {
|
||||
memoryPressureSource = dispatch_source_create(
|
||||
DISPATCH_SOURCE_TYPE_MEMORYPRESSURE,
|
||||
0,
|
||||
DISPATCH_MEMORYPRESSURE_CRITICAL,
|
||||
dispatch_get_global_queue(QOS_CLASS_DEFAULT, 0)
|
||||
);
|
||||
dispatch_source_set_event_handler(memoryPressureSource, ^{
|
||||
unsigned long status = dispatch_source_get_data(memoryPressureSource);
|
||||
goMemoryPressureCallback(status);
|
||||
});
|
||||
dispatch_activate(memoryPressureSource);
|
||||
}
|
||||
|
||||
static void stopMemoryPressureMonitor() {
|
||||
if (memoryPressureSource) {
|
||||
dispatch_source_cancel(memoryPressureSource);
|
||||
memoryPressureSource = NULL;
|
||||
}
|
||||
}
|
||||
*/
|
||||
import "C"
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
"github.com/sagernet/sing-box/adapter"
|
||||
"github.com/sagernet/sing/common/byteformats"
|
||||
E "github.com/sagernet/sing/common/exceptions"
|
||||
)
|
||||
|
||||
var (
|
||||
globalAccess sync.Mutex
|
||||
globalServices []*Service
|
||||
)
|
||||
|
||||
func (s *Service) Start(stage adapter.StartStage) error {
|
||||
if stage != adapter.StartStateStart {
|
||||
return nil
|
||||
}
|
||||
if s.timerConfig.policyMode == policyModeNetworkExtension {
|
||||
s.createTimer()
|
||||
globalAccess.Lock()
|
||||
isFirst := len(globalServices) == 0
|
||||
globalServices = append(globalServices, s)
|
||||
globalAccess.Unlock()
|
||||
if isFirst {
|
||||
C.startMemoryPressureMonitor()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if !s.timerConfig.policyMode.hasTimerMode() {
|
||||
return E.New("memory pressure monitoring is not available on this platform without memory_limit")
|
||||
}
|
||||
s.startTimer()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) Close() error {
|
||||
s.stopTimer()
|
||||
if s.timerConfig.policyMode == policyModeNetworkExtension {
|
||||
globalAccess.Lock()
|
||||
for i, svc := range globalServices {
|
||||
if svc == s {
|
||||
globalServices = append(globalServices[:i], globalServices[i+1:]...)
|
||||
break
|
||||
}
|
||||
}
|
||||
isLast := len(globalServices) == 0
|
||||
globalAccess.Unlock()
|
||||
if isLast {
|
||||
C.stopMemoryPressureMonitor()
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
//export goMemoryPressureCallback
|
||||
func goMemoryPressureCallback(status C.ulong) {
|
||||
globalAccess.Lock()
|
||||
services := make([]*Service, len(globalServices))
|
||||
copy(services, globalServices)
|
||||
globalAccess.Unlock()
|
||||
if len(services) == 0 {
|
||||
return
|
||||
}
|
||||
sample := readMemorySample(policyModeNetworkExtension)
|
||||
for _, s := range services {
|
||||
s.logger.Warn("memory pressure: critical, usage: ", byteformats.FormatMemoryBytes(sample.usage))
|
||||
s.adaptiveTimer.notifyPressure()
|
||||
}
|
||||
}
|
||||
@@ -3,79 +3,22 @@
|
||||
package oomkiller
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/sagernet/sing-box/adapter"
|
||||
boxService "github.com/sagernet/sing-box/adapter/service"
|
||||
boxConstant "github.com/sagernet/sing-box/constant"
|
||||
"github.com/sagernet/sing-box/log"
|
||||
"github.com/sagernet/sing-box/option"
|
||||
E "github.com/sagernet/sing/common/exceptions"
|
||||
"github.com/sagernet/sing/common/memory"
|
||||
"github.com/sagernet/sing/service"
|
||||
)
|
||||
|
||||
func RegisterService(registry *boxService.Registry) {
|
||||
boxService.Register[option.OOMKillerServiceOptions](registry, boxConstant.TypeOOMKiller, NewService)
|
||||
}
|
||||
|
||||
type Service struct {
|
||||
boxService.Adapter
|
||||
logger log.ContextLogger
|
||||
router adapter.Router
|
||||
adaptiveTimer *adaptiveTimer
|
||||
timerConfig timerConfig
|
||||
hasTimerMode bool
|
||||
useAvailable bool
|
||||
memoryLimit uint64
|
||||
}
|
||||
|
||||
func NewService(ctx context.Context, logger log.ContextLogger, tag string, options option.OOMKillerServiceOptions) (adapter.Service, error) {
|
||||
s := &Service{
|
||||
Adapter: boxService.NewAdapter(boxConstant.TypeOOMKiller, tag),
|
||||
logger: logger,
|
||||
router: service.FromContext[adapter.Router](ctx),
|
||||
}
|
||||
|
||||
if options.MemoryLimit != nil {
|
||||
s.memoryLimit = options.MemoryLimit.Value()
|
||||
}
|
||||
if s.memoryLimit > 0 {
|
||||
s.hasTimerMode = true
|
||||
} else if memory.AvailableSupported() {
|
||||
s.useAvailable = true
|
||||
s.hasTimerMode = true
|
||||
}
|
||||
|
||||
config, err := buildTimerConfig(options, s.memoryLimit, s.useAvailable)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.timerConfig = config
|
||||
|
||||
return s, nil
|
||||
}
|
||||
|
||||
func (s *Service) Start(stage adapter.StartStage) error {
|
||||
if stage != adapter.StartStateStart {
|
||||
return nil
|
||||
}
|
||||
if !s.hasTimerMode {
|
||||
if !s.timerConfig.policyMode.hasTimerMode() {
|
||||
return E.New("memory pressure monitoring is not available on this platform without memory_limit")
|
||||
}
|
||||
s.adaptiveTimer = newAdaptiveTimer(s.logger, s.router, s.timerConfig)
|
||||
s.adaptiveTimer.start(0)
|
||||
if s.useAvailable {
|
||||
s.logger.Info("started memory monitor with available memory detection")
|
||||
} else {
|
||||
s.logger.Info("started memory monitor with limit: ", s.memoryLimit/(1024*1024), " MiB")
|
||||
}
|
||||
s.startTimer()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Service) Close() error {
|
||||
if s.adaptiveTimer != nil {
|
||||
s.adaptiveTimer.stop()
|
||||
}
|
||||
s.stopTimer()
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
package oomkiller
|
||||
|
||||
import (
|
||||
runtimeDebug "runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/sagernet/sing-box/adapter"
|
||||
"github.com/sagernet/sing-box/log"
|
||||
"github.com/sagernet/sing/common/memory"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultChecksBeforeLimit = 4
|
||||
defaultMinInterval = 500 * time.Millisecond
|
||||
defaultMaxInterval = 10 * time.Second
|
||||
defaultSafetyMargin = 5 * 1024 * 1024
|
||||
)
|
||||
|
||||
type adaptiveTimer struct {
|
||||
logger log.ContextLogger
|
||||
router adapter.Router
|
||||
memoryLimit uint64
|
||||
safetyMargin uint64
|
||||
minInterval time.Duration
|
||||
maxInterval time.Duration
|
||||
checksBeforeLimit int
|
||||
useAvailable bool
|
||||
|
||||
access sync.Mutex
|
||||
timer *time.Timer
|
||||
previousUsage uint64
|
||||
lastInterval time.Duration
|
||||
}
|
||||
|
||||
type timerConfig struct {
|
||||
memoryLimit uint64
|
||||
safetyMargin uint64
|
||||
minInterval time.Duration
|
||||
maxInterval time.Duration
|
||||
checksBeforeLimit int
|
||||
useAvailable bool
|
||||
}
|
||||
|
||||
func newAdaptiveTimer(logger log.ContextLogger, router adapter.Router, config timerConfig) *adaptiveTimer {
|
||||
return &adaptiveTimer{
|
||||
logger: logger,
|
||||
router: router,
|
||||
memoryLimit: config.memoryLimit,
|
||||
safetyMargin: config.safetyMargin,
|
||||
minInterval: config.minInterval,
|
||||
maxInterval: config.maxInterval,
|
||||
checksBeforeLimit: config.checksBeforeLimit,
|
||||
useAvailable: config.useAvailable,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) start(_ uint64) {
|
||||
t.access.Lock()
|
||||
defer t.access.Unlock()
|
||||
t.startLocked()
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) startNow() {
|
||||
t.access.Lock()
|
||||
t.startLocked()
|
||||
t.access.Unlock()
|
||||
t.poll()
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) startLocked() {
|
||||
if t.timer != nil {
|
||||
return
|
||||
}
|
||||
t.previousUsage = memory.Total()
|
||||
t.lastInterval = t.minInterval
|
||||
t.timer = time.AfterFunc(t.minInterval, t.poll)
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) stop() {
|
||||
t.access.Lock()
|
||||
defer t.access.Unlock()
|
||||
t.stopLocked()
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) stopLocked() {
|
||||
if t.timer != nil {
|
||||
t.timer.Stop()
|
||||
t.timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) running() bool {
|
||||
t.access.Lock()
|
||||
defer t.access.Unlock()
|
||||
return t.timer != nil
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) poll() {
|
||||
t.access.Lock()
|
||||
defer t.access.Unlock()
|
||||
if t.timer == nil {
|
||||
return
|
||||
}
|
||||
|
||||
usage := memory.Total()
|
||||
delta := int64(usage) - int64(t.previousUsage)
|
||||
t.previousUsage = usage
|
||||
|
||||
var remaining uint64
|
||||
var triggered bool
|
||||
|
||||
if t.memoryLimit > 0 {
|
||||
if usage >= t.memoryLimit {
|
||||
remaining = 0
|
||||
triggered = true
|
||||
} else {
|
||||
remaining = t.memoryLimit - usage
|
||||
}
|
||||
} else if t.useAvailable {
|
||||
available := memory.Available()
|
||||
if available <= t.safetyMargin {
|
||||
remaining = 0
|
||||
triggered = true
|
||||
} else {
|
||||
remaining = available - t.safetyMargin
|
||||
}
|
||||
} else {
|
||||
remaining = 0
|
||||
}
|
||||
|
||||
if triggered {
|
||||
t.logger.Error("memory threshold reached, usage: ", usage/(1024*1024), " MiB, resetting network")
|
||||
t.router.ResetNetwork()
|
||||
runtimeDebug.FreeOSMemory()
|
||||
}
|
||||
|
||||
var interval time.Duration
|
||||
if triggered {
|
||||
interval = t.maxInterval
|
||||
} else if delta <= 0 {
|
||||
interval = t.maxInterval
|
||||
} else if t.checksBeforeLimit <= 0 {
|
||||
interval = t.maxInterval
|
||||
} else {
|
||||
timeToLimit := time.Duration(float64(remaining) / float64(delta) * float64(t.lastInterval))
|
||||
interval = timeToLimit / time.Duration(t.checksBeforeLimit)
|
||||
if interval < t.minInterval {
|
||||
interval = t.minInterval
|
||||
}
|
||||
if interval > t.maxInterval {
|
||||
interval = t.maxInterval
|
||||
}
|
||||
}
|
||||
|
||||
t.lastInterval = interval
|
||||
t.timer.Reset(interval)
|
||||
}
|
||||
@@ -0,0 +1,325 @@
|
||||
package oomkiller
|
||||
|
||||
import (
|
||||
runtimeDebug "runtime/debug"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/sagernet/sing-box/adapter"
|
||||
"github.com/sagernet/sing-box/log"
|
||||
"github.com/sagernet/sing-box/option"
|
||||
"github.com/sagernet/sing/common/byteformats"
|
||||
E "github.com/sagernet/sing/common/exceptions"
|
||||
"github.com/sagernet/sing/common/memory"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMinInterval = 100 * time.Millisecond
|
||||
defaultArmedInterval = time.Second
|
||||
defaultMaxInterval = 10 * time.Second
|
||||
defaultSafetyMargin = 5 * 1024 * 1024
|
||||
defaultAvailableTriggerMarginMin = 32 * 1024 * 1024
|
||||
defaultAvailableTriggerMarginMax = 128 * 1024 * 1024
|
||||
)
|
||||
|
||||
type pressureState uint8
|
||||
|
||||
const (
|
||||
pressureStateNormal pressureState = iota
|
||||
pressureStateArmed
|
||||
pressureStateTriggered
|
||||
)
|
||||
|
||||
type memorySample struct {
|
||||
usage uint64
|
||||
available uint64
|
||||
availableKnown bool
|
||||
}
|
||||
|
||||
type pressureThresholds struct {
|
||||
trigger uint64
|
||||
armed uint64
|
||||
resume uint64
|
||||
}
|
||||
|
||||
type timerConfig struct {
|
||||
memoryLimit uint64
|
||||
safetyMargin uint64
|
||||
hasSafetyMargin bool
|
||||
minInterval time.Duration
|
||||
armedInterval time.Duration
|
||||
maxInterval time.Duration
|
||||
policyMode policyMode
|
||||
killerDisabled bool
|
||||
}
|
||||
|
||||
func buildTimerConfig(options option.OOMKillerServiceOptions, memoryLimit uint64, policyMode policyMode, killerDisabled bool) (timerConfig, error) {
|
||||
minInterval := defaultMinInterval
|
||||
if options.MinInterval != 0 {
|
||||
minInterval = time.Duration(options.MinInterval.Build())
|
||||
if minInterval <= 0 {
|
||||
return timerConfig{}, E.New("min_interval must be greater than 0")
|
||||
}
|
||||
}
|
||||
|
||||
maxInterval := defaultMaxInterval
|
||||
if options.MaxInterval != 0 {
|
||||
maxInterval = time.Duration(options.MaxInterval.Build())
|
||||
if maxInterval <= 0 {
|
||||
return timerConfig{}, E.New("max_interval must be greater than 0")
|
||||
}
|
||||
}
|
||||
if maxInterval < minInterval {
|
||||
return timerConfig{}, E.New("max_interval must be greater than or equal to min_interval")
|
||||
}
|
||||
|
||||
var (
|
||||
safetyMargin uint64
|
||||
hasSafetyMargin bool
|
||||
)
|
||||
if options.SafetyMargin != nil && options.SafetyMargin.Value() > 0 {
|
||||
safetyMargin = options.SafetyMargin.Value()
|
||||
hasSafetyMargin = true
|
||||
} else if memoryLimit > 0 {
|
||||
safetyMargin = defaultSafetyMargin
|
||||
hasSafetyMargin = true
|
||||
}
|
||||
|
||||
return timerConfig{
|
||||
memoryLimit: memoryLimit,
|
||||
safetyMargin: safetyMargin,
|
||||
hasSafetyMargin: hasSafetyMargin,
|
||||
minInterval: minInterval,
|
||||
armedInterval: max(min(defaultArmedInterval, maxInterval), minInterval),
|
||||
maxInterval: maxInterval,
|
||||
policyMode: policyMode,
|
||||
killerDisabled: killerDisabled,
|
||||
}, nil
|
||||
}
|
||||
|
||||
type adaptiveTimer struct {
|
||||
timerConfig
|
||||
logger log.ContextLogger
|
||||
router adapter.Router
|
||||
onTriggered func(uint64)
|
||||
limitThresholds pressureThresholds
|
||||
|
||||
access sync.Mutex
|
||||
timer *time.Timer
|
||||
state pressureState
|
||||
forceMinInterval bool
|
||||
pendingPressureBaseline bool
|
||||
pressureBaseline memorySample
|
||||
pressureBaselineTime time.Time
|
||||
}
|
||||
|
||||
func newAdaptiveTimer(logger log.ContextLogger, router adapter.Router, config timerConfig, onTriggered func(uint64)) *adaptiveTimer {
|
||||
t := &adaptiveTimer{
|
||||
timerConfig: config,
|
||||
logger: logger,
|
||||
router: router,
|
||||
onTriggered: onTriggered,
|
||||
}
|
||||
if config.policyMode == policyModeMemoryLimit || config.policyMode == policyModeNetworkExtension {
|
||||
t.limitThresholds = computeLimitThresholds(config.memoryLimit, config.safetyMargin)
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) start() {
|
||||
t.access.Lock()
|
||||
defer t.access.Unlock()
|
||||
t.startLocked()
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) notifyPressure() {
|
||||
t.access.Lock()
|
||||
t.startLocked()
|
||||
t.forceMinInterval = true
|
||||
t.pendingPressureBaseline = true
|
||||
t.access.Unlock()
|
||||
t.poll()
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) startLocked() {
|
||||
if t.timer != nil {
|
||||
return
|
||||
}
|
||||
t.state = pressureStateNormal
|
||||
t.forceMinInterval = false
|
||||
t.timer = time.AfterFunc(t.minInterval, t.poll)
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) stop() {
|
||||
t.access.Lock()
|
||||
defer t.access.Unlock()
|
||||
if t.timer != nil {
|
||||
t.timer.Stop()
|
||||
t.timer = nil
|
||||
}
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) poll() {
|
||||
var triggered bool
|
||||
var rateTriggered bool
|
||||
sample := readMemorySample(t.policyMode)
|
||||
|
||||
t.access.Lock()
|
||||
if t.timer == nil {
|
||||
t.access.Unlock()
|
||||
return
|
||||
}
|
||||
if t.pendingPressureBaseline {
|
||||
t.pressureBaseline = sample
|
||||
t.pressureBaselineTime = time.Now()
|
||||
t.pendingPressureBaseline = false
|
||||
}
|
||||
previousState := t.state
|
||||
t.state = t.nextState(sample)
|
||||
if t.state == pressureStateNormal {
|
||||
t.forceMinInterval = false
|
||||
t.pressureBaselineTime = time.Time{}
|
||||
}
|
||||
t.timer.Reset(t.intervalForState())
|
||||
triggered = previousState != pressureStateTriggered && t.state == pressureStateTriggered
|
||||
if !triggered && !t.pressureBaselineTime.IsZero() && t.memoryLimit > 0 &&
|
||||
sample.usage > t.pressureBaseline.usage && sample.usage < t.memoryLimit {
|
||||
elapsed := time.Since(t.pressureBaselineTime)
|
||||
if elapsed >= t.minInterval/2 {
|
||||
growth := sample.usage - t.pressureBaseline.usage
|
||||
ratePerSecond := float64(growth) / elapsed.Seconds()
|
||||
headroom := t.memoryLimit - sample.usage
|
||||
timeToLimit := time.Duration(float64(headroom)/ratePerSecond) * time.Second
|
||||
if timeToLimit < t.minInterval {
|
||||
triggered = true
|
||||
rateTriggered = true
|
||||
t.state = pressureStateTriggered
|
||||
}
|
||||
}
|
||||
}
|
||||
t.access.Unlock()
|
||||
|
||||
if !triggered {
|
||||
return
|
||||
}
|
||||
if rateTriggered {
|
||||
if t.killerDisabled {
|
||||
t.logger.Warn("memory growth rate critical (report only), usage: ", byteformats.FormatMemoryBytes(sample.usage), t.logDetails(sample))
|
||||
} else {
|
||||
t.logger.Error("memory growth rate critical, usage: ", byteformats.FormatMemoryBytes(sample.usage), t.logDetails(sample), ", resetting network")
|
||||
t.router.ResetNetwork()
|
||||
}
|
||||
} else {
|
||||
if t.killerDisabled {
|
||||
t.logger.Warn("memory threshold reached (report only), usage: ", byteformats.FormatMemoryBytes(sample.usage), t.logDetails(sample))
|
||||
} else {
|
||||
t.logger.Error("memory threshold reached, usage: ", byteformats.FormatMemoryBytes(sample.usage), t.logDetails(sample), ", resetting network")
|
||||
t.router.ResetNetwork()
|
||||
}
|
||||
}
|
||||
t.onTriggered(sample.usage)
|
||||
runtimeDebug.FreeOSMemory()
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) nextState(sample memorySample) pressureState {
|
||||
switch t.policyMode {
|
||||
case policyModeMemoryLimit, policyModeNetworkExtension:
|
||||
return nextPressureState(t.state,
|
||||
sample.usage >= t.limitThresholds.trigger,
|
||||
sample.usage >= t.limitThresholds.armed,
|
||||
sample.usage >= t.limitThresholds.resume,
|
||||
)
|
||||
case policyModeAvailable:
|
||||
if !sample.availableKnown {
|
||||
return pressureStateNormal
|
||||
}
|
||||
thresholds := t.availableThresholds(sample)
|
||||
return nextPressureState(t.state,
|
||||
sample.available <= thresholds.trigger,
|
||||
sample.available <= thresholds.armed,
|
||||
sample.available <= thresholds.resume,
|
||||
)
|
||||
default:
|
||||
return pressureStateNormal
|
||||
}
|
||||
}
|
||||
|
||||
func computeLimitThresholds(memoryLimit uint64, safetyMargin uint64) pressureThresholds {
|
||||
triggerMargin := min(safetyMargin, memoryLimit)
|
||||
armedMargin := min(triggerMargin*2, memoryLimit)
|
||||
resumeMargin := min(triggerMargin*4, memoryLimit)
|
||||
return pressureThresholds{
|
||||
trigger: memoryLimit - triggerMargin,
|
||||
armed: memoryLimit - armedMargin,
|
||||
resume: memoryLimit - resumeMargin,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) availableThresholds(sample memorySample) pressureThresholds {
|
||||
var triggerMargin uint64
|
||||
if t.hasSafetyMargin {
|
||||
triggerMargin = t.safetyMargin
|
||||
} else if sample.usage == 0 {
|
||||
triggerMargin = defaultAvailableTriggerMarginMin
|
||||
} else {
|
||||
triggerMargin = max(defaultAvailableTriggerMarginMin, min(sample.usage/4, defaultAvailableTriggerMarginMax))
|
||||
}
|
||||
return pressureThresholds{
|
||||
trigger: triggerMargin,
|
||||
armed: triggerMargin * 2,
|
||||
resume: triggerMargin * 4,
|
||||
}
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) intervalForState() time.Duration {
|
||||
if t.state == pressureStateNormal {
|
||||
return t.maxInterval
|
||||
}
|
||||
if t.forceMinInterval || t.state == pressureStateTriggered {
|
||||
return t.minInterval
|
||||
}
|
||||
return t.armedInterval
|
||||
}
|
||||
|
||||
func (t *adaptiveTimer) logDetails(sample memorySample) string {
|
||||
switch t.policyMode {
|
||||
case policyModeMemoryLimit, policyModeNetworkExtension:
|
||||
headroom := uint64(0)
|
||||
if sample.usage < t.memoryLimit {
|
||||
headroom = t.memoryLimit - sample.usage
|
||||
}
|
||||
return ", limit: " + byteformats.FormatMemoryBytes(t.memoryLimit) + ", headroom: " + byteformats.FormatMemoryBytes(headroom)
|
||||
case policyModeAvailable:
|
||||
if sample.availableKnown {
|
||||
return ", available: " + byteformats.FormatMemoryBytes(sample.available)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func nextPressureState(current pressureState, shouldTrigger, shouldArm, shouldStayTriggered bool) pressureState {
|
||||
if current == pressureStateTriggered {
|
||||
if shouldStayTriggered {
|
||||
return pressureStateTriggered
|
||||
}
|
||||
return pressureStateNormal
|
||||
}
|
||||
if shouldTrigger {
|
||||
return pressureStateTriggered
|
||||
}
|
||||
if shouldArm {
|
||||
return pressureStateArmed
|
||||
}
|
||||
return pressureStateNormal
|
||||
}
|
||||
|
||||
func readMemorySample(mode policyMode) memorySample {
|
||||
sample := memorySample{
|
||||
usage: memory.Total(),
|
||||
}
|
||||
if mode == policyModeAvailable {
|
||||
sample.availableKnown = true
|
||||
sample.available = memory.Available()
|
||||
}
|
||||
return sample
|
||||
}
|
||||
Reference in New Issue
Block a user