Refactor: Optimize Memory Usage At Startup

https://github.com/XTLS/Xray-core/issues/68#issuecomment-745231528
This commit is contained in:
RPRX 2020-12-15 20:27:18 +08:00 committed by GitHub
parent 2e942e0303
commit 45f44c401a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 101 additions and 30 deletions

View File

@ -2,6 +2,7 @@ package conf
import ( import (
"encoding/json" "encoding/json"
"runtime"
"strconv" "strconv"
"strings" "strings"
@ -147,46 +148,109 @@ func ParseIP(s string) (*router.CIDR, error) {
} }
} }
func loadGeoIP(country string) ([]*router.CIDR, error) { func loadGeoIP(code string) ([]*router.CIDR, error) {
return loadIP("geoip.dat", country) return loadIP("geoip.dat", code)
} }
func loadIP(filename, country string) ([]*router.CIDR, error) { var (
geoipBytes, err := filesystem.ReadAsset(filename) FileCache = make(map[string][]byte)
IPCache = make(map[string]*router.GeoIP)
SiteCache = make(map[string]*router.GeoSite)
)
func loadFile(file string) ([]byte, error) {
if FileCache[file] == nil {
bs, err := filesystem.ReadAsset(file)
if err != nil { if err != nil {
return nil, newError("failed to open file: ", filename).Base(err) return nil, newError("failed to open file: ", file).Base(err)
} }
var geoipList router.GeoIPList if len(bs) == 0 {
if err := proto.Unmarshal(geoipBytes, &geoipList); err != nil { return nil, newError("empty file: ", file)
return nil, err }
// Do not cache file, may save RAM when there
// are many files, but consume CPU each time.
return bs, nil
FileCache[file] = bs
}
return FileCache[file], nil
} }
for _, geoip := range geoipList.Entry { func loadIP(file, code string) ([]*router.CIDR, error) {
if geoip.CountryCode == country { index := file + ":" + code
return geoip.Cidr, nil if IPCache[index] == nil {
} bs, err := loadFile(file)
}
return nil, newError("country not found in ", filename, ": ", country)
}
func loadSite(filename, country string) ([]*router.Domain, error) {
geositeBytes, err := filesystem.ReadAsset(filename)
if err != nil { if err != nil {
return nil, newError("failed to open file: ", filename).Base(err) return nil, newError("failed to load file: ", file).Base(err)
} }
var geositeList router.GeoSiteList bs = find(bs, []byte(code))
if err := proto.Unmarshal(geositeBytes, &geositeList); err != nil { if bs == nil {
return nil, err return nil, newError("code not found in ", file, ": ", code)
}
var geoip router.GeoIP
if err := proto.Unmarshal(bs, &geoip); err != nil {
return nil, newError("error unmarshal IP in ", file, ": ", code).Base(err)
}
defer runtime.GC() // or debug.FreeOSMemory()
return geoip.Cidr, nil // do not cache geoip
IPCache[index] = &geoip
}
return IPCache[index].Cidr, nil
} }
for _, site := range geositeList.Entry { func loadSite(file, code string) ([]*router.Domain, error) {
if site.CountryCode == country { index := file + ":" + code
return site.Domain, nil if SiteCache[index] == nil {
bs, err := loadFile(file)
if err != nil {
return nil, newError("failed to load file: ", file).Base(err)
} }
bs = find(bs, []byte(code))
if bs == nil {
return nil, newError("list not found in ", file, ": ", code)
}
var geosite router.GeoSite
if err := proto.Unmarshal(bs, &geosite); err != nil {
return nil, newError("error unmarshal Site in ", file, ": ", code).Base(err)
}
defer runtime.GC() // or debug.FreeOSMemory()
return geosite.Domain, nil // do not cache geosite
SiteCache[index] = &geosite
}
return SiteCache[index].Domain, nil
} }
return nil, newError("list not found in ", filename, ": ", country) func find(data, code []byte) []byte {
codeL := len(code)
if codeL == 0 {
return nil
}
for {
dataL := len(data)
if dataL < 2 {
return nil
}
x, y := proto.DecodeVarint(data[1:])
if x == 0 && y == 0 {
return nil
}
headL, bodyL := 1+y, int(x)
dataL -= headL
if dataL < bodyL {
return nil
}
data = data[headL:]
if int(data[1]) == codeL {
for i := 0; i < codeL && data[2+i] == code[i]; i++ {
if i+1 == codeL {
return data[:bodyL]
}
}
}
if dataL == bodyL {
return nil
}
data = data[bodyL:]
}
} }
type AttributeMatcher interface { type AttributeMatcher interface {

View File

@ -9,12 +9,14 @@ import (
"path" "path"
"path/filepath" "path/filepath"
"runtime" "runtime"
"runtime/debug"
"strings" "strings"
"syscall" "syscall"
"github.com/xtls/xray-core/common/cmdarg" "github.com/xtls/xray-core/common/cmdarg"
"github.com/xtls/xray-core/common/platform" "github.com/xtls/xray-core/common/platform"
"github.com/xtls/xray-core/core" "github.com/xtls/xray-core/core"
"github.com/xtls/xray-core/infra/conf"
"github.com/xtls/xray-core/main/commands/base" "github.com/xtls/xray-core/main/commands/base"
) )
@ -78,8 +80,13 @@ func executeRun(cmd *base.Command, args []string) {
} }
defer server.Close() defer server.Close()
conf.FileCache = nil
conf.IPCache = nil
conf.SiteCache = nil
// Explicitly triggering GC to remove garbage from config loading. // Explicitly triggering GC to remove garbage from config loading.
runtime.GC() runtime.GC()
debug.FreeOSMemory()
{ {
osSignals := make(chan os.Signal, 1) osSignals := make(chan os.Signal, 1)