mirrors/Xray-core
1
0
Fork 0
mirror of https://github.com/XTLS/Xray-core.git synced 2025-05-05 11:48:41 +00:00
Code Issues Projects Releases Packages Wiki Activity

Refactor: GeoSite & GeoIP

Browse source
This commit is contained in:
JimhHan 2021-03-24 23:01:20 +08:00
parent 8382b29922
commit b11429eaee
No known key found for this signature in database
GPG key ID: 48D5D7CF95157AC5
54 changed files with 2110 additions and 1633 deletions
Download patch file Download diff file Expand all files Collapse all files

56
infra/conf/dns.go
View file

@ -6,7 +6,9 @@ import (
"strings"
"github.com/xtls/xray-core/app/dns"
"github.com/xtls/xray-core/app/router"
"github.com/xtls/xray-core/common/matcher/conf"
dm "github.com/xtls/xray-core/common/matcher/domain"
"github.com/xtls/xray-core/common/matcher/geosite"
"github.com/xtls/xray-core/common/net"
)
@ -41,39 +43,24 @@ func (c *NameServerConfig) UnmarshalJSON(data []byte) error {
return newError("failed to parse name server: ", string(data))
}
func toDomainMatchingType(t router.Domain_Type) dns.DomainMatchingType {
switch t {
case router.Domain_Domain:
return dns.DomainMatchingType_Subdomain
case router.Domain_Full:
return dns.DomainMatchingType_Full
case router.Domain_Plain:
return dns.DomainMatchingType_Keyword
case router.Domain_Regex:
return dns.DomainMatchingType_Regex
default:
panic("unknown domain type")
}
}
func (c *NameServerConfig) Build() (*dns.NameServer, error) {
if c.Address == nil {
return nil, newError("NameServer address is not specified.")
}
var domains []*dns.NameServer_PriorityDomain
var domains []*dm.Domain
var originalRules []*dns.NameServer_OriginalRule
for _, rule := range c.Domains {
parsedDomain, err := parseDomainRule(rule)
parsedDomain, err := conf.ParaseDomainRule(rule)
if err != nil {
return nil, newError("invalid domain rule: ", rule).Base(err)
}
for _, pd := range parsedDomain {
domains = append(domains, &dns.NameServer_PriorityDomain{
Type: toDomainMatchingType(pd.Type),
Domain: pd.Value,
domains = append(domains, &dm.Domain{
Type: pd.Type,
Value: pd.Value,
})
}
originalRules = append(originalRules, &dns.NameServer_OriginalRule{
@ -99,13 +86,6 @@ func (c *NameServerConfig) Build() (*dns.NameServer, error) {
}, nil
}
var typeMap = map[router.Domain_Type]dns.DomainMatchingType{
router.Domain_Full: dns.DomainMatchingType_Full,
router.Domain_Domain: dns.DomainMatchingType_Subdomain,
router.Domain_Plain: dns.DomainMatchingType_Keyword,
router.Domain_Regex: dns.DomainMatchingType_Regex,
}
// DNSConfig is a JSON serializable object for dns.Config.
type DNSConfig struct {
Servers []*NameServerConfig `json:"servers"`
@ -177,7 +157,7 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
return nil, newError("empty domain type of rule: ", domain)
}
mapping := getHostMapping(addr)
mapping.Type = dns.DomainMatchingType_Subdomain
mapping.Type = dm.MatchingType_Subdomain
mapping.Domain = domainName
mappings = append(mappings, mapping)
@ -186,13 +166,13 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
if len(listName) == 0 {
return nil, newError("empty geosite rule: ", domain)
}
domains, err := loadGeositeWithAttr("geosite.dat", listName)
domains, err := geosite.LoadGeositeWithAttr("geosite.dat", listName)
if err != nil {
return nil, newError("failed to load geosite: ", listName).Base(err)
}
for _, d := range domains {
mapping := getHostMapping(addr)
mapping.Type = typeMap[d.Type]
mapping.Type = d.Type
mapping.Domain = d.Value
mappings = append(mappings, mapping)
}
@ -203,7 +183,7 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
return nil, newError("empty regexp type of rule: ", domain)
}
mapping := getHostMapping(addr)
mapping.Type = dns.DomainMatchingType_Regex
mapping.Type = dm.MatchingType_Regex
mapping.Domain = regexpVal
mappings = append(mappings, mapping)
@ -213,7 +193,7 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
return nil, newError("empty keyword type of rule: ", domain)
}
mapping := getHostMapping(addr)
mapping.Type = dns.DomainMatchingType_Keyword
mapping.Type = dm.MatchingType_Keyword
mapping.Domain = keywordVal
mappings = append(mappings, mapping)
@ -223,13 +203,13 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
return nil, newError("empty full domain type of rule: ", domain)
}
mapping := getHostMapping(addr)
mapping.Type = dns.DomainMatchingType_Full
mapping.Type = dm.MatchingType_Full
mapping.Domain = fullVal
mappings = append(mappings, mapping)
case strings.HasPrefix(domain, "dotless:"):
mapping := getHostMapping(addr)
mapping.Type = dns.DomainMatchingType_Regex
mapping.Type = dm.MatchingType_Regex
switch substr := domain[8:]; {
case substr == "":
mapping.Domain = "^[^.]*$"
@ -247,20 +227,20 @@ func (c *DNSConfig) Build() (*dns.Config, error) {
}
filename := kv[0]
list := kv[1]
domains, err := loadGeositeWithAttr(filename, list)
domains, err := geosite.LoadGeositeWithAttr(filename, list)
if err != nil {
return nil, newError("failed to load domain list: ", list, " from ", filename).Base(err)
}
for _, d := range domains {
mapping := getHostMapping(addr)
mapping.Type = typeMap[d.Type]
mapping.Type = d.Type
mapping.Domain = d.Value
mappings = append(mappings, mapping)
}
default:
mapping := getHostMapping(addr)
mapping.Type = dns.DomainMatchingType_Full
mapping.Type = dm.MatchingType_Full
mapping.Domain = domain
mappings = append(mappings, mapping)
}

27
infra/conf/dns_test.go
View file

@ -8,8 +8,9 @@ import (
"github.com/golang/protobuf/proto"
"github.com/xtls/xray-core/app/dns"
"github.com/xtls/xray-core/app/router"
"github.com/xtls/xray-core/common"
"github.com/xtls/xray-core/common/matcher/domain"
"github.com/xtls/xray-core/common/matcher/geosite"
"github.com/xtls/xray-core/common/net"
"github.com/xtls/xray-core/common/platform"
"github.com/xtls/xray-core/common/platform/filesystem"
@ -30,12 +31,12 @@ func init() {
common.Must(err)
defer geositeFile.Close()
list := &router.GeoSiteList{
Entry: []*router.GeoSite{
list := &geosite.GeoSiteList{
Entry: []*geosite.GeoSite{
{
CountryCode: "TEST",
Domain: []*router.Domain{
{Type: router.Domain_Full, Value: "example.com"},
Domain: []*geosite.Domain{
{Type: domain.MatchingType_Full, Value: "example.com"},
},
},
},
@ -94,10 +95,10 @@ func TestDNSConfigParsing(t *testing.T) {
Network: net.Network_UDP,
Port: 5353,
},
PrioritizedDomain: []*dns.NameServer_PriorityDomain{
PrioritizedDomain: []*domain.Domain{
{
Type: dns.DomainMatchingType_Subdomain,
Domain: "example.com",
Type: domain.MatchingType_Subdomain,
Value: "example.com",
},
},
OriginalRules: []*dns.NameServer_OriginalRule{
@ -110,27 +111,27 @@ func TestDNSConfigParsing(t *testing.T) {
},
StaticHosts: []*dns.Config_HostMapping{
{
Type: dns.DomainMatchingType_Subdomain,
Type: domain.MatchingType_Subdomain,
Domain: "example.com",
ProxiedDomain: "google.com",
},
{
Type: dns.DomainMatchingType_Full,
Type: domain.MatchingType_Full,
Domain: "example.com",
Ip: [][]byte{{127, 0, 0, 1}},
},
{
Type: dns.DomainMatchingType_Full,
Type: domain.MatchingType_Full,
Domain: "example.com",
Ip: [][]byte{{10, 0, 0, 1}},
},
{
Type: dns.DomainMatchingType_Keyword,
Type: domain.MatchingType_Keyword,
Domain: "google",
Ip: [][]byte{{8, 8, 8, 8}},
},
{
Type: dns.DomainMatchingType_Regex,
Type: domain.MatchingType_Regex,
Domain: ".*\\.com",
Ip: [][]byte{{8, 8, 4, 4}},
},

264
infra/conf/router.go
View file

@ -2,13 +2,13 @@ package conf
import (
"encoding/json"
"runtime"
"strconv"
"strings"
"github.com/golang/protobuf/proto"
"github.com/xtls/xray-core/app/router"
"github.com/xtls/xray-core/common/matcher/conf"
"github.com/xtls/xray-core/common/matcher/geoip"
"github.com/xtls/xray-core/common/matcher/geosite"
"github.com/xtls/xray-core/common/net"
"github.com/xtls/xray-core/common/platform/filesystem"
)
@ -100,7 +100,7 @@ type RouterRule struct {
BalancerTag string `json:"balancerTag"`
}
func ParseIP(s string) (*router.CIDR, error) {
func ParseIP(s string) (*geoip.CIDR, error) {
var addr, mask string
i := strings.Index(s, "/")
if i < 0 {
@ -123,7 +123,7 @@ func ParseIP(s string) (*router.CIDR, error) {
if bits > 32 {
return nil, newError("invalid network mask for router: ", bits)
}
return &router.CIDR{
return &geoip.CIDR{
Ip: []byte(ip.IP()),
Prefix: bits,
}, nil
@ -139,8 +139,8 @@ func ParseIP(s string) (*router.CIDR, error) {
if bits > 128 {
return nil, newError("invalid network mask for router: ", bits)
}
return &router.CIDR{
Ip: []byte(ip.IP()),
return &geoip.CIDR{
Ip: ip.IP(),
Prefix: bits,
}, nil
default:
@ -148,14 +148,9 @@ func ParseIP(s string) (*router.CIDR, error) {
}
}
func loadGeoIP(code string) ([]*router.CIDR, error) {
return loadIP("geoip.dat", code)
}
var (
FileCache = make(map[string][]byte)
IPCache = make(map[string]*router.GeoIP)
SiteCache = make(map[string]*router.GeoSite)
IPCache = make(map[string]*geoip.GeoIP)
)
func loadFile(file string) ([]byte, error) {
@ -175,236 +170,21 @@ func loadFile(file string) ([]byte, error) {
return FileCache[file], nil
}
func loadIP(file, code string) ([]*router.CIDR, error) {
index := file + ":" + code
if IPCache[index] == nil {
bs, err := loadFile(file)
if err != nil {
return nil, newError("failed to load file: ", file).Base(err)
}
bs = find(bs, []byte(code))
if bs == nil {
return nil, newError("code not found in ", file, ": ", code)
}
var geoip router.GeoIP
if err := proto.Unmarshal(bs, &geoip); err != nil {
return nil, newError("error unmarshal IP in ", file, ": ", code).Base(err)
}
defer runtime.GC() // or debug.FreeOSMemory()
return geoip.Cidr, nil // do not cache geoip
IPCache[index] = &geoip
}
return IPCache[index].Cidr, nil
}
func loadSite(file, code string) ([]*router.Domain, error) {
index := file + ":" + code
if SiteCache[index] == nil {
bs, err := loadFile(file)
if err != nil {
return nil, newError("failed to load file: ", file).Base(err)
}
bs = find(bs, []byte(code))
if bs == nil {
return nil, newError("list not found in ", file, ": ", code)
}
var geosite router.GeoSite
if err := proto.Unmarshal(bs, &geosite); err != nil {
return nil, newError("error unmarshal Site in ", file, ": ", code).Base(err)
}
defer runtime.GC() // or debug.FreeOSMemory()
return geosite.Domain, nil // do not cache geosite
SiteCache[index] = &geosite
}
return SiteCache[index].Domain, nil
}
func find(data, code []byte) []byte {
codeL := len(code)
if codeL == 0 {
return nil
}
for {
dataL := len(data)
if dataL < 2 {
return nil
}
x, y := proto.DecodeVarint(data[1:])
if x == 0 && y == 0 {
return nil
}
headL, bodyL := 1+y, int(x)
dataL -= headL
if dataL < bodyL {
return nil
}
data = data[headL:]
if int(data[1]) == codeL {
for i := 0; i < codeL && data[2+i] == code[i]; i++ {
if i+1 == codeL {
return data[:bodyL]
}
}
}
if dataL == bodyL {
return nil
}
data = data[bodyL:]
}
}
type AttributeMatcher interface {
Match(*router.Domain) bool
}
type BooleanMatcher string
func (m BooleanMatcher) Match(domain *router.Domain) bool {
for _, attr := range domain.Attribute {
if attr.Key == string(m) {
return true
}
}
return false
}
type AttributeList struct {
matcher []AttributeMatcher
}
func (al *AttributeList) Match(domain *router.Domain) bool {
for _, matcher := range al.matcher {
if !matcher.Match(domain) {
return false
}
}
return true
}
func (al *AttributeList) IsEmpty() bool {
return len(al.matcher) == 0
}
func parseAttrs(attrs []string) *AttributeList {
al := new(AttributeList)
for _, attr := range attrs {
lc := strings.ToLower(attr)
al.matcher = append(al.matcher, BooleanMatcher(lc))
}
return al
}
func loadGeositeWithAttr(file string, siteWithAttr string) ([]*router.Domain, error) {
parts := strings.Split(siteWithAttr, "@")
if len(parts) == 0 {
return nil, newError("empty site")
}
country := strings.ToUpper(parts[0])
attrs := parseAttrs(parts[1:])
domains, err := loadSite(file, country)
if err != nil {
return nil, err
}
if attrs.IsEmpty() {
return domains, nil
}
filteredDomains := make([]*router.Domain, 0, len(domains))
for _, domain := range domains {
if attrs.Match(domain) {
filteredDomains = append(filteredDomains, domain)
}
}
return filteredDomains, nil
}
func parseDomainRule(domain string) ([]*router.Domain, error) {
if strings.HasPrefix(domain, "geosite:") {
country := strings.ToUpper(domain[8:])
domains, err := loadGeositeWithAttr("geosite.dat", country)
if err != nil {
return nil, newError("failed to load geosite: ", country).Base(err)
}
return domains, nil
}
var isExtDatFile = 0
{
const prefix = "ext:"
if strings.HasPrefix(domain, prefix) {
isExtDatFile = len(prefix)
}
const prefixQualified = "ext-domain:"
if strings.HasPrefix(domain, prefixQualified) {
isExtDatFile = len(prefixQualified)
}
}
if isExtDatFile != 0 {
kv := strings.Split(domain[isExtDatFile:], ":")
if len(kv) != 2 {
return nil, newError("invalid external resource: ", domain)
}
filename := kv[0]
country := kv[1]
domains, err := loadGeositeWithAttr(filename, country)
if err != nil {
return nil, newError("failed to load external sites: ", country, " from ", filename).Base(err)
}
return domains, nil
}
domainRule := new(router.Domain)
switch {
case strings.HasPrefix(domain, "regexp:"):
domainRule.Type = router.Domain_Regex
domainRule.Value = domain[7:]
case strings.HasPrefix(domain, "domain:"):
domainRule.Type = router.Domain_Domain
domainRule.Value = domain[7:]
case strings.HasPrefix(domain, "full:"):
domainRule.Type = router.Domain_Full
domainRule.Value = domain[5:]
case strings.HasPrefix(domain, "keyword:"):
domainRule.Type = router.Domain_Plain
domainRule.Value = domain[8:]
case strings.HasPrefix(domain, "dotless:"):
domainRule.Type = router.Domain_Regex
switch substr := domain[8:]; {
case substr == "":
domainRule.Value = "^[^.]*$"
case !strings.Contains(substr, "."):
domainRule.Value = "^[^.]*" + substr + "[^.]*$"
default:
return nil, newError("substr in dotless rule should not contain a dot: ", substr)
}
default:
domainRule.Type = router.Domain_Plain
domainRule.Value = domain
}
return []*router.Domain{domainRule}, nil
}
func toCidrList(ips StringList) ([]*router.GeoIP, error) {
var geoipList []*router.GeoIP
var customCidrs []*router.CIDR
func toCidrList(ips StringList) ([]*geoip.GeoIP, error) {
var geoipList []*geoip.GeoIP
var customCidrs []*geoip.CIDR
for _, ip := range ips {
if strings.HasPrefix(ip, "geoip:") {
country := ip[6:]
geoip, err := loadGeoIP(strings.ToUpper(country))
geoipc, err := geoip.LoadGeoIP(strings.ToUpper(country))
if err != nil {
return nil, newError("failed to load GeoIP: ", country).Base(err)
}
geoipList = append(geoipList, &router.GeoIP{
geoipList = append(geoipList, &geoip.GeoIP{
CountryCode: strings.ToUpper(country),
Cidr: geoip,
Cidr: geoipc,
})
continue
}
@ -427,14 +207,14 @@ func toCidrList(ips StringList) ([]*router.GeoIP, error) {
filename := kv[0]
country := kv[1]
geoip, err := loadIP(filename, strings.ToUpper(country))
geoipc, err := geoip.LoadIPFile(filename, strings.ToUpper(country))
if err != nil {
return nil, newError("failed to load IPs: ", country, " from ", filename).Base(err)
}
geoipList = append(geoipList, &router.GeoIP{
geoipList = append(geoipList, &geoip.GeoIP{
CountryCode: strings.ToUpper(filename + "_" + country),
Cidr: geoip,
Cidr: geoipc,
})
continue
@ -448,7 +228,7 @@ func toCidrList(ips StringList) ([]*router.GeoIP, error) {
}
if len(customCidrs) > 0 {
geoipList = append(geoipList, &router.GeoIP{
geoipList = append(geoipList, &geoip.GeoIP{
Cidr: customCidrs,
})
}
@ -493,7 +273,7 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
if rawFieldRule.Domain != nil {
for _, domain := range *rawFieldRule.Domain {
rules, err := parseDomainRule(domain)
rules, err := conf.ParaseDomainRule(domain)
if err != nil {
return nil, newError("failed to parse domain rule: ", domain).Base(err)
}
@ -503,7 +283,7 @@ func parseFieldRule(msg json.RawMessage) (*router.RoutingRule, error) {
if rawFieldRule.Domains != nil {
for _, domain := range *rawFieldRule.Domains {
rules, err := parseDomainRule(domain)
rules, err := conf.ParaseDomainRule(domain)
if err != nil {
return nil, newError("failed to parse domain rule: ", domain).Base(err)
}
@ -600,7 +380,7 @@ func parseChinaIPRule(data []byte) (*router.RoutingRule, error) {
if err != nil {
return nil, newError("invalid router rule").Base(err)
}
chinaIPs, err := loadGeoIP("CN")
chinaIPs, err := geoip.LoadGeoIP("CN")
if err != nil {
return nil, newError("failed to load geoip:cn").Base(err)
}
@ -618,7 +398,7 @@ func parseChinaSitesRule(data []byte) (*router.RoutingRule, error) {
if err != nil {
return nil, newError("invalid router rule").Base(err).AtError()
}
domains, err := loadGeositeWithAttr("geosite.dat", "CN")
domains, err := geosite.LoadGeositeWithAttr("geosite.dat", "CN")
if err != nil {
return nil, newError("failed to load geosite:cn.").Base(err)
}

32
infra/conf/router_test.go
View file

@ -7,6 +7,8 @@ import (
"github.com/golang/protobuf/proto"
"github.com/xtls/xray-core/app/router"
"github.com/xtls/xray-core/common/matcher/domain"
"github.com/xtls/xray-core/common/matcher/geoip"
"github.com/xtls/xray-core/common/net"
. "github.com/xtls/xray-core/infra/conf"
)
@ -73,13 +75,13 @@ func TestRouterConfig(t *testing.T) {
},
Rule: []*router.RoutingRule{
{
Domain: []*router.Domain{
Domain: []*domain.Domain{
{
Type: router.Domain_Plain,
Type: domain.MatchingType_Keyword,
Value: "baidu.com",
},
{
Type: router.Domain_Plain,
Type: domain.MatchingType_Keyword,
Value: "qq.com",
},
},
@ -88,9 +90,9 @@ func TestRouterConfig(t *testing.T) {
},
},
{
Geoip: []*router.GeoIP{
Geoip: []*geoip.GeoIP{
{
Cidr: []*router.CIDR{
Cidr: []*geoip.CIDR{
{
Ip: []byte{10, 0, 0, 0},
Prefix: 8,
@ -161,13 +163,13 @@ func TestRouterConfig(t *testing.T) {
DomainStrategy: router.Config_IpIfNonMatch,
Rule: []*router.RoutingRule{
{
Domain: []*router.Domain{
Domain: []*domain.Domain{
{
Type: router.Domain_Plain,
Type: domain.MatchingType_Keyword,
Value: "baidu.com",
},
{
Type: router.Domain_Plain,
Type: domain.MatchingType_Keyword,
Value: "qq.com",
},
},
@ -176,9 +178,9 @@ func TestRouterConfig(t *testing.T) {
},
},
{
Geoip: []*router.GeoIP{
Geoip: []*geoip.GeoIP{
{
Cidr: []*router.CIDR{
Cidr: []*geoip.CIDR{
{
Ip: []byte{10, 0, 0, 0},
Prefix: 8,
@ -224,13 +226,13 @@ func TestRouterConfig(t *testing.T) {
DomainStrategy: router.Config_AsIs,
Rule: []*router.RoutingRule{
{
Domain: []*router.Domain{
Domain: []*domain.Domain{
{
Type: router.Domain_Plain,
Type: domain.MatchingType_Keyword,
Value: "baidu.com",
},
{
Type: router.Domain_Plain,
Type: domain.MatchingType_Keyword,
Value: "qq.com",
},
},
@ -239,9 +241,9 @@ func TestRouterConfig(t *testing.T) {
},
},
{
Geoip: []*router.GeoIP{
Geoip: []*geoip.GeoIP{
{
Cidr: []*router.CIDR{
Cidr: []*geoip.CIDR{
{
Ip: []byte{10, 0, 0, 0},
Prefix: 8,

5
infra/conf/xray_test.go
View file

@ -13,6 +13,7 @@ import (
"github.com/xtls/xray-core/app/router"
"github.com/xtls/xray-core/common"
clog "github.com/xtls/xray-core/common/log"
"github.com/xtls/xray-core/common/matcher/geoip"
"github.com/xtls/xray-core/common/net"
"github.com/xtls/xray-core/common/protocol"
"github.com/xtls/xray-core/common/serial"
@ -154,9 +155,9 @@ func TestXrayConfig(t *testing.T) {
DomainStrategy: router.Config_AsIs,
Rule: []*router.RoutingRule{
{
Geoip: []*router.GeoIP{
Geoip: []*geoip.GeoIP{
{
Cidr: []*router.CIDR{
Cidr: []*geoip.CIDR{
{
Ip: []byte{10, 0, 0, 0},
Prefix: 8,