diff --git a/go.mod b/go.mod index bd6a77b..6d87587 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/rxdn/gdl v0.0.0-20220726141522-788c9ce67ad0 github.com/sirupsen/logrus v1.6.0 + github.com/weppos/publicsuffix-go v0.20.0 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a ) @@ -79,6 +80,7 @@ require ( github.com/ugorji/go/codec v1.2.6 // indirect go.uber.org/atomic v1.6.0 // indirect golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect + golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect golang.org/x/text v0.3.7 // indirect diff --git a/go.sum b/go.sum index 735e8b0..147d438 100644 --- a/go.sum +++ b/go.sum @@ -39,10 +39,6 @@ github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc h1:n15W8 github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc/go.mod h1:2KcfHS0JnSsgcxZBs3NyWMXNQzEo67mBSGOyzHPWOCc= github.com/TicketsBot/common v0.0.0-20220703211704-f792aa9f0c42 h1:3/qnbrEfL8gqSbjJ4o7WKkdoPngmhjAGEXFwteEjpqs= github.com/TicketsBot/common v0.0.0-20220703211704-f792aa9f0c42/go.mod h1:WxHh6bY7KhIqdayeOp5f0Zj2NNi/7QqCQfMEqHnpdAM= -github.com/TicketsBot/database v0.0.0-20220802140804-643695cd8347 h1:JVdXXxs6vxxldfesM/mmBNVBEVfkrFOpjcMdwxJQ4fw= -github.com/TicketsBot/database v0.0.0-20220802140804-643695cd8347/go.mod h1:gAtOoQKZfCkQ4AoNWQUSl51Fnlqk+odzD/hZ1e1sXyI= -github.com/TicketsBot/database v0.0.0-20220802153118-96e9a2c9dc3f h1:LNyimYmQ7tLdw4tj2X80KPwdb2u4PZIVqn+4I4jSjeY= -github.com/TicketsBot/database v0.0.0-20220802153118-96e9a2c9dc3f/go.mod h1:gAtOoQKZfCkQ4AoNWQUSl51Fnlqk+odzD/hZ1e1sXyI= github.com/TicketsBot/database v0.0.0-20220803192010-3003f6bfcb73 h1:+KDNvovOyjYRIhoZvP2thtGvjQ4EI56Cf7rgXsQDhPE= github.com/TicketsBot/database v0.0.0-20220803192010-3003f6bfcb73/go.mod h1:gAtOoQKZfCkQ4AoNWQUSl51Fnlqk+odzD/hZ1e1sXyI= github.com/TicketsBot/logarchiver v0.0.0-20220326162808-cdf0310f5e1c h1:OqGjFH6mbE6gd+NqI2ARJdtH3UUvhiAkD0r0fhGJK2s= @@ -202,6 +198,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -445,6 +443,10 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ= github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw= +github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k= +github.com/weppos/publicsuffix-go v0.20.0 h1:59ypvSUbW3Dunc6zVm+v+MmXf2Q6cGiNDkxgRIzEnaA= +github.com/weppos/publicsuffix-go v0.20.0/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA= +github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= diff --git a/utils/netutils.go b/utils/netutils.go index 72e3790..602c932 100644 --- a/utils/netutils.go +++ b/utils/netutils.go @@ -1,8 +1,8 @@ package utils import ( + "github.com/weppos/publicsuffix-go/publicsuffix" "net/url" - "strings" ) func GetUrlHost(rawUrl string) string { @@ -15,10 +15,10 @@ func GetUrlHost(rawUrl string) string { } func SecondLevelDomain(domain string) string { - split := strings.Split(strings.TrimRight(domain, "."), ".") - if len(split) > 2 { - return strings.Join(split[len(split)-2:len(split)], ".") - } else { - return strings.Join(split, ".") + domain, err := publicsuffix.Domain(domain) + if err != nil { + return "Invalid domain" } + + return domain }