scout_ante/dashboard-v2
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix domain parsing

Browse Source
This commit is contained in:
rxdn 2022-08-04 14:51:55 +01:00
parent 7bee551802
commit 6aa8e947fc
3 changed files with 14 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -27,6 +27,7 @@ require (
github.com/pkg/errors v0.9.1
github.com/rxdn/gdl v0.0.0-20220726141522-788c9ce67ad0
github.com/sirupsen/logrus v1.6.0
github.com/weppos/publicsuffix-go v0.20.0
golang.org/x/sync v0.0.0-20201207232520-09787c993a3a
)
@ -79,6 +80,7 @@ require (
github.com/ugorji/go/codec v1.2.6 // indirect
go.uber.org/atomic v1.6.0 // indirect
golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3 // indirect
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2 // indirect
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 // indirect
golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b // indirect
golang.org/x/text v0.3.7 // indirect

10
go.sum
View File

@ -39,10 +39,6 @@ github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc h1:n15W8
github.com/TicketsBot/archiverclient v0.0.0-20220326163414-558fd52746dc/go.mod h1:2KcfHS0JnSsgcxZBs3NyWMXNQzEo67mBSGOyzHPWOCc=
github.com/TicketsBot/common v0.0.0-20220703211704-f792aa9f0c42 h1:3/qnbrEfL8gqSbjJ4o7WKkdoPngmhjAGEXFwteEjpqs=
github.com/TicketsBot/common v0.0.0-20220703211704-f792aa9f0c42/go.mod h1:WxHh6bY7KhIqdayeOp5f0Zj2NNi/7QqCQfMEqHnpdAM=
github.com/TicketsBot/database v0.0.0-20220802140804-643695cd8347 h1:JVdXXxs6vxxldfesM/mmBNVBEVfkrFOpjcMdwxJQ4fw=
github.com/TicketsBot/database v0.0.0-20220802140804-643695cd8347/go.mod h1:gAtOoQKZfCkQ4AoNWQUSl51Fnlqk+odzD/hZ1e1sXyI=
github.com/TicketsBot/database v0.0.0-20220802153118-96e9a2c9dc3f h1:LNyimYmQ7tLdw4tj2X80KPwdb2u4PZIVqn+4I4jSjeY=
github.com/TicketsBot/database v0.0.0-20220802153118-96e9a2c9dc3f/go.mod h1:gAtOoQKZfCkQ4AoNWQUSl51Fnlqk+odzD/hZ1e1sXyI=
github.com/TicketsBot/database v0.0.0-20220803192010-3003f6bfcb73 h1:+KDNvovOyjYRIhoZvP2thtGvjQ4EI56Cf7rgXsQDhPE=
github.com/TicketsBot/database v0.0.0-20220803192010-3003f6bfcb73/go.mod h1:gAtOoQKZfCkQ4AoNWQUSl51Fnlqk+odzD/hZ1e1sXyI=
github.com/TicketsBot/logarchiver v0.0.0-20220326162808-cdf0310f5e1c h1:OqGjFH6mbE6gd+NqI2ARJdtH3UUvhiAkD0r0fhGJK2s=
@ -202,6 +198,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
@ -445,6 +443,10 @@ github.com/ugorji/go v1.2.6/go.mod h1:anCg0y61KIhDlPZmnH+so+RQbysYVyDko0IMgJv0Nn
github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY=
github.com/ugorji/go/codec v1.2.6 h1:7kbGefxLoDBuYXOms4yD7223OpNMMPNPZxXk5TvFcyQ=
github.com/ugorji/go/codec v1.2.6/go.mod h1:V6TCNZ4PHqoHGFZuSG1W8nrCzzdgA2DozYxWFFpvxTw=
github.com/weppos/publicsuffix-go v0.12.0/go.mod h1:z3LCPQ38eedDQSwmsSRW4Y7t2L8Ln16JPQ02lHAdn5k=
github.com/weppos/publicsuffix-go v0.20.0 h1:59ypvSUbW3Dunc6zVm+v+MmXf2Q6cGiNDkxgRIzEnaA=
github.com/weppos/publicsuffix-go v0.20.0/go.mod h1:5ZC/Uv3fIEUE0eP6o9+Yg4+5+W8V0/BieMi05feGXVA=
github.com/weppos/publicsuffix-go/publicsuffix/generator v0.0.0-20220704091424-e0182326a282/go.mod h1:GHfoeIdZLdZmLjMlzBftbTDntahTttUMWjxZwQJhULE=
github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

12
utils/netutils.go
View File

@ -1,8 +1,8 @@
package utils
import (
"github.com/weppos/publicsuffix-go/publicsuffix"
"net/url"
"strings"
)
func GetUrlHost(rawUrl string) string {
@ -15,10 +15,10 @@ func GetUrlHost(rawUrl string) string {
}
func SecondLevelDomain(domain string) string {
split := strings.Split(strings.TrimRight(domain, "."), ".")
if len(split) > 2 {
return strings.Join(split[len(split)-2:len(split)], ".")
} else {
return strings.Join(split, ".")
domain, err := publicsuffix.Domain(domain)
if err != nil {
return "Invalid domain"
}
return domain
}