Implements embedded TOR support

This commit is contained in:
Mikhail Klementev 2019-06-20 16:20:51 +00:00
parent e78fc03417
commit d8865df7a9
No known key found for this signature in database
GPG key ID: BE44DA8C062D87DC
2 changed files with 53 additions and 1 deletions

View file

@ -25,6 +25,11 @@ import (
"golang.org/x/net/html/charset"
)
var (
Transport *http.Transport = nil
UserAgent string
)
func fixLinks(db *sql.DB, doc *goquery.Document, pageUrl *url.URL) (err error) {
doc.Find("a").Each(func(i int, s *goquery.Selection) {
@ -133,6 +138,9 @@ func handleResponse(db *sql.DB, resp *http.Response, lastUrl *url.URL) {
func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
client := &http.Client{Jar: jar}
if Transport != nil {
client.Transport = Transport
}
var lastUrl *url.URL
@ -155,7 +163,7 @@ func Get(db *sql.DB, jar *cookiejar.Jar, linkUrl string) {
log.Fatalln(err)
}
req.Header.Set("User-Agent", "Wi 0.0")
req.Header.Set("User-Agent", UserAgent)
resp, err := client.Do(req)
if err != nil {
@ -214,6 +222,9 @@ func Form(db *sql.DB, jar *cookiejar.Jar, formID int64, formArgs []string) {
}
client := &http.Client{Jar: jar}
if Transport != nil {
client.Transport = Transport
}
var lastUrl *url.URL