diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..fa75212 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,16 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Launch Package", + "type": "go", + "request": "launch", + "mode": "debug", + "program": "${fileDirname}", + "showLog": true + } + ] +} \ No newline at end of file diff --git a/README.md b/README.md index 8f70404..1c2e10f 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Devianter -[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) +[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0) A DeviantART API library for Go. diff --git a/comments.go b/comments.go index 7b3bb3d..928de44 100644 --- a/comments.go +++ b/comments.go @@ -2,48 +2,45 @@ package devianter import ( "encoding/json" + "net/url" "strconv" - "strings" ) -type comments struct { +type Thread struct { + Replies, Likes int + ID int `json:"commentId"` + Parent int `json:"parentId"` + + Posted time + Author bool `json:"isAuthorHighlited"` + + Desctiption string + Comment string + + TextContent Text + + User struct { + Username string + Banned bool `json:"isBanned"` + } +} + +type Comments struct { Cursor string PrevOffset int HasMore, HasLess bool Total int - Thread []struct { - Replies, Likes int - ID int `json:"commentId"` - Parent int `json:"ParrentId"` - - Posted time - Author bool `json:"isAuthorHighlited"` - - Desctiption string - Comment string - - TextContent text - - User struct { - Username string - Banned bool `json:"isBanned"` - } - } + Thread []Thread } -// функция для обработки комментариев поста, пользователя, группы и многого другого -func Comments( - postid string, - cursor string, - page int, - typ int, // 1 - комментарии поста; 4 - комментарии на стене группы или пользователя -) (cmmts comments) { +// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя +func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Comments) { for x := 0; x <= page; x++ { ujson( "dashared/comments/thread?typeid="+strconv.Itoa(typ)+ "&itemid="+postid+"&maxdepth=1000&order=newest"+ - "&limit=50&cursor="+strings.ReplaceAll(cursor, "+", `%2B`), + "&limit=50&cursor="+url.QueryEscape(cursor), &cmmts, ) @@ -55,7 +52,7 @@ func Comments( cmmts.Thread[i].Comment = m // если начало строки {, а конец }, то срабатывает этот иф - if m[0] == 123 && m[l-1] == 125 { + if m[0] == '{' && m[l-1] == '}' { var content struct { Blocks []struct { Text string diff --git a/deviantion.go b/deviantion.go index 9803855..2f9410f 100644 --- a/deviantion.go +++ b/deviantion.go @@ -3,6 +3,7 @@ package devianter import ( "encoding/json" "strconv" + "strings" timelib "time" ) @@ -20,9 +21,10 @@ func (t *time) UnmarshalJSON(b []byte) (err error) { } // самая главная структура для поста -type deviantion struct { +type Deviation struct { Title, Url, License string PublishedTime time + ID int `json:"deviationId"` NSFW bool `json:"isMature"` AI bool `json:"isAiGenerated"` @@ -34,21 +36,27 @@ type deviantion struct { Stats struct { Favourites, Views, Downloads int } - Media media + Media Media Extended struct { Tags []struct { Name string } - DescriptionText text + OriginalFile struct { + Type string + Width int + Height int + Filesize int + } + DescriptionText Text RelatedContent []struct { - Deviations []deviantion + Deviations []Deviation } } - TextContent text + TextContent Text } // её выпердыши -type media struct { +type Media struct { BaseUri string Token []string Types []struct { @@ -57,7 +65,7 @@ type media struct { } } -type text struct { +type Text struct { Excerpt string Html struct { Markup, Type string @@ -65,8 +73,8 @@ type text struct { } // структура поста -type Deviantion struct { - Deviation deviantion +type Post struct { + Deviation Deviation Comments struct { Total int Cursor string @@ -78,32 +86,46 @@ type Deviantion struct { Replies, Likes int } - IMG, Desctiption string + IMG, Description string +} + +// преобразование урла в правильный +func UrlFromMedia(m Media) string { + var url strings.Builder + for _, t := range m.Types { + if t.T == "fullview" { + url.WriteString(m.BaseUri) + if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 { + url.WriteString("/v1/fit/w_") + url.WriteString(strconv.Itoa(t.W)) + url.WriteString(",h_") + url.WriteString(strconv.Itoa(t.H)) + url.WriteString("/") + url.WriteString("image") + url.WriteString(".gif") + } + if len(m.Token) > 0 { + url.WriteString("?token=") + url.WriteString(m.Token[0]) + } + } + } + return url.String() } // для работы функции нужно ID поста и имя пользователя. -func Deviation(id string, user string) Deviantion { - var st Deviantion +func DeviationFunc(id string, user string) Post { + var st Post ujson( "dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true", &st, ) - // преобразование урла в правильный - for _, t := range st.Deviation.Media.Types { - if m := st.Deviation.Media; t.T == "fullview" { - if len(m.Token) > 0 { - st.IMG = m.BaseUri + "?token=" - } else { - st.IMG = m.BaseUri + "/v1/fill/w_" + strconv.Itoa(t.W) + ",h_" + strconv.Itoa(t.H) + "/" + id + "_" + user + ".gif" + "?token=" - } - st.IMG += m.Token[0] - } - } + st.IMG = UrlFromMedia(st.Deviation.Media) // базовая обработка описания txt := st.Deviation.TextContent.Html.Markup - if len(txt) > 0 && txt[1] == 125 { + if len(txt) > 0 && txt[1] == '{' { var description struct { Blocks []struct { Text string @@ -116,7 +138,7 @@ func Deviation(id string, user string) Deviantion { } } - st.Desctiption = txt + st.Description = txt return st } diff --git a/misc.go b/misc.go index d5f6793..654ed5e 100644 --- a/misc.go +++ b/misc.go @@ -1,80 +1,14 @@ package devianter import ( - "encoding/json" "errors" - "io" "log" "math" - "net/http" + u "net/url" "strconv" "strings" ) -// функция для высера ошибки в stderr -func err(txt error) { - if txt != nil { - println(txt.Error()) - } -} - -// сокращение для вызова щенка и парсинга жсона -func ujson(data string, output any) { - input, e := puppy(data) - err(e) - - eee := json.Unmarshal([]byte(input), output) - err(eee) -} - -/* REQUEST SECTION */ -// структура для ответа сервера -type reqrt struct { - Body string - Status int - Cookies []*http.Cookie - Headers http.Header -} - -// функция для совершения запроса -func request(uri string, other ...string) reqrt { - var r reqrt - - // создаём новый запрос - cli := &http.Client{} - req, e := http.NewRequest("GET", uri, nil) - err(e) - - req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") - - // куки и UA-шник - if other != nil { - for num, rng := range other { - switch num { - case 1: - req.Header.Set("User-Agent", rng) - case 0: - req.Header.Set("Cookie", rng) - } - } - } - - resp, e := cli.Do(req) - err(e) - defer resp.Body.Close() - - body, e := io.ReadAll(resp.Body) - err(e) - - // заполняем структуру - r.Body = string(body) - r.Cookies = resp.Cookies() - r.Headers = resp.Header - r.Status = resp.StatusCode - - return r -} - /* AVATARS AND EMOJIS */ func AEmedia(name string, t rune) (string, error) { // список всех возможных расширений @@ -112,17 +46,35 @@ func AEmedia(name string, t rune) (string, error) { } } - return "", errors.New("User not exists") + return "", errors.New("user not exists") +} + +/* DAILY DEVIATIONS */ +type DailyDeviations struct { + HasMore bool + Strips []struct { + Codename, Title string + TitleType string + Deviations []Deviation + } + Deviations []Deviation +} + +func DailyDeviationsFunc(page int) (dd DailyDeviations) { + ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd) + return } /* SEARCH */ -type search struct { - Total int `json:"estTotal"` - Pages int // only for 'a' and 'g' scope. - Results []deviantion `json:"deviations,results"` +type Search struct { + Total int `json:"estTotal"` + Pages int // only for 'a' and 'g' scope. + HasMore bool + Results []Deviation `json:"deviations"` + ResultsGalleryTemp []Deviation `json:"results"` } -func Search(query string, page int, scope rune, user ...string) (ss search, e error) { +func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) { var url strings.Builder e = nil @@ -140,14 +92,14 @@ func Search(query string, page int, scope rune, user ...string) (ss search, e er } url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=") } else { - e = errors.New("Missing username (last argument)") + e = errors.New("missing username (last argument)") return } default: log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.") } - url.WriteString(query) + url.WriteString(u.QueryEscape(query)) if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код url.WriteString("&page=") } else { // иначе вместо страницы будет оффсет и страница умножится на 50 @@ -158,8 +110,13 @@ func Search(query string, page int, scope rune, user ...string) (ss search, e er ujson(url.String(), &ss) + if scope == 'g' { + ss.Results = ss.ResultsGalleryTemp + } + // расчёт, сколько всего страниц по запросу. без токена 417 страниц - максимум - for x := 0; x < int(math.Round(float64(ss.Total/25))); x++ { + totalfloat := int(math.Round(float64(ss.Total / 25))) + for x := 0; x < totalfloat; x++ { if x <= 417 { ss.Pages = x } @@ -167,53 +124,3 @@ func Search(query string, page int, scope rune, user ...string) (ss search, e er return } - -/* PUPPY aka DeviantArt API */ -func puppy(data string) (string, error) { - // получение или обновление токена - update := func() (string, string, error) { - var cookie string - if cookie == "" { - req := request("https://www.deviantart.com/_puppy") - - for _, content := range req.Cookies { - cookie = content.Raw - } - } - - req := request("https://www.deviantart.com", cookie) - if req.Status != 200 { - return "", "", errors.New(req.Body) - } - - return cookie, req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3], nil - } - - // использование токена - var ( - cookie, token string - ) - if cookie == "" || token == "" { - var e error - cookie, token, e = update() - if e != nil { - return "", e - } - } - - var url strings.Builder - url.WriteString("https://www.deviantart.com/_puppy/") - url.WriteString(data) - url.WriteString("&csrf_token=") - url.WriteString(token) - url.WriteString("&da_minor_version=20230710") - - body := request(url.String(), cookie) - - // если код ответа не 200, возвращается ошибка - if body.Status != 200 { - return "", errors.New(body.Body) - } - - return body.Body, nil -} diff --git a/user-group.go b/user-group.go index 32342fb..1592568 100644 --- a/user-group.go +++ b/user-group.go @@ -6,7 +6,40 @@ import ( ) // структура группы или пользователя -type Group struct { +type GroupAbout struct { + FoundatedAt time `json:"foundationTs"` + Description Text +} +type GroupAdmins struct { + Results []struct { + TypeId int + User struct { + Username string + } + } +} + +type About struct { + Country, Website, WebsiteLabel, Gender string + RegDate int64 `json:"deviantFor"` + Description Text `json:"textContent"` + + SocialLinks []struct { + Value string + } + Interests []struct { + Label, Value string + } +} + +type users struct { + About About + CoverDeviation struct { + Deviation Deviation `json:"coverDeviation"` + } +} + +type GRuser struct { ErrorDescription string Owner struct { Group bool `json:"isGroup"` @@ -18,75 +51,89 @@ type Group struct { Modules []struct { Name string ModuleData struct { - About struct { - Country, Website, WebsiteLabel, Gender, Tagline string - DeviantFor int64 - SocialLinks []struct { - Value string - } - TextContent text - Interests []struct { - Label, Value string - } - } - CoverDeviation struct { - Deviation deviantion `json:"coverDeviation"` - } - - // группы - GroupAbout struct { - Tagline string - CreatinDate time `json:"foundationTs"` - Description text - } - GroupAdmins struct { - Results []struct { - Username string - } - } - Folders struct { - Results []struct { - FolderId int - Name string - } - } - - // галерея - ModuleData struct { - Folder struct { - Username string - Pages int `json:"totalPageCount"` - Deviations []deviantion - } `json:"folderDeviations"` - } + GroupAbout GroupAbout + GroupAdmins GroupAdmins + users } } } } - PageExtraData struct { - GruserTagline string - Stats struct { + Extra struct { + Tag string `json:"gruserTagline"` + Stats struct { Deviations, Watchers, Watching, Pageviews, CommentsMade, Favourites, Friends int FeedComments int `json:"commentsReceivedProfile"` } - } + } `json:"pageExtraData"` } -func UGroup(name string) (g Group) { - ujson("dauserprofile/init/about?username="+name, &g) +type Gallery struct { + Gruser struct { + ID int `json:"gruserId"` + Page struct { + Modules []struct { + Name string + ModuleData struct { + // группы + Folders struct { + HasMore bool + Results []struct { + FolderId int + Size int + Name string + Thumb Deviation + } + } + + // галерея + Folder struct { + HasMore bool + Username string + Pages int `json:"totalPageCount"` + Deviations []Deviation + } `json:"folderDeviations"` + } + } + } + } + HasMore bool + Results []Deviation +} + +type Group struct { + Name string // обязательно заполнить + Content Gallery +} + +// подходит как группа, так и пользователь +func (s Group) GroupFunc() (g GRuser) { + ujson("dauserprofile/init/about?username="+s.Name, &g) return } // гарелея пользователя или группы -func Gallery(name string, page int) (g Group) { +func (s Group) Gallery(page int, folderid ...int) (g Group) { var url strings.Builder - url.WriteString("dauserprofile/init/gallery?username=") - url.WriteString(name) - url.WriteString("&page=") - url.WriteString(strconv.Itoa(page)) - url.WriteString("&deviations_limit=50&with_subfolders=false") + if folderid[0] > 0 { + page-- + url.WriteString("dashared/gallection/contents?username=") + url.WriteString(s.Name) + url.WriteString("&folderid=") + url.WriteString(strconv.Itoa(folderid[0])) + url.WriteString("&offset=") + url.WriteString(strconv.Itoa(page * 50)) + url.WriteString("&type=gallery&") + } else { + url.WriteString("dauserprofile/init/gallery?username=") + url.WriteString(s.Name) + url.WriteString("&page=") + url.WriteString(strconv.Itoa(page)) + url.WriteString("&deviations_") + } + url.WriteString("limit=50") + url.WriteString("&with_subfolders=false") - ujson(url.String(), &g) + ujson(url.String(), &g.Content) return } diff --git a/util.go b/util.go new file mode 100644 index 0000000..b98aa1f --- /dev/null +++ b/util.go @@ -0,0 +1,112 @@ +package devianter + +import ( + "encoding/json" + "errors" + "io" + "net/http" + "strings" +) + +// функция для высера ошибки в stderr +func err(txt error) { + if txt != nil { + println(txt.Error()) + } +} + +// сокращение для вызова щенка и парсинга жсона +func ujson(data string, output any) { + input, e := puppy(data) + err(e) + + eee := json.Unmarshal([]byte(input), output) + err(eee) +} + +/* REQUEST SECTION */ +// структура для ответа сервера +type reqrt struct { + Body string + Status int + Cookies []*http.Cookie + Headers http.Header +} + +// функция для совершения запроса +func request(uri string, other ...string) reqrt { + var r reqrt + + // создаём новый запрос + cli := &http.Client{} + req, e := http.NewRequest("GET", uri, nil) + err(e) + + req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0") + + // куки и UA-шник + for num, rng := range other { + switch num { + case 1: + req.Header.Set("User-Agent", rng) + case 0: + req.Header.Set("Cookie", rng) + } + } + + resp, e := cli.Do(req) + err(e) + defer resp.Body.Close() + + body, e := io.ReadAll(resp.Body) + err(e) + + // заполняем структуру + r.Body = string(body) + r.Cookies = resp.Cookies() + r.Headers = resp.Header + r.Status = resp.StatusCode + + return r +} + +/* PUPPY aka DeviantArt API */ +// получение или обновление токена +var cookie string +var token string + +func UpdateCSRF() error { + if cookie == "" { + req := request("https://www.deviantart.com/_puppy") + + for _, content := range req.Cookies { + cookie = content.Raw + } + } + + req := request("https://www.deviantart.com", cookie) + if req.Status != 200 { + return errors.New(req.Body) + } + token = req.Body[strings.Index(req.Body, "window.__CSRF_TOKEN__ = '")+25 : strings.Index(req.Body, "window.__XHR_LOCAL__")-3] + + return nil +} + +func puppy(data string) (string, error) { + var url strings.Builder + url.WriteString("https://www.deviantart.com/_puppy/") + url.WriteString(data) + url.WriteString("&csrf_token=") + url.WriteString(token) + url.WriteString("&da_minor_version=20230710") + + body := request(url.String(), cookie) + + // если код ответа не 200, возвращается ошибка + if body.Status != 200 { + return "", errors.New(body.Body) + } + + return body.Body, nil +}