Парсер медии с кастомным разрешением

This commit is contained in:
lost+skunk 2024-07-30 01:20:00 +03:00
parent 4d166ad5f9
commit 42ea2980f9
9 changed files with 105 additions and 106 deletions

16
.vscode/launch.json vendored
View File

@ -1,16 +0,0 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Launch Package",
"type": "go",
"request": "launch",
"mode": "debug",
"program": "${fileDirname}",
"showLog": true
}
]
}

View File

@ -1,5 +0,0 @@
{
"go.toolsEnvVars": {
"GOROOT": ""
}
}

View File

@ -1,9 +1,9 @@
# Devianter
[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter)
A DeviantART API library for Go.
A DeviantART guest API library for Go.
I'll probably write up some documentation, but more on that later.

View File

@ -11,7 +11,7 @@ type Thread struct {
ID int `json:"commentId"`
Parent int `json:"parentId"`
Posted time
Posted timeStamp
Author bool `json:"isAuthorHighlited"`
Desctiption string
@ -35,7 +35,7 @@ type Comments struct {
}
// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя
func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Comments) {
func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) {
for x := 0; x <= page; x++ {
ujson(
"dashared/comments/thread?typeid="+strconv.Itoa(typ)+
@ -60,7 +60,7 @@ func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Commen
}
e := json.Unmarshal([]byte(m), &content)
err(e)
try(e)
for _, a := range content.Blocks {
cmmts.Thread[i].Comment = a.Text

View File

@ -4,26 +4,26 @@ import (
"encoding/json"
"strconv"
"strings"
timelib "time"
"time"
)
// хрень для парсинга времени публикации
type time struct {
timelib.Time
type timeStamp struct {
time.Time
}
func (t *time) UnmarshalJSON(b []byte) (err error) {
func (t *timeStamp) UnmarshalJSON(b []byte) (err error) {
if b[0] == '"' && b[len(b)-1] == '"' {
b = b[1 : len(b)-1]
}
t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b))
t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b))
return
}
// самая главная структура для поста
type Deviation struct {
Title, Url, License string
PublishedTime time
PublishedTime timeStamp
ID int `json:"deviationId"`
NSFW bool `json:"isMature"`
@ -82,7 +82,7 @@ type Post struct {
ParsedComments []struct {
Author string
Posted time
Posted timeStamp
Replies, Likes int
}
@ -90,12 +90,25 @@ type Post struct {
}
// преобразование урла в правильный
func UrlFromMedia(m Media) string {
func UrlFromMedia(m Media, thumb ...int) string {
var url strings.Builder
subtractWidthHeight := func(to int, target ...*int) {
for i, l := 0, len(target); i < l; i++ {
for x := *target[i]; x > to; x -= to {
*target[i] = x
}
}
}
for _, t := range m.Types {
if t.T == "fullview" {
url.WriteString(m.BaseUri)
if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 {
if len(thumb) != 0 {
subtractWidthHeight(thumb[0], &t.W, &t.H)
}
url.WriteString("/v1/fit/w_")
url.WriteString(strconv.Itoa(t.W))
url.WriteString(",h_")
@ -110,11 +123,12 @@ func UrlFromMedia(m Media) string {
}
}
}
return url.String()
}
// для работы функции нужно ID поста и имя пользователя.
func DeviationFunc(id string, user string) Post {
func GetDeviation(id string, user string) Post {
var st Post
ujson(
"dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true",

38
misc.go
View File

@ -4,13 +4,16 @@ import (
"errors"
"log"
"math"
u "net/url"
"net/url"
"strconv"
"strings"
)
/* AVATARS AND EMOJIS */
func AEmedia(name string, t rune) (string, error) {
if len(name) < 2 {
return "", errors.New("name must be specified")
}
// список всех возможных расширений
var extensions = [3]string{
".jpg",
@ -25,9 +28,10 @@ func AEmedia(name string, t rune) (string, error) {
switch t {
case 'a':
b.WriteString("https://a.deviantart.net/avatars-big/")
b.WriteString(name[:1])
name_without_dashes := strings.ReplaceAll(name, "-", "_")
b.WriteString(name_without_dashes[:1])
b.WriteString("/")
b.WriteString(name[1:2])
b.WriteString(name_without_dashes[1:2])
b.WriteString("/")
case 'e':
b.WriteString("https://e.deviantart.net/emoticons/")
@ -60,7 +64,7 @@ type DailyDeviations struct {
Deviations []Deviation
}
func DailyDeviationsFunc(page int) (dd DailyDeviations) {
func GetDailyDeviations(page int) (dd DailyDeviations) {
ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd)
return
}
@ -74,23 +78,21 @@ type Search struct {
ResultsGalleryTemp []Deviation `json:"results"`
}
func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) {
var url strings.Builder
func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) {
var buildurl strings.Builder
e = nil
// о5 построение ссылок.
switch scope {
case 'a': // поиск артов по названию
url.WriteString("dabrowse/search/all?q=")
buildurl.WriteString("dabrowse/search/all?q=")
case 't': // поиск артов по тегам
url.WriteString("dabrowse/networkbar/tag/deviations?tag=")
buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=")
case 'g': // поиск артов пользователя или группы
if user != nil {
url.WriteString("dashared/gallection/search?username=")
for _, a := range user {
url.WriteString(a)
}
url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
buildurl.WriteString("dashared/gallection/search?username=")
buildurl.WriteString(user[0])
buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
} else {
e = errors.New("missing username (last argument)")
return
@ -99,16 +101,16 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search,
log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.")
}
url.WriteString(u.QueryEscape(query))
buildurl.WriteString(url.QueryEscape(query))
if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код
url.WriteString("&page=")
buildurl.WriteString("&page=")
} else { // иначе вместо страницы будет оффсет и страница умножится на 50
url.WriteString("&offset=")
buildurl.WriteString("&offset=")
page = 50 * page
}
url.WriteString(strconv.Itoa(page))
buildurl.WriteString(strconv.Itoa(page))
ujson(url.String(), &ss)
ujson(buildurl.String(), &ss)
if scope == 'g' {
ss.Results = ss.ResultsGalleryTemp

View File

@ -1,2 +0,0 @@
- groups
- images in comments

View File

@ -1,44 +1,12 @@
package devianter
import (
"errors"
"strconv"
"strings"
)
// структура группы или пользователя
type GroupAbout struct {
FoundatedAt time `json:"foundationTs"`
Description Text
}
type GroupAdmins struct {
Results []struct {
TypeId int
User struct {
Username string
}
}
}
type About struct {
Country, Website, WebsiteLabel, Gender string
RegDate int64 `json:"deviantFor"`
Description Text `json:"textContent"`
SocialLinks []struct {
Value string
}
Interests []struct {
Label, Value string
}
}
type users struct {
About About
CoverDeviation struct {
Deviation Deviation `json:"coverDeviation"`
}
}
type GRuser struct {
ErrorDescription string
Owner struct {
@ -106,14 +74,21 @@ type Group struct {
}
// подходит как группа, так и пользователь
func (s Group) GroupFunc() (g GRuser) {
func (s Group) GetGroup() (g GRuser, err error) {
if s.Name == "" {
return g, errors.New("missing Name field")
}
ujson("dauserprofile/init/about?username="+s.Name, &g)
return
}
// гарелея пользователя или группы
func (s Group) Gallery(page int, folderid ...int) (g Group) {
func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) {
if s.Name == "" {
return g, errors.New("missing Name field")
}
var url strings.Builder
if folderid[0] > 0 {
page--
@ -137,3 +112,36 @@ func (s Group) Gallery(page int, folderid ...int) (g Group) {
ujson(url.String(), &g.Content)
return
}
type GroupAbout struct {
FoundatedAt timeStamp `json:"foundationTs"`
Description Text
}
type GroupAdmins struct {
Results []struct {
TypeId int
User struct {
Username string
}
}
}
type About struct {
Country, Website, WebsiteLabel, Gender string
RegDate int64 `json:"deviantFor"`
Description Text `json:"textContent"`
SocialLinks []struct {
Value string
}
Interests []struct {
Label, Value string
}
}
type users struct {
About About
CoverDeviation struct {
Deviation Deviation `json:"coverDeviation"`
}
}

28
util.go
View File

@ -9,7 +9,7 @@ import (
)
// функция для высера ошибки в stderr
func err(txt error) {
func try(txt error) {
if txt != nil {
println(txt.Error())
}
@ -17,11 +17,9 @@ func err(txt error) {
// сокращение для вызова щенка и парсинга жсона
func ujson(data string, output any) {
input, e := puppy(data)
err(e)
eee := json.Unmarshal([]byte(input), output)
err(eee)
input, err := puppy(data)
try(err)
try(json.Unmarshal([]byte(input), output))
}
/* REQUEST SECTION */
@ -34,32 +32,32 @@ type reqrt struct {
}
// функция для совершения запроса
var UserAgent string
func request(uri string, other ...string) reqrt {
var r reqrt
// создаём новый запрос
cli := &http.Client{}
req, e := http.NewRequest("GET", uri, nil)
err(e)
try(e)
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
// куки и UA-шник
for num, rng := range other {
switch num {
case 1:
req.Header.Set("User-Agent", rng)
case 0:
req.Header.Set("Cookie", rng)
if UserAgent != "" {
req.Header.Set("User-Agent", UserAgent)
}
if len(other) != 0 {
req.Header.Set("Cookie", other[0])
}
resp, e := cli.Do(req)
err(e)
try(e)
defer resp.Body.Close()
body, e := io.ReadAll(resp.Body)
err(e)
try(e)
// заполняем структуру
r.Body = string(body)