Парсер медии с кастомным разрешением
This commit is contained in:
parent
4d166ad5f9
commit
42ea2980f9
16
.vscode/launch.json
vendored
16
.vscode/launch.json
vendored
@ -1,16 +0,0 @@
|
||||
{
|
||||
// Use IntelliSense to learn about possible attributes.
|
||||
// Hover to view descriptions of existing attributes.
|
||||
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
|
||||
"version": "0.2.0",
|
||||
"configurations": [
|
||||
{
|
||||
"name": "Launch Package",
|
||||
"type": "go",
|
||||
"request": "launch",
|
||||
"mode": "debug",
|
||||
"program": "${fileDirname}",
|
||||
"showLog": true
|
||||
}
|
||||
]
|
||||
}
|
5
.vscode/settings.json
vendored
5
.vscode/settings.json
vendored
@ -1,5 +0,0 @@
|
||||
{
|
||||
"go.toolsEnvVars": {
|
||||
"GOROOT": ""
|
||||
}
|
||||
}
|
@ -1,9 +1,9 @@
|
||||
# Devianter
|
||||
|
||||
[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter) [![License: AGPL v3](https://img.shields.io/badge/License-AGPL%20v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
|
||||
[![Please don't upload to GitHub](https://nogithub.codeberg.page/badge.svg)](https://nogithub.codeberg.page) [![Go Reference](https://pkg.go.dev/badge/git.macaw.me/skunky/devianter.svg)](https://pkg.go.dev/git.macaw.me/skunky/devianter)
|
||||
|
||||
|
||||
A DeviantART API library for Go.
|
||||
A DeviantART guest API library for Go.
|
||||
|
||||
|
||||
I'll probably write up some documentation, but more on that later.
|
||||
|
@ -11,7 +11,7 @@ type Thread struct {
|
||||
ID int `json:"commentId"`
|
||||
Parent int `json:"parentId"`
|
||||
|
||||
Posted time
|
||||
Posted timeStamp
|
||||
Author bool `json:"isAuthorHighlited"`
|
||||
|
||||
Desctiption string
|
||||
@ -35,7 +35,7 @@ type Comments struct {
|
||||
}
|
||||
|
||||
// 1 - комментарии поста; 4 - комментарии на стене группы или пользователя
|
||||
func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Comments) {
|
||||
func GetComments(postid string, cursor string, page int, typ int) (cmmts Comments) {
|
||||
for x := 0; x <= page; x++ {
|
||||
ujson(
|
||||
"dashared/comments/thread?typeid="+strconv.Itoa(typ)+
|
||||
@ -60,7 +60,7 @@ func CommentsFunc(postid string, cursor string, page int, typ int) (cmmts Commen
|
||||
}
|
||||
|
||||
e := json.Unmarshal([]byte(m), &content)
|
||||
err(e)
|
||||
try(e)
|
||||
|
||||
for _, a := range content.Blocks {
|
||||
cmmts.Thread[i].Comment = a.Text
|
||||
|
@ -4,26 +4,26 @@ import (
|
||||
"encoding/json"
|
||||
"strconv"
|
||||
"strings"
|
||||
timelib "time"
|
||||
"time"
|
||||
)
|
||||
|
||||
// хрень для парсинга времени публикации
|
||||
type time struct {
|
||||
timelib.Time
|
||||
type timeStamp struct {
|
||||
time.Time
|
||||
}
|
||||
|
||||
func (t *time) UnmarshalJSON(b []byte) (err error) {
|
||||
func (t *timeStamp) UnmarshalJSON(b []byte) (err error) {
|
||||
if b[0] == '"' && b[len(b)-1] == '"' {
|
||||
b = b[1 : len(b)-1]
|
||||
}
|
||||
t.Time, err = timelib.Parse("2006-01-02T15:04:05-0700", string(b))
|
||||
t.Time, err = time.Parse("2006-01-02T15:04:05-0700", string(b))
|
||||
return
|
||||
}
|
||||
|
||||
// самая главная структура для поста
|
||||
type Deviation struct {
|
||||
Title, Url, License string
|
||||
PublishedTime time
|
||||
PublishedTime timeStamp
|
||||
ID int `json:"deviationId"`
|
||||
|
||||
NSFW bool `json:"isMature"`
|
||||
@ -82,7 +82,7 @@ type Post struct {
|
||||
|
||||
ParsedComments []struct {
|
||||
Author string
|
||||
Posted time
|
||||
Posted timeStamp
|
||||
Replies, Likes int
|
||||
}
|
||||
|
||||
@ -90,12 +90,25 @@ type Post struct {
|
||||
}
|
||||
|
||||
// преобразование урла в правильный
|
||||
func UrlFromMedia(m Media) string {
|
||||
func UrlFromMedia(m Media, thumb ...int) string {
|
||||
var url strings.Builder
|
||||
|
||||
subtractWidthHeight := func(to int, target ...*int) {
|
||||
for i, l := 0, len(target); i < l; i++ {
|
||||
for x := *target[i]; x > to; x -= to {
|
||||
*target[i] = x
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, t := range m.Types {
|
||||
if t.T == "fullview" {
|
||||
url.WriteString(m.BaseUri)
|
||||
if m.BaseUri[len(m.BaseUri)-3:] != "gif" && t.W*t.H < 33177600 {
|
||||
if len(thumb) != 0 {
|
||||
subtractWidthHeight(thumb[0], &t.W, &t.H)
|
||||
}
|
||||
|
||||
url.WriteString("/v1/fit/w_")
|
||||
url.WriteString(strconv.Itoa(t.W))
|
||||
url.WriteString(",h_")
|
||||
@ -110,11 +123,12 @@ func UrlFromMedia(m Media) string {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return url.String()
|
||||
}
|
||||
|
||||
// для работы функции нужно ID поста и имя пользователя.
|
||||
func DeviationFunc(id string, user string) Post {
|
||||
func GetDeviation(id string, user string) Post {
|
||||
var st Post
|
||||
ujson(
|
||||
"dadeviation/init?deviationid="+id+"&username="+user+"&type=art&include_session=false&expand=deviation.related&preload=true",
|
||||
|
38
misc.go
38
misc.go
@ -4,13 +4,16 @@ import (
|
||||
"errors"
|
||||
"log"
|
||||
"math"
|
||||
u "net/url"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
/* AVATARS AND EMOJIS */
|
||||
func AEmedia(name string, t rune) (string, error) {
|
||||
if len(name) < 2 {
|
||||
return "", errors.New("name must be specified")
|
||||
}
|
||||
// список всех возможных расширений
|
||||
var extensions = [3]string{
|
||||
".jpg",
|
||||
@ -25,9 +28,10 @@ func AEmedia(name string, t rune) (string, error) {
|
||||
switch t {
|
||||
case 'a':
|
||||
b.WriteString("https://a.deviantart.net/avatars-big/")
|
||||
b.WriteString(name[:1])
|
||||
name_without_dashes := strings.ReplaceAll(name, "-", "_")
|
||||
b.WriteString(name_without_dashes[:1])
|
||||
b.WriteString("/")
|
||||
b.WriteString(name[1:2])
|
||||
b.WriteString(name_without_dashes[1:2])
|
||||
b.WriteString("/")
|
||||
case 'e':
|
||||
b.WriteString("https://e.deviantart.net/emoticons/")
|
||||
@ -60,7 +64,7 @@ type DailyDeviations struct {
|
||||
Deviations []Deviation
|
||||
}
|
||||
|
||||
func DailyDeviationsFunc(page int) (dd DailyDeviations) {
|
||||
func GetDailyDeviations(page int) (dd DailyDeviations) {
|
||||
ujson("dabrowse/networkbar/rfy/deviations?page="+strconv.Itoa(page), &dd)
|
||||
return
|
||||
}
|
||||
@ -74,23 +78,21 @@ type Search struct {
|
||||
ResultsGalleryTemp []Deviation `json:"results"`
|
||||
}
|
||||
|
||||
func SearchFunc(query string, page int, scope rune, user ...string) (ss Search, e error) {
|
||||
var url strings.Builder
|
||||
func PerformSearch(query string, page int, scope rune, user ...string) (ss Search, e error) {
|
||||
var buildurl strings.Builder
|
||||
e = nil
|
||||
|
||||
// о5 построение ссылок.
|
||||
switch scope {
|
||||
case 'a': // поиск артов по названию
|
||||
url.WriteString("dabrowse/search/all?q=")
|
||||
buildurl.WriteString("dabrowse/search/all?q=")
|
||||
case 't': // поиск артов по тегам
|
||||
url.WriteString("dabrowse/networkbar/tag/deviations?tag=")
|
||||
buildurl.WriteString("dabrowse/networkbar/tag/deviations?tag=")
|
||||
case 'g': // поиск артов пользователя или группы
|
||||
if user != nil {
|
||||
url.WriteString("dashared/gallection/search?username=")
|
||||
for _, a := range user {
|
||||
url.WriteString(a)
|
||||
}
|
||||
url.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
|
||||
buildurl.WriteString("dashared/gallection/search?username=")
|
||||
buildurl.WriteString(user[0])
|
||||
buildurl.WriteString("&type=gallery&order=most-recent&init=true&limit=50&q=")
|
||||
} else {
|
||||
e = errors.New("missing username (last argument)")
|
||||
return
|
||||
@ -99,16 +101,16 @@ func SearchFunc(query string, page int, scope rune, user ...string) (ss Search,
|
||||
log.Fatalln("Invalid type.\n- 'a' -- all;\n- 't' -- tag;\n- 'g' - gallery.")
|
||||
}
|
||||
|
||||
url.WriteString(u.QueryEscape(query))
|
||||
buildurl.WriteString(url.QueryEscape(query))
|
||||
if scope != 'g' { // если область поиска не равна поиску по группам, то активируется этот код
|
||||
url.WriteString("&page=")
|
||||
buildurl.WriteString("&page=")
|
||||
} else { // иначе вместо страницы будет оффсет и страница умножится на 50
|
||||
url.WriteString("&offset=")
|
||||
buildurl.WriteString("&offset=")
|
||||
page = 50 * page
|
||||
}
|
||||
url.WriteString(strconv.Itoa(page))
|
||||
buildurl.WriteString(strconv.Itoa(page))
|
||||
|
||||
ujson(url.String(), &ss)
|
||||
ujson(buildurl.String(), &ss)
|
||||
|
||||
if scope == 'g' {
|
||||
ss.Results = ss.ResultsGalleryTemp
|
||||
|
@ -1,44 +1,12 @@
|
||||
package devianter
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// структура группы или пользователя
|
||||
type GroupAbout struct {
|
||||
FoundatedAt time `json:"foundationTs"`
|
||||
Description Text
|
||||
}
|
||||
type GroupAdmins struct {
|
||||
Results []struct {
|
||||
TypeId int
|
||||
User struct {
|
||||
Username string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type About struct {
|
||||
Country, Website, WebsiteLabel, Gender string
|
||||
RegDate int64 `json:"deviantFor"`
|
||||
Description Text `json:"textContent"`
|
||||
|
||||
SocialLinks []struct {
|
||||
Value string
|
||||
}
|
||||
Interests []struct {
|
||||
Label, Value string
|
||||
}
|
||||
}
|
||||
|
||||
type users struct {
|
||||
About About
|
||||
CoverDeviation struct {
|
||||
Deviation Deviation `json:"coverDeviation"`
|
||||
}
|
||||
}
|
||||
|
||||
type GRuser struct {
|
||||
ErrorDescription string
|
||||
Owner struct {
|
||||
@ -106,14 +74,21 @@ type Group struct {
|
||||
}
|
||||
|
||||
// подходит как группа, так и пользователь
|
||||
func (s Group) GroupFunc() (g GRuser) {
|
||||
func (s Group) GetGroup() (g GRuser, err error) {
|
||||
if s.Name == "" {
|
||||
return g, errors.New("missing Name field")
|
||||
}
|
||||
ujson("dauserprofile/init/about?username="+s.Name, &g)
|
||||
|
||||
return
|
||||
}
|
||||
|
||||
// гарелея пользователя или группы
|
||||
func (s Group) Gallery(page int, folderid ...int) (g Group) {
|
||||
func (s Group) GetGallery(page int, folderid ...int) (g Group, err error) {
|
||||
if s.Name == "" {
|
||||
return g, errors.New("missing Name field")
|
||||
}
|
||||
|
||||
var url strings.Builder
|
||||
if folderid[0] > 0 {
|
||||
page--
|
||||
@ -137,3 +112,36 @@ func (s Group) Gallery(page int, folderid ...int) (g Group) {
|
||||
ujson(url.String(), &g.Content)
|
||||
return
|
||||
}
|
||||
|
||||
type GroupAbout struct {
|
||||
FoundatedAt timeStamp `json:"foundationTs"`
|
||||
Description Text
|
||||
}
|
||||
type GroupAdmins struct {
|
||||
Results []struct {
|
||||
TypeId int
|
||||
User struct {
|
||||
Username string
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type About struct {
|
||||
Country, Website, WebsiteLabel, Gender string
|
||||
RegDate int64 `json:"deviantFor"`
|
||||
Description Text `json:"textContent"`
|
||||
|
||||
SocialLinks []struct {
|
||||
Value string
|
||||
}
|
||||
Interests []struct {
|
||||
Label, Value string
|
||||
}
|
||||
}
|
||||
|
||||
type users struct {
|
||||
About About
|
||||
CoverDeviation struct {
|
||||
Deviation Deviation `json:"coverDeviation"`
|
||||
}
|
||||
}
|
||||
|
28
util.go
28
util.go
@ -9,7 +9,7 @@ import (
|
||||
)
|
||||
|
||||
// функция для высера ошибки в stderr
|
||||
func err(txt error) {
|
||||
func try(txt error) {
|
||||
if txt != nil {
|
||||
println(txt.Error())
|
||||
}
|
||||
@ -17,11 +17,9 @@ func err(txt error) {
|
||||
|
||||
// сокращение для вызова щенка и парсинга жсона
|
||||
func ujson(data string, output any) {
|
||||
input, e := puppy(data)
|
||||
err(e)
|
||||
|
||||
eee := json.Unmarshal([]byte(input), output)
|
||||
err(eee)
|
||||
input, err := puppy(data)
|
||||
try(err)
|
||||
try(json.Unmarshal([]byte(input), output))
|
||||
}
|
||||
|
||||
/* REQUEST SECTION */
|
||||
@ -34,32 +32,32 @@ type reqrt struct {
|
||||
}
|
||||
|
||||
// функция для совершения запроса
|
||||
var UserAgent string
|
||||
|
||||
func request(uri string, other ...string) reqrt {
|
||||
var r reqrt
|
||||
|
||||
// создаём новый запрос
|
||||
cli := &http.Client{}
|
||||
req, e := http.NewRequest("GET", uri, nil)
|
||||
err(e)
|
||||
try(e)
|
||||
|
||||
req.Header.Set("User-Agent", "Mozilla/5.0 (X11; Linux x86_64; rv:123.0) Gecko/20100101 Firefox/123.0.0")
|
||||
|
||||
// куки и UA-шник
|
||||
for num, rng := range other {
|
||||
switch num {
|
||||
case 1:
|
||||
req.Header.Set("User-Agent", rng)
|
||||
case 0:
|
||||
req.Header.Set("Cookie", rng)
|
||||
if UserAgent != "" {
|
||||
req.Header.Set("User-Agent", UserAgent)
|
||||
}
|
||||
if len(other) != 0 {
|
||||
req.Header.Set("Cookie", other[0])
|
||||
}
|
||||
|
||||
resp, e := cli.Do(req)
|
||||
err(e)
|
||||
try(e)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, e := io.ReadAll(resp.Body)
|
||||
err(e)
|
||||
try(e)
|
||||
|
||||
// заполняем структуру
|
||||
r.Body = string(body)
|
||||
|
Loading…
Reference in New Issue
Block a user