furryboard/spider-scheduler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

2022-2023稳定运行版本

Browse Source
This commit is contained in:
Eigeen
2023-04-09 13:35:34 +08:00
parent 4f44e60168
commit cf80b07977
21 changed files with 332 additions and 321 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
pkg/conf/conf.go
View File

@@ -3,8 +3,8 @@ package conf
import (
"encoding/json"
"github.com/BurntSushi/toml"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/eigeen/furryboard/spider-scheduler/pkg/util"
"github.com/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/pkg/util"
"github.com/mcuadros/go-defaults"
"os"
)

6
pkg/dao/dao.go
View File

@@ -2,9 +2,9 @@ package dao
import (
"fmt"
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/pkg/conf"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/log"
"github.com/sirupsen/logrus"
"gorm.io/driver/postgres"
"gorm.io/gorm"

7
pkg/dao/model/changelog.go
View File

@@ -1,9 +1,12 @@
package model
import "gorm.io/gorm"
import (
"time"
)
type Changelog struct {
gorm.Model
ID uint `gorm:"primarykey"`
CreatedAt time.Time
Operation string `gorm:"index;not null;default:''"`
Target string `gorm:"index;not null;default:''"`
Result string `gorm:"not null;default:''"`

11
pkg/dao/model/fans.go
View File

@@ -1,11 +1,14 @@
package model
import "gorm.io/gorm"
import (
"time"
)
type Fans struct {
gorm.Model
UID uint `gorm:"index;not null"`
Fans uint `gorm:"index;not null;default:0"`
ID uint `gorm:"primarykey"`
CreatedAt time.Time
UID uint `gorm:"index;not null"`
Fans uint `gorm:"index;not null;default:0"`
}
func (Fans) TableName() string {

2
pkg/dao/model/furry.go
View File

@@ -10,7 +10,7 @@ const (
)
type Furry struct {
ID uint `gorm:"primaryKey"`
ID uint `gorm:"primarykey"`
UID uint `gorm:"uniqueIndex;not null"`
Name string `gorm:"index;not null;default:''"`
Status int `gorm:"index;not null;default:0"`

2
pkg/dao/model/liveroom.go
View File

@@ -1,7 +1,7 @@
package model
type LiveRoom struct {
ID uint `gorm:"primaryKey"`
ID uint `gorm:"primarykey"`
RoomID uint `gorm:"uniqueIndex;not null"`
Title string `gorm:"not null;default:''"`
Tags string `gorm:"not null;default:''"`

2
pkg/dao/model/metadata.go
View File

@@ -1,7 +1,7 @@
package model
type Metadata struct {
ID uint `gorm:"primaryKey"`
ID uint `gorm:"primarykey"`
Key string `gorm:"uniqueIndex;not null"`
Value string `gorm:"not null;default:''"`
}

4
pkg/logic/liveroom.go
View File

@@ -1,8 +1,8 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/dao"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
)
func GetLiveRoomsFromDB(roomIDs *[]uint64) ([]*model.LiveRoom, error) {

4
pkg/logic/metadata.go
View File

@@ -2,8 +2,8 @@ package logic
import (
"errors"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/dao"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
"gorm.io/gorm"
)

10
pkg/logic/search.go
View File

@@ -3,11 +3,11 @@ package logic
import (
"context"
"fmt"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/exception"
"github.com/eigeen/furryboard/spider-scheduler/rpc"
"github.com/eigeen/furryboard/spider-scheduler/rpc/pb"
"github.com/furryboard/spider-scheduler/pkg/dao"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/exception"
"github.com/furryboard/spider-scheduler/rpc"
"github.com/furryboard/spider-scheduler/rpc/pb"
"gorm.io/gorm"
"time"
)

2
pkg/logic/search_test.go
View File

@@ -1,7 +1,7 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"github.com/furryboard/spider-scheduler/pkg/conf"
"testing"
)

23
pkg/logic/user.go
View File

@@ -4,12 +4,13 @@ import (
"context"
"encoding/json"
"fmt"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/exception"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/eigeen/furryboard/spider-scheduler/rpc"
"github.com/eigeen/furryboard/spider-scheduler/rpc/pb"
"github.com/furryboard/spider-scheduler/pkg/conf"
"github.com/furryboard/spider-scheduler/pkg/dao"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/exception"
"github.com/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/rpc"
"github.com/furryboard/spider-scheduler/rpc/pb"
"gorm.io/gorm"
"strconv"
"sync"
@@ -20,7 +21,8 @@ import (
func GetValidUsers(page, pageSize int) ([]*model.Furry, error) {
// 分页查询
var furries []*model.Furry
tx := dao.DB().Where("status = 0").Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
//tx := dao.DB().Where("status = 0").Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
tx := dao.DB().Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
if tx.Error != nil {
return nil, exception.ErrFetchFurries("分页获取Furry列表时失败：" + tx.Error.Error())
}
@@ -31,10 +33,14 @@ func GetValidUsers(page, pageSize int) ([]*model.Furry, error) {
func GetUserInfoByUID(uid uint) (*pb.InfoReply_Data, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
info, err := rpc.SpiderCore().GetUserBasicInfo(ctx, &pb.InfoRequest{Uid: uint64(uid)})
info, err := rpc.SpiderCore().GetUserBasicInfo(ctx, &pb.InfoRequest{
Uid: uint64(uid),
Cookie: conf.Conf.SpiderCore.BiliCookie,
})
if err != nil {
return nil, exception.InternalError("获取用户信息失败：" + err.Error())
}
if info.Code != 200 {
if info.Msg == "" {
info.Msg = "Unknown"
@@ -42,6 +48,7 @@ func GetUserInfoByUID(uid uint) (*pb.InfoReply_Data, error) {
return nil, exception.ErrFetchUserInfo(
fmt.Sprintf("用户[%d]基础信息获取失败：%s(Code: %d)", uid, info.Msg, info.Code))
}
return info.Data, nil
}

4
pkg/logic/user_test.go
View File

@@ -1,8 +1,8 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/pkg/conf"
"github.com/furryboard/spider-scheduler/pkg/log"
"testing"
)

13
pkg/task/config.toml
View File

@@ -1,13 +0,0 @@
[common]
[database]
host = "localhost"
port = 5432
user = "postgres"
password = "postrootpwd"
db = "furryboard"
[spider_core]
host = "localhost"
port = 9101
bili_cookie = "buvid3=E114B569-ECF7-50D6-5874-7221C956F4F323116infoc; i-wanna-go-back=-1; _uuid=9DC109D51-C164-D1A8-D18D-CDB410A6B733223200infoc; buvid4=2EFA9D48-F352-9314-066E-800EAD1A3B3024163-022061118-7XDfT9HnZ75xcCl6GNqbKvvAaotbbCqK1dff9AggS2AFysI2RsiDgg%3D%3D; CURRENT_BLACKGAP=0; blackside_state=0; nostalgia_conf=-1; buvid_fp_plain=undefined; b_ut=5; LIVE_BUVID=AUTO2216549597211800; hit-dyn-v2=1; b_nut=100; fingerprint3=8b45c9c8e2c3bde2334046651927bc11; CURRENT_QUALITY=80; hit-new-style-dyn=0; CURRENT_FNVAL=4048; rpdid=|(J~R|~)~|Y)0J'uYY)Y|JmkY; dy_spec_agreed=1; fingerprint=64f6f507fa149d279783afea30316e2b; SESSDATA=0312b6e2%2C1687167210%2C5c48e%2Ac1; bili_jct=12bdeb674ff87c20afd2d245128d6734; DedeUserID=90931399; DedeUserID__ckMd5=507c5d481d60b7bb; sid=5n2lvshx; buvid_fp=64f6f507fa149d279783afea30316e2b; bp_t_offset_90931399=742134893995098144; PVID=2; share_source_origin=QQ; bsource=share_source_qqchat; b_lsid=4578914A_1853786E484; innersign=0"

8
pkg/task/search.go
View File

@@ -1,10 +1,10 @@
package task
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/eigeen/furryboard/spider-scheduler/pkg/logic"
"github.com/furryboard/spider-scheduler/pkg/conf"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/pkg/logic"
"time"
)

15
pkg/task/user.go
View File

@@ -1,11 +1,11 @@
package task
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/exception"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/eigeen/furryboard/spider-scheduler/pkg/logic"
"github.com/furryboard/spider-scheduler/pkg/dao"
"github.com/furryboard/spider-scheduler/pkg/dao/model"
"github.com/furryboard/spider-scheduler/pkg/exception"
"github.com/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/pkg/logic"
"math"
"time"
)
@@ -23,7 +23,7 @@ func UpdateUserInfo() {
return
}
pageSize := 32
pageSize := 1
maxPage := int(math.Ceil(float64(count) / float64(pageSize)))
for page := 0; page <= maxPage; page++ {
// 获取目标用户列表（分页）
@@ -61,7 +61,8 @@ func UpdateFans() {
// TODO: 与UpdateUserInfo有大量重复，可抽离复用代码
// 预查询数量
var count int64
tx := dao.DB().Model(&model.Furry{}).Where("status = 0").Count(&count)
//tx := dao.DB().Model(&model.Furry{}).Where("status = 0").Count(&count)
tx := dao.DB().Model(&model.Furry{}).Count(&count)
if tx.Error != nil {
err := exception.ErrFetchFurries("获取furries数量失败：" + tx.Error.Error())
log.Logger().Errorf("获取furries数量失败：%s", err)

4
pkg/task/user_test.go
View File

@@ -1,8 +1,8 @@
package task
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/furryboard/spider-scheduler/pkg/conf"
"github.com/furryboard/spider-scheduler/pkg/log"
"testing"
)