furryboard/spider-scheduler
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

实现通过直播间获取furry等

Browse Source 
用户信息新增直播间房间号字段

能用了，但不是很好用
This commit is contained in:
Eigeen
2022-12-25 18:06:03 +08:00
parent 511d889557
commit 4f44e60168
24 changed files with 2919 additions and 783 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
pkg/conf/conf.go
View File

@@ -27,8 +27,9 @@ type Database struct {
}
type SpiderCore struct {
Host string `toml:"host" default:"localhost"`
Port uint16 `toml:"port" default:"9996"`
Host string `toml:"host" default:"localhost"`
Port uint16 `toml:"port" default:"9996"`
BiliCookie string `toml:"bili_cookie"`
}
var (

2
pkg/dao/dao.go
View File

@@ -67,6 +67,8 @@ func autoMigrate() {
var err error
err = db.AutoMigrate(&model.Changelog{})
err = db.AutoMigrate(&model.Fans{})
err = db.AutoMigrate(&model.Metadata{})
err = db.AutoMigrate(&model.LiveRoom{})
if err != nil {
return
}

18
pkg/dao/model/furry.go
View File

@@ -1,10 +1,20 @@
package model
const (
StatusOK = iota
StatusIgnore
StatusAccountClosed
StatusPendingVideo
StatusPendingLive
StatusUntracked
)
type Furry struct {
ID uint `gorm:"primaryKey"`
UID uint `gorm:"uniqueIndex;not null"`
Name string `gorm:"index;not null;default:''"`
Status int `gorm:"index;not null;default:0"`
ID uint `gorm:"primaryKey"`
UID uint `gorm:"uniqueIndex;not null"`
Name string `gorm:"index;not null;default:''"`
Status int `gorm:"index;not null;default:0"`
LiveRoomID uint `gorm:"index;not null;default:0"`
}
func (Furry) TableName() string {

8
pkg/dao/model/liveroom.go Normal file
View File

@@ -0,0 +1,8 @@
package model
type LiveRoom struct {
ID uint `gorm:"primaryKey"`
RoomID uint `gorm:"uniqueIndex;not null"`
Title string `gorm:"not null;default:''"`
Tags string `gorm:"not null;default:''"`
}

7
pkg/dao/model/metadata.go Normal file
View File

@@ -0,0 +1,7 @@
package model
type Metadata struct {
ID uint `gorm:"primaryKey"`
Key string `gorm:"uniqueIndex;not null"`
Value string `gorm:"not null;default:''"`
}

30
pkg/exception/business.go
View File

@@ -1,5 +1,11 @@
package exception
/**
4xxx 数据库异常
5xxx 业务逻辑异常
6xxx 网络API异常
*/
func InternalError(msg string) BaseError {
shortMsg := "系统异常，请联系管理员"
return BaseError{Code: 500, Msg: msg, ShortMsg: shortMsg}
@@ -7,20 +13,30 @@ func InternalError(msg string) BaseError {
func ErrFetchUserInfo(msg string) BaseError {
shortMsg := "用户信息获取失败"
return BaseError{Code: 4001, Msg: msg, ShortMsg: shortMsg}
}
func ErrFetchStat(msg string) BaseError {
shortMsg := "获取用户统计信息失败"
return BaseError{Code: 4002, Msg: msg, ShortMsg: shortMsg}
}
func ErrUserMismatch(msg string) BaseError {
shortMsg := "操作的两个用户不匹配"
return BaseError{Code: 5001, Msg: msg, ShortMsg: shortMsg}
}
func ErrFetchFurries(msg string) BaseError {
shortMsg := "获取Furry列表失败"
return BaseError{Code: 5002, Msg: msg, ShortMsg: shortMsg}
return BaseError{Code: 6001, Msg: msg, ShortMsg: shortMsg}
}
func ErrUserMismatch(msg string) BaseError {
shortMsg := "操作的两个用户不匹配"
return BaseError{Code: 5003, Msg: msg, ShortMsg: shortMsg}
func ErrSearchVideo(msg string) BaseError {
shortMsg := "分类视频搜索失败"
return BaseError{Code: 6002, Msg: msg, ShortMsg: shortMsg}
}
func ErrFetchStat(msg string) BaseError {
shortMsg := "获取用户统计信息失败"
return BaseError{Code: 5004, Msg: msg, ShortMsg: shortMsg}
func ErrSearchLiveRoom(msg string) BaseError {
shortMsg := "直播间搜索失败"
return BaseError{Code: 6003, Msg: msg, ShortMsg: shortMsg}
}

1
pkg/logic/config.toml
View File

@@ -10,3 +10,4 @@
[spider_core]
host = "localhost"
port = 9101
bili_cookie = "buvid3=E114B569-ECF7-50D6-5874-7221C956F4F323116infoc; i-wanna-go-back=-1; _uuid=9DC109D51-C164-D1A8-D18D-CDB410A6B733223200infoc; buvid4=2EFA9D48-F352-9314-066E-800EAD1A3B3024163-022061118-7XDfT9HnZ75xcCl6GNqbKvvAaotbbCqK1dff9AggS2AFysI2RsiDgg%3D%3D; CURRENT_BLACKGAP=0; blackside_state=0; nostalgia_conf=-1; buvid_fp_plain=undefined; b_ut=5; LIVE_BUVID=AUTO2216549597211800; hit-dyn-v2=1; b_nut=100; fingerprint3=8b45c9c8e2c3bde2334046651927bc11; CURRENT_QUALITY=80; hit-new-style-dyn=0; CURRENT_FNVAL=4048; rpdid=|(J~R|~)~|Y)0J'uYY)Y|JmkY; dy_spec_agreed=1; fingerprint=64f6f507fa149d279783afea30316e2b; SESSDATA=0312b6e2%2C1687167210%2C5c48e%2Ac1; bili_jct=12bdeb674ff87c20afd2d245128d6734; DedeUserID=90931399; DedeUserID__ckMd5=507c5d481d60b7bb; sid=5n2lvshx; buvid_fp=64f6f507fa149d279783afea30316e2b; bp_t_offset_90931399=742134893995098144; PVID=2; share_source_origin=QQ; bsource=share_source_qqchat; b_lsid=4578914A_1853786E484; innersign=0"

35
pkg/logic/liveroom.go Normal file
View File

@@ -0,0 +1,35 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
)
func GetLiveRoomsFromDB(roomIDs *[]uint64) ([]*model.LiveRoom, error) {
var liveRooms []*model.LiveRoom
tx := dao.DB().Model(&model.LiveRoom{}).Where("room_id in (?)", *roomIDs).Find(&liveRooms)
return liveRooms, tx.Error
}
//
//func UpdateLiveRoom(infos []*pb.InfoReply_Data) (int, error) {
// // 获取数据库里已有的info
// var roomIDs []uint64
// for _, info := range infos {
// if info.LiveRoom == nil {
// continue
// }
// roomIDs = append(roomIDs, info.LiveRoom.Roomid)
// }
// oldInfos, err := GetLiveRoomsFromDB(&roomIDs)
//
// err = dao.DB().Transaction(func(tx *gorm.DB) error {
// for i, info := range infos {
// if oldInfos[i].Title != info.LiveRoom
// }
// })
// if err != nil {
// return
// }
//
//}

42
pkg/logic/metadata.go Normal file
View File

@@ -0,0 +1,42 @@
package logic
import (
"errors"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"gorm.io/gorm"
)
func AddOrUpdateMetadata(key, value string) error {
metadata, err := GetMetadata(key)
if err != nil {
return err
}
var tx *gorm.DB
if metadata != nil {
metadata.Value = value
tx = dao.DB().Save(metadata)
} else {
tx = dao.DB().Create(&model.Metadata{
Key: key,
Value: value,
})
}
return tx.Error
}
func DeleteMetadata(key string) error {
tx := dao.DB().Model(&model.Metadata{}).Delete("key = ?", key)
return tx.Error
}
func GetMetadata(key string) (*model.Metadata, error) {
var metadata model.Metadata
tx := dao.DB().Model(&model.Metadata{}).Where("key = ?", key).First(&metadata)
err := tx.Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
return &metadata, tx.Error
}

148
pkg/logic/search.go Normal file
View File

@@ -0,0 +1,148 @@
package logic
import (
"context"
"fmt"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/exception"
"github.com/eigeen/furryboard/spider-scheduler/rpc"
"github.com/eigeen/furryboard/spider-scheduler/rpc/pb"
"gorm.io/gorm"
"time"
)
// SearchNewestVideos b站搜索 最新发布排序 仅搜索视频
func SearchNewestVideos(keyword, cookie string, page uint) ([]*pb.SearchVideoResult, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
info, err := rpc.SpiderCore().SearchVideo(ctx, &pb.SearchVideoReq{
Keyword: keyword,
Order: "pubdate",
Page: uint32(page),
Cookie: cookie,
})
if err != nil {
return nil, exception.InternalError("获取最新视频搜索信息失败：" + err.Error())
}
if info.Code != 200 { // -400 -411请求被拦截
if info.Msg == "" {
info.Msg = "Unknown"
}
return nil, exception.ErrSearchVideo(
fmt.Sprintf("最新视频搜索信息[keyword=%s]获取失败：%s(Code: %d)", keyword, info.Msg, info.Code))
}
return info.Data.Result, nil
}
func UpdateVideoCheckpoint(value string) error {
return AddOrUpdateMetadata("SearchVideoCheckpoint", value)
}
// SearchLiveRooms b站搜索 默认排序 仅搜索直播间
func SearchLiveRooms(keyword, cookie string, page uint) ([]*pb.SearchLiveRoomResult, error) {
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
info, err := rpc.SpiderCore().SearchLiveRoom(ctx, &pb.SearchLiveRoomReq{
Keyword: keyword,
Order: "online",
Page: uint32(page),
Cookie: cookie,
})
if err != nil {
return nil, exception.InternalError("获取直播间搜索信息失败：" + err.Error())
}
if info.Code != 200 {
if info.Msg == "" {
info.Msg = "Unknown"
}
return nil, exception.ErrSearchLiveRoom(
fmt.Sprintf("直播间搜索信息[keyword=%s]获取失败：%s(Code: %d)", keyword, info.Msg, info.Code))
}
return info.Data.Result, nil
}
func GetLiveRoomPageNum(keyword, cookie string) (int, error) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
info, err := rpc.SpiderCore().SearchLiveRoom(ctx, &pb.SearchLiveRoomReq{
Keyword: keyword,
Order: "online",
Page: 1,
Cookie: cookie,
})
if err != nil {
return 0, exception.InternalError("获取直播间搜索信息失败：" + err.Error())
}
if info.Code != 200 {
if info.Msg == "" {
info.Msg = "Unknown"
}
return 0, exception.ErrSearchLiveRoom(
fmt.Sprintf("直播间搜索信息[keyword=%s]获取失败：%s(Code: %d)", keyword, info.Msg, info.Code))
}
return int(info.Data.NumPages), nil
}
// GetLiveRoomByRoomID 通过room_id从数据库查询直播间信息 返回值不为nil
func GetLiveRoomByRoomID(roomID uint64) (*model.LiveRoom, error) {
var room model.LiveRoom
tx := dao.DB().Model(&model.LiveRoom{}).Where("room_id = ?", roomID).Find(&room)
if tx.Error != nil {
return &room, tx.Error
}
return &room, nil
}
func UpdateLiveRoomInfo(rooms []*pb.SearchLiveRoomResult) (int, error) {
err := dao.DB().Transaction(func(tx *gorm.DB) error {
for _, room := range rooms {
roomInfo, err := GetLiveRoomByRoomID(room.Roomid)
if err != nil {
return err
}
roomInfo.RoomID = uint(room.Roomid)
roomInfo.Title = room.Title
roomInfo.Tags = room.Tags
if err = tx.Save(roomInfo).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
return 0, err
}
err = dao.DB().Transaction(func(tx *gorm.DB) error {
for _, room := range rooms {
exists, err := IsUserExistsByUID(uint(room.Uid))
if err != nil {
return err
}
if !exists {
tx.Create(&model.Furry{
UID: uint(room.Uid),
Name: room.Uname,
Status: model.StatusPendingLive,
LiveRoomID: uint(room.Roomid),
})
continue
}
if err = tx.Model(&model.Furry{}).Where("uid = ?", room.Uid).Update("live_room_id", room.Roomid).Error; err != nil {
return err
}
}
return nil
})
if err != nil {
return 0, err
}
return len(rooms), nil
}

35
pkg/logic/search_test.go Normal file
View File

@@ -0,0 +1,35 @@
package logic
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"testing"
)
func TestSearchNewestVideos(t *testing.T) {
BeforeTesting()
videos, err := SearchNewestVideos("furry", conf.Conf.SpiderCore.BiliCookie, 1)
if err != nil {
t.Error(err)
}
t.Log(len(videos))
t.Log(videos[0])
}
func TestGetLiveRoomPageNum(t *testing.T) {
BeforeTesting()
num, err := GetLiveRoomPageNum("furry", conf.Conf.SpiderCore.BiliCookie)
if err != nil {
t.Error(err)
}
t.Log(num)
}
func TestSearchLiveRooms(t *testing.T) {
BeforeTesting()
rooms, err := SearchLiveRooms("furry", conf.Conf.SpiderCore.BiliCookie, 1)
if err != nil {
t.Error(err)
}
t.Log(len(rooms))
t.Log(rooms[0])
}

38
pkg/logic/user.go
View File

@@ -16,11 +16,11 @@ import (
"time"
)
// GetUsers 分页获取用户信息（数据库）
func GetUsers(page, pageSize int) ([]*model.Furry, error) {
// GetValidUsers 分页获取用户信息（数据库）
func GetValidUsers(page, pageSize int) ([]*model.Furry, error) {
// 分页查询
var furries []*model.Furry
tx := dao.DB().Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
tx := dao.DB().Where("status = 0").Offset((page - 1) * pageSize).Limit(pageSize).Find(&furries)
if tx.Error != nil {
return nil, exception.ErrFetchFurries("分页获取Furry列表时失败：" + tx.Error.Error())
}
@@ -45,6 +45,20 @@ func GetUserInfoByUID(uid uint) (*pb.InfoReply_Data, error) {
return info.Data, nil
}
// IsUserExistsByUID 通过UID判断用户是否存在数据库
func IsUserExistsByUID(uid uint) (bool, error) {
var count int64
tx := dao.DB().Model(&model.Furry{}).Where("uid = ?", uid).Count(&count)
if tx.Error != nil {
return false, exception.ErrFetchFurries(fmt.Sprintf("查找用户[%d]时发生错误：%s", uid, tx.Error.Error()))
}
if count == 0 {
return false, nil
} else {
return true, nil
}
}
// BatchGetUserInfo 批量获取用户基础信息
func BatchGetUserInfo(userIds *[]uint) []*pb.InfoReply_Data {
poolSize := 8
@@ -82,8 +96,8 @@ func BatchGetUserInfo(userIds *[]uint) []*pb.InfoReply_Data {
func UpdateUserInfo(users []*model.Furry, infos []*pb.InfoReply_Data) (int, error) {
// 以UID为索引转换为map
infoMap := make(map[uint]*pb.InfoReply_Data)
for _, i := range infos {
infoMap[uint(i.Mid)] = i
for _, info := range infos {
infoMap[uint(info.Mid)] = info
}
// 检测变化
changelogs, changes, err := createChangelogs(users, infos)
@@ -120,7 +134,7 @@ func userInfoDiff(user *model.Furry, info *pb.InfoReply_Data) (*model.Changelog,
return nil, nil, exception.ErrUserMismatch(fmt.Sprintf("对比用户ID不匹配：%d和%d", user.UID, info.Mid))
}
changelog := model.Changelog{
Operation: "UpdateName",
Operation: "",
Target: strconv.FormatUint(uint64(user.UID), 10),
Result: "",
}
@@ -129,9 +143,15 @@ func userInfoDiff(user *model.Furry, info *pb.InfoReply_Data) (*model.Changelog,
}
if user.Name != info.Name {
result, _ := json.Marshal(map[string]string{"previous_name": user.Name, "new_name": info.Name})
changelog.Operation = "UpdateName"
changelog.Result = string(result)
updated.Name = info.Name
return &changelog, &updated, nil
} else if info.LiveRoom != nil && user.LiveRoomID != uint(info.LiveRoom.Roomid) {
updated.LiveRoomID = uint(info.LiveRoom.Roomid)
changelog.Operation = "AddLiveRoom"
changelog.Result = strconv.FormatUint(info.LiveRoom.Roomid, 10)
return &changelog, &updated, nil
} else {
return nil, nil, nil
}
@@ -238,5 +258,9 @@ func UpdateFansToDB(stats []*pb.StatReply_Data) (int, error) {
return 0, err
}
return len(stats), nil
}
func AddUser(user *model.Furry) error {
tx := dao.DB().Create(user)
return tx.Error
}

1
pkg/task/config.toml
View File

@@ -10,3 +10,4 @@
[spider_core]
host = "localhost"
port = 9101
bili_cookie = "buvid3=E114B569-ECF7-50D6-5874-7221C956F4F323116infoc; i-wanna-go-back=-1; _uuid=9DC109D51-C164-D1A8-D18D-CDB410A6B733223200infoc; buvid4=2EFA9D48-F352-9314-066E-800EAD1A3B3024163-022061118-7XDfT9HnZ75xcCl6GNqbKvvAaotbbCqK1dff9AggS2AFysI2RsiDgg%3D%3D; CURRENT_BLACKGAP=0; blackside_state=0; nostalgia_conf=-1; buvid_fp_plain=undefined; b_ut=5; LIVE_BUVID=AUTO2216549597211800; hit-dyn-v2=1; b_nut=100; fingerprint3=8b45c9c8e2c3bde2334046651927bc11; CURRENT_QUALITY=80; hit-new-style-dyn=0; CURRENT_FNVAL=4048; rpdid=|(J~R|~)~|Y)0J'uYY)Y|JmkY; dy_spec_agreed=1; fingerprint=64f6f507fa149d279783afea30316e2b; SESSDATA=0312b6e2%2C1687167210%2C5c48e%2Ac1; bili_jct=12bdeb674ff87c20afd2d245128d6734; DedeUserID=90931399; DedeUserID__ckMd5=507c5d481d60b7bb; sid=5n2lvshx; buvid_fp=64f6f507fa149d279783afea30316e2b; bp_t_offset_90931399=742134893995098144; PVID=2; share_source_origin=QQ; bsource=share_source_qqchat; b_lsid=4578914A_1853786E484; innersign=0"

46
pkg/task/liveroom.go Normal file
View File

@@ -0,0 +1,46 @@
package task
//func UpdateLiveRoomInfo() {
// // 预查询数量
// // TODO: 预查询数量和分页策略功能可复用
// var count int64
// tx := dao.DB().Model(&model.Furry{}).Count(&count)
// if tx.Error != nil {
// err := exception.ErrFetchFurries("获取furries数量失败：" + tx.Error.Error())
// log.Logger().Errorf("获取furries数量失败：%s", err)
// return
// }
//
// pageSize := 32
// maxPage := int(math.Ceil(float64(count) / float64(pageSize)))
// for page := 0; page <= maxPage; page++ {
// // 获取目标用户列表（分页）
// users, err := logic.GetUsers(page, pageSize)
// if err != nil {
// log.Logger().Errorf("获取用户列表时发生错误：%s", err)
// continue
// }
// // 已获取完毕
// if users == nil {
// return
// }
// // 抽取uid列表
// var uids []uint
// for _, user := range users {
// uids = append(uids, user.UID)
// }
// // 通过API获取用户信息
// infos := logic.BatchGetUserInfo(&uids)
//
// // 更新live_rooms表
// c, err := logic.UpdateLiveRoom(infos)
// if err != nil {
// log.Logger().Errorf("更新数据时发生错误：%s", err)
// continue
// }
// log.Logger().Infof("成功更新了%d个用户信息", c)
// // delay
// time.Sleep(2 * time.Second)
// }
//
//}

141
pkg/task/search.go Normal file
View File

@@ -0,0 +1,141 @@
package task
import (
"github.com/eigeen/furryboard/spider-scheduler/pkg/conf"
"github.com/eigeen/furryboard/spider-scheduler/pkg/dao/model"
"github.com/eigeen/furryboard/spider-scheduler/pkg/log"
"github.com/eigeen/furryboard/spider-scheduler/pkg/logic"
"time"
)
// SearchUpsFromVideo
// 通过关键词进行视频搜索的方式获取用户
// 新用户即写入用户表，状态为StatusPending
//
// 策略：每10分钟抓取前3页（每页42条）视频，若抓取的内容中存在
// 一条视频与上次抓取内容的最后一条视频相同，则判断为已到达结尾
func SearchUpsFromVideo() {
// 获取最后一次抓取的记录
meta, err := logic.GetMetadata("SearchVideoCheckpoint")
if err != nil {
log.Logger().Warnf("获取上一次视频抓取记录失败：%s", err)
err = nil
}
var history = ""
if meta != nil {
history = meta.Value
}
var newHistory = ""
// 逐页获取判断
for page := 1; page <= 3; page++ {
videos, err := logic.SearchNewestVideos("furry", conf.Conf.SpiderCore.BiliCookie, 1)
if err != nil {
log.Logger().Errorf("搜索视频失败：%s", err)
return
}
if len(videos) == 0 {
log.Logger().Info("视频搜索已达末页")
break
}
// 第一次获取，将第一个视频bv号作为newHistory
if page == 1 {
newHistory = videos[0].Bvid
}
// 从videos列表抽取视频bv号 map索引
var bvids = make(map[string]int, 0)
for i, video := range videos {
bvids[video.Bvid] = i
}
// 先判断是否到达上一次结尾
var historyPos = 42
if i, ok := bvids[history]; ok {
historyPos = i
}
// 防越界
if len(videos) < historyPos {
historyPos = len(videos)
}
// 先判断再抽取uid列表，节省数据库资源
// 从videos列表抽取uid(mid) map去重
var uids = make(map[uint64]*model.Furry, 0)
for i := 0; i < historyPos; i++ {
uids[videos[i].Mid] = &model.Furry{
UID: uint(videos[i].Mid),
Name: videos[i].Author,
Status: model.StatusPendingVideo,
}
}
// 逐个判断用户是否存在于数据库，不存在则新增
for uid, user := range uids {
exists, err := logic.IsUserExistsByUID(uint(uid))
if err != nil {
log.Logger().Warnf("判断用户存在性时发生错误：%s", err)
continue
}
if !exists {
err = logic.AddUser(user)
if err != nil {
log.Logger().Warnf("新增用户[%d]时失败：%s", uid, err)
continue
}
log.Logger().Infof("通过视频搜索新增用户：%s(%d)", user.Name, uid)
}
}
// 已到达上一次结尾，中断循环，否则继续获取下一页
if historyPos != 42 {
break
}
time.Sleep(1 * time.Second)
}
// 更新history
err = logic.UpdateVideoCheckpoint(newHistory)
if err != nil {
log.Logger().Warnf("更新Metadata: SearchVideoCheckpoint时发生错误：%s", err)
}
}
func SearchUpsFromLiveRoom() {
// TODO: 多关键词可复用
// 先获取页数，后逐页获取
pageNum, err := logic.GetLiveRoomPageNum("furry", conf.Conf.SpiderCore.BiliCookie)
if err != nil {
log.Logger().Errorf("搜索直播间错误：%s", err)
return
}
for page := 1; page <= pageNum; page++ {
rooms, err := logic.SearchLiveRooms("furry", conf.Conf.SpiderCore.BiliCookie, uint(page))
if err != nil {
log.Logger().Errorf("搜索直播间错误：%s", err)
return
}
count, err := logic.UpdateLiveRoomInfo(rooms)
if err != nil {
log.Logger().Errorf("更新直播间信息错误：%s", err)
return
}
log.Logger().Infof("更新了%d个直播间信息", count)
time.Sleep(1 * time.Second)
}
// 先获取页数，后逐页获取
pageNum, err = logic.GetLiveRoomPageNum("冬聚", conf.Conf.SpiderCore.BiliCookie)
if err != nil {
log.Logger().Errorf("搜索直播间错误：%s", err)
return
}
for page := 1; page <= pageNum; page++ {
rooms, err := logic.SearchLiveRooms("冬聚", conf.Conf.SpiderCore.BiliCookie, uint(page))
if err != nil {
log.Logger().Errorf("搜索直播间错误：%s", err)
return
}
count, err := logic.UpdateLiveRoomInfo(rooms)
if err != nil {
log.Logger().Errorf("更新直播间信息错误：%s", err)
return
}
log.Logger().Infof("更新了%d个直播间信息", count)
time.Sleep(1 * time.Second)
}
}

13
pkg/task/search_test.go Normal file
View File

@@ -0,0 +1,13 @@
package task
import "testing"
func TestSearchUntrackedUps(t *testing.T) {
BeforeTesting()
SearchUpsFromVideo()
}
func TestSearchUpsFromLiveRoom(t *testing.T) {
BeforeTesting()
SearchUpsFromLiveRoom()
}

11
pkg/task/user.go
View File

@@ -27,7 +27,7 @@ func UpdateUserInfo() {
maxPage := int(math.Ceil(float64(count) / float64(pageSize)))
for page := 0; page <= maxPage; page++ {
// 获取目标用户列表（分页）
users, err := logic.GetUsers(page, pageSize)
users, err := logic.GetValidUsers(page, pageSize)
if err != nil {
log.Logger().Errorf("获取用户列表时发生错误：%s", err)
continue
@@ -56,11 +56,12 @@ func UpdateUserInfo() {
}
}
// UpdateFans 更新粉丝数，将记录写入表中，增量记录
func UpdateFans() {
// TODO: 与UpdateUserInfo有大量重复，可抽离复用代码
// 预查询数量
var count int64
tx := dao.DB().Model(&model.Furry{}).Count(&count)
tx := dao.DB().Model(&model.Furry{}).Where("status = 0").Count(&count)
if tx.Error != nil {
err := exception.ErrFetchFurries("获取furries数量失败：" + tx.Error.Error())
log.Logger().Errorf("获取furries数量失败：%s", err)
@@ -71,7 +72,7 @@ func UpdateFans() {
maxPage := int(math.Ceil(float64(count) / float64(pageSize)))
for page := 0; page <= maxPage; page++ {
// 获取目标用户列表（分页）
users, err := logic.GetUsers(page, pageSize)
users, err := logic.GetValidUsers(page, pageSize)
if err != nil {
log.Logger().Errorf("获取用户列表时发生错误：%s", err)
continue
@@ -97,7 +98,3 @@ func UpdateFans() {
time.Sleep(2 * time.Second)
}
}
func UpdateFansAndInfo() {
}