Mirrors/bitmagnet
1
0
Fork 0
mirror of https://github.com/bitmagnet-io/bitmagnet.git synced 2025-12-28 06:34:17 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Reduce crawler persistence batch size (#121)

Browse Source
This commit is contained in:
mgdigital 2024-01-27 16:47:20 +00:00 committed by GitHub
parent 807a5946e9
commit b4e4e02fe3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/dhtcrawler/persist.go
View File

@ -40,7 +40,7 @@ func (c *crawler) runPersistTorrents(ctx context.Context) {
string(c.dao.Torrent.PieceLength.ColumnName()),
string(c.dao.Torrent.Pieces.ColumnName()),
}),
}).CreateInBatches(ts, 100); persistErr != nil {
}).CreateInBatches(ts, 20); persistErr != nil {
c.logger.Errorf("error persisting torrents: %s", persistErr)
} else {
c.persistedTotal.With(prometheus.Labels{"entity": "Torrent"}).Add(float64(len(ts)))
@ -152,7 +152,7 @@ func (c *crawler) runPersistSources(ctx context.Context) {
}
if persistErr := c.dao.WithContext(ctx).TorrentsTorrentSource.Clauses(clause.OnConflict{
UpdateAll: true,
}).CreateInBatches(srcs, 100); persistErr != nil {
}).CreateInBatches(srcs, 20); persistErr != nil {
c.logger.Errorf("error persisting torrent sources: %s", persistErr.Error())
} else {
c.persistedTotal.With(prometheus.Labels{"entity": "TorrentsTorrentSource"}).Add(float64(len(srcs)))