alihan/torrent-paradise-torpar
1
0
Fork 0
mirror of https://github.com/urbanguacamole/torrent-paradise.git synced 2022-02-03 00:44:15 +03:00
Code Issues Projects Releases Wiki Activity

Remove import TPB dump tool

Browse Source 
The Pirate Bay no longer provides dumps AFAIK.

Commit 8fdc8d6 added crawling of top 48h TPB torrents list.
This commit is contained in:
Urban Guacamole
2020-07-05 14:04:35 +02:00
parent 8fdc8d6508
commit 3304722c76
3 changed files with 2 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -28,9 +28,9 @@ Here's what the setup looks like rn:
The programs create their own tables in the DB that they need. Database name is "nextgen".
What I did first after getting the server up and running was importing the TPB dump. Download https://thepiratebay.org/static/dump/csv/torrent_dump_full.csv.gz to the import-tpb-dump directory and run `go run`.
What I did first after getting the server up and running was importing the TPB dump. This is sadly no longer possible, since TPB stopped providing the dump.
There is a complete database dump available in torrentparadise-staticbackup.torrent, so you don't have to do that. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed.
There is a database dump available in torrentparadise-staticbackup.torrent. This same database dump is available on https://mega.nz/#!ddESlChb!3YBqfxG-a4fwpXzPG3QsXa-C6FeQ9AbNSGXxY7W7xm4. It contains the same data as the torrent, only .xz compressed.
# Usage

3
import-tpb-dump/deploy.sh
View File

@@ -1,3 +0,0 @@
go build
scp ./import-tpb-dump user@server:/home/nextgen/
ssh user@server sudo -u nextgen /home/nextgen/import-tpb-dump

84
import-tpb-dump/main.go
View File

@@ -1,84 +0,0 @@
package main
import (
"compress/gzip"
"database/sql"
"encoding/base64"
"encoding/csv"
"encoding/hex"
"io"
"log"
"os"
"time"
"github.com/lib/pq"
)
func main() {
f, err := os.Open("/home/nextgen/torrent_dump_full.csv.gz")
if err != nil {
log.Fatal(err)
}
defer f.Close()
gr, err := gzip.NewReader(f)
if err != nil {
log.Fatal(err)
}
defer gr.Close()
db := initDb()
cr := csv.NewReader(gr)
cr.LazyQuotes = true
cr.Comma = rune(';')
const layout = "2006-Jan-02 15:04:05"
if err != nil {
log.Fatal(err)
}
_, err = cr.Read() // read first line and throw it away
if err != nil {
log.Fatal(err)
}
for {
line, error := cr.Read()
if error == io.EOF {
break
} else if error != nil {
if perr, ok := err.(*csv.ParseError); ok && perr.Err == csv.ErrFieldCount {
log.Println(err)
}
}
added, err := time.Parse(layout, line[0])
if err != nil {
log.Println(err)
}
ihBytes, _ := base64.StdEncoding.DecodeString(line[1])
ih := hex.EncodeToString(ihBytes)
_, err = db.Exec("INSERT INTO torrent (infohash, name, length, added) VALUES ($1, $2, $3, $4)", ih, line[2], line[3], added)
if err, ok := err.(*pq.Error); ok { //dark magic
if err.Code != "23505" {
log.Fatal(err)
}
}
}
}
func initDb() *sql.DB {
connStr := "user=nextgen dbname=nextgen host=/var/run/postgresql"
db, err := sql.Open("postgres", connStr)
if err != nil {
log.Fatal(err)
}
_, err = db.Exec(`CREATE TABLE IF NOT EXISTS torrent (
infohash char(40) PRIMARY KEY NOT NULL,
name varchar NOT NULL,
length bigint,
added timestamp DEFAULT current_timestamp
)`)
if err != nil {
log.Fatal(err)
}
return db
}