Compare commits

..

1 Commits

Author SHA1 Message Date
6273e833f8 testing 2024-10-06 03:23:44 -04:00
7 changed files with 223 additions and 293 deletions

View File

@@ -1,25 +0,0 @@
.PHONY: default venv build install clean
default: install
venv:
@[ ! -d ./venv ] && python3 -m venv venv && bash -c \
"source venv/bin/activate && \
pip install --upgrade pip && \
pip install -r requirements.txt" || true
build: venv
@if [ -n "$$(git status --porcelain)" ]; then \
echo "[ERROR]: There are uncommitted changes or untracked files."; \
exit 1; \
fi
@bash -c \
"source venv/bin/activate && \
pip install build twine && \
python -m build"
install: venv
@bash -c "source venv/bin/activate && pip install -e ."
clean:
rm -rf venv dist tarc.egg-info

View File

@@ -1,7 +1,4 @@
[project] [project]
name = "tarc" name = "tarc"
version = "0.0.1dev3" version = "0.0.1dev2"
description = "Manage BT archives" description = "Manage BT archives"
[project.scripts]
tarc = "tarc:main"

View File

@@ -1,2 +1 @@
python-qbittorrent==0.4.3 python-qbittorrent==0.4.3
sqlalchemy==2.0.27

20
tarc.sh Executable file
View File

@@ -0,0 +1,20 @@
#!/bin/bash
# Create a virtual environment if it does not exist
if [ ! -d "venv" ]; then
# shellcheck disable=SC1091
python3 -m venv venv && \
source venv/bin/activate && \
pip install -r requirements.txt && \
deactivate
fi
# Activate the virtual environment
# shellcheck disable=SC1091
source venv/bin/activate
# Run the Python script
python tarc/main.py "$@"
# Deactivate the virtual environment
deactivate

View File

@@ -1,5 +0,0 @@
"""
tarc - Manage BT archives
"""
from .main import main

View File

@@ -13,197 +13,250 @@ import sys
import re import re
import uuid import uuid
import argparse import argparse
import sqlite3
from datetime import datetime, timezone from datetime import datetime, timezone
import qbittorrent import qbittorrent
from sqlalchemy import create_engine, event
from sqlalchemy.orm import sessionmaker
from .models import Base, Client
# SCHEMA format is YYYYMMDDX # SCHEMA format is YYYYMMDDX
SCHEMA = 202410060 SCHEMA = 202410060
def init_db(engine):
def init_db(conn):
""" """
Initialize database Initialize database
""" """
# Create all tables first
Base.metadata.create_all(engine) c = conn.cursor()
c.executescript(
# Set the schema version using SQLAlchemy primitives f"""
@event.listens_for(engine, 'connect') PRAGMA user_version = {SCHEMA};
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor() CREATE TABLE IF NOT EXISTS clients (
cursor.execute(f"PRAGMA user_version = {SCHEMA}") id INTEGER PRIMARY KEY AUTOINCREMENT,
cursor.close() name TEXT NOT NULL UNIQUE,
uuid TEXT NOT NULL UNIQUE,
endpoint TEXT NOT NULL,
last_seen DATETIME NOT NULL
);
CREATE TABLE IF NOT EXISTS torrents (
id INTEGER PRIMARY KEY AUTOINCREMENT,
info_hash_v1 TEXT NOT NULL UNIQUE,
info_hash_v2 TEXT UNIQUE,
file_count INTEGER NOT NULL,
completed_on DATETIME NOT NULL
);
CREATE TABLE IF NOT EXISTS torrent_clients (
id INTEGER PRIMARY KEY AUTOINCREMENT,
torrent_id INTEGER NOT NULL,
client_id INTEGER NOT NULL,
name TEXT NOT NULL,
content_path TEXT NOT NULL,
last_seen DATETIME NOT NULL,
FOREIGN KEY (torrent_id) REFERENCES torrents(id),
FOREIGN KEY (client_id) REFERENCES clients(id),
UNIQUE (torrent_id, client_id)
);
CREATE TABLE IF NOT EXISTS trackers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
url TEXT NOT NULL UNIQUE,
last_seen DATETIME NOT NULL
);
CREATE TABLE IF NOT EXISTS torrent_trackers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
client_id INTEGER NOT NULL,
torrent_id INTEGER NOT NULL,
tracker_id INTEGER NOT NULL,
last_seen DATETIME NOT NULL,
FOREIGN KEY (client_id) REFERENCES clients(id),
FOREIGN KEY (torrent_id) REFERENCES torrents(id),
FOREIGN KEY (tracker_id) REFERENCES trackers(id),
UNIQUE (client_id, torrent_id, tracker_id)
);
CREATE TABLE IF NOT EXISTS files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
size INTEGER NOT NULL,
oshash TEXT NOT NULL UNIQUE,
hash TEXT UNIQUE
);
CREATE TABLE IF NOT EXISTS torrent_files (
id INTEGER PRIMARY KEY AUTOINCREMENT,
file_id INTEGER NOT NULL,
torrent_id INTEGER NOT NULL,
client_id INTEGER NOT NULL,
file_index INTEGER NOT NULL,
file_path TEXT NOT NULL,
is_downloaded BOOLEAN NOT NULL,
last_checked DATETIME NOT NULL,
FOREIGN KEY (file_id) REFERENCES files(id),
FOREIGN KEY (torrent_id) REFERENCES torrents(id),
FOREIGN KEY (client_id) REFERENCES clients(id),
UNIQUE (file_id, torrent_id, client_id, file_index)
);
"""
)
conn.commit()
c.close()
def list_tables(engine): def list_tables(conn):
""" """
List all tables in database List all tables in database
""" """
return Base.metadata.tables.keys() c = conn.cursor()
c.execute("SELECT name FROM sqlite_master WHERE type='table';")
table_list = c.fetchall()
c.close()
return [table[0] for table in table_list]
def add_client(session, name, endpoint, last_seen): def add_client(conn, name, endpoint, last_seen):
""" """
Add a new client endpoint to database Add a new client endpoint to database
""" """
new_client = Client( c = conn.cursor()
uuid=str(uuid.uuid4()), c.execute(
name=name, f"""
endpoint=endpoint, INSERT INTO clients (uuid, name, endpoint, last_seen)
last_seen=last_seen VALUES ("{uuid.uuid4()}", "{name}", "{endpoint}", "{last_seen}");
"""
) )
session.add(new_client) conn.commit()
session.commit() c.close()
def find_client(session, endpoint): def find_client(conn, endpoint):
""" """
Find existing client Find existing client
""" """
clients = session.query(Client.id, Client.name, Client.uuid).filter(Client.endpoint == endpoint).all() c = conn.cursor()
return clients c.execute(f'SELECT id, name, uuid FROM clients WHERE endpoint="{endpoint}";')
response = c.fetchall()
c.close()
return response
def list_clients(session): def list_clients(conn):
""" """
List all stored clients List all stored clients
""" """
return session.query(Client).all() c = conn.cursor()
c.execute("SELECT * FROM clients;")
rows = c.fetchall()
c.close()
return rows
def main(): parser = argparse.ArgumentParser(description="Manage BT archives", prog="tarc")
""" subparsers = parser.add_subparsers(
Entrypoint of the program. dest="command", required=True, help="Available commands"
""" )
parser = argparse.ArgumentParser(description="Manage BT archives", prog="tarc") scan_parser = subparsers.add_parser("scan", help="Scan command")
subparsers = parser.add_subparsers( scan_parser.add_argument("--debug", action="store_true", help="Enable debug mode")
dest="command", required=True, help="Available commands" scan_parser.add_argument(
) "--confirm-add", action="store_true", help="Confirm adding a new client"
)
scan_parser.add_argument("-n", "--name", help="Name of client")
scan_parser.add_argument("-d", "--directory", help="Directory to scan")
scan_parser.add_argument("-t", "--type", help="Scan type")
scan_parser.add_argument("-e", "--endpoint", help="Endpoint URL")
scan_parser.add_argument("-u", "--username", help="Username")
scan_parser.add_argument("-p", "--password", help="Password")
scan_parser.add_argument("-s", "--storage", help="Path of sqlite3 database")
scan_parser = subparsers.add_parser("scan", help="Scan command") args = parser.parse_args()
scan_parser.add_argument("--debug", action="store_true", help="Enable debug mode")
scan_parser.add_argument(
"--confirm-add", action="store_true", help="Confirm adding a new client"
)
scan_parser.add_argument("-n", "--name", help="Name of client")
scan_parser.add_argument("-d", "--directory", help="Directory to scan")
scan_parser.add_argument("-t", "--type", help="Scan type")
scan_parser.add_argument("-e", "--endpoint", help="Endpoint URL")
scan_parser.add_argument("-u", "--username", help="Username")
scan_parser.add_argument("-p", "--password", help="Password")
scan_parser.add_argument("-s", "--storage", help="Path of sqlite3 database")
args = parser.parse_args() if args.command == "scan":
if args.storage is None:
if args.command == "scan": STORAGE = os.path.expanduser("~/.tarch.db")
if args.storage is None: else:
STORAGE = os.path.expanduser("~/.tarch.db") STORAGE = args.storage
try:
sqlitedb = sqlite3.connect(STORAGE)
tables = list_tables(sqlitedb)
except sqlite3.DatabaseError as e:
print(f'[ERROR]: Database Error "{STORAGE}" ({str(e)})')
sys.exit(1)
if len(tables) == 0:
print(f"[INFO]: Initializing database at {STORAGE}")
init_db(sqlitedb)
cursor = sqlitedb.cursor()
cursor.execute("PRAGMA user_version;")
SCHEMA_FOUND = cursor.fetchone()[0]
cursor.close()
if not SCHEMA == SCHEMA_FOUND:
print(f"[ERROR]: SCHEMA {SCHEMA_FOUND}, expected {SCHEMA}")
sys.exit(1)
if not args.directory is None:
print("[INFO]: --directory is not implemented")
sys.exit(0)
elif not args.endpoint is None:
qb = qbittorrent.Client(args.endpoint)
if qb.qbittorrent_version is None:
print(f'[ERROR]: Couldn\'t find client version at "{args.endpoint}"')
sys.exit(1)
elif not re.match(r"^v?\d+(\.\d+)*$", qb.qbittorrent_version):
print(f'[ERROR]: Invalid version found at "{args.endpoint}"')
if args.debug:
print(f"[DEBUG]: {qb.qbittorrent_version}")
sys.exit(1)
else: else:
STORAGE = args.storage print(
try: f'[INFO]: Found qbittorrent {qb.qbittorrent_version} at "{args.endpoint}"'
engine = create_engine(f"sqlite:///{STORAGE}") )
tables = list_tables(engine) clients = find_client(sqlitedb, args.endpoint)
Session = sessionmaker(bind=engine) if args.confirm_add:
session = Session() if len(clients) == 0:
except Exception as e: if not args.name is None:
print(f'[ERROR]: Database Error "{STORAGE}" ({str(e)})') now = datetime.now(timezone.utc).isoformat(
sys.exit(1) sep=" ", timespec="seconds"
if len(tables) == 0:
print(f"[INFO]: Initializing database at {STORAGE}")
init_db(engine)
# Check schema version using SQLAlchemy primitives
schema_found = None
@event.listens_for(engine, 'connect')
def get_sqlite_pragma(dbapi_connection, connection_record):
nonlocal schema_found
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA user_version")
schema_found = cursor.fetchone()[0]
cursor.close()
# Force a connection to trigger the event
with engine.connect() as conn:
pass
if not SCHEMA == schema_found:
print(f"[ERROR]: SCHEMA {schema_found}, expected {SCHEMA}")
sys.exit(1)
if not args.directory is None:
print("[INFO]: --directory is not implemented")
sys.exit(0)
elif not args.endpoint is None:
qb = qbittorrent.Client(args.endpoint)
if args.username and args.password:
qb.login(args.username, args.password)
if qb.qbittorrent_version is None:
print(f'[ERROR]: Couldn\'t find client version at "{args.endpoint}"')
sys.exit(1)
elif not re.match(r"^v?\d+(\.\d+)*$", qb.qbittorrent_version):
print(f'[ERROR]: Invalid version found at "{args.endpoint}"')
if args.debug:
print(f"[DEBUG]: {qb.qbittorrent_version}")
sys.exit(1)
else:
print(
f'[INFO]: Found qbittorrent {qb.qbittorrent_version} at "{args.endpoint}"'
)
clients = find_client(session, args.endpoint)
if args.confirm_add:
if len(clients) == 0:
if not args.name is None:
now = datetime.now(timezone.utc)
add_client(session, args.name, args.endpoint, now)
print(f"[INFO]: Added client {args.name} ({args.endpoint})")
else:
print("[ERROR]: Must specify --name for a new client")
sys.exit(1)
elif len(clients) == 1:
print(f"[ERROR]: {clients[0][1]} ({clients[0][2]}) already exists")
sys.exit(1)
else:
print(
f"[ERROR]: Multiple clients with the same endpoint: {args.endpoint}"
) )
add_client(sqlitedb, args.name, args.endpoint, now)
print(f"[INFO]: Added client {args.name} ({args.endpoint})")
else:
print("[ERROR]: Must specify --name for a new client")
sys.exit(1) sys.exit(1)
elif len(clients) == 0:
print(f'[ERROR]: Client using endpoint "{args.endpoint}" not found')
print("[ERROR]: Use --confirm-add to add a new endpoint")
sys.exit(1)
elif len(clients) == 1: elif len(clients) == 1:
torrents = qb.torrents() print(f"[ERROR]: {clients[0][1]} ({clients[0][2]}) already exists")
print(f"[INFO]: There are {len(torrents)} torrents\n") sys.exit(1)
for torrent in torrents[:2]:
files = qb.get_torrent_files(torrent["hash"])
trackers = qb.get_torrent_trackers(torrent["hash"])
print(f"[name]: {torrent['name']}")
print(f"[infohash_v1]: {torrent['infohash_v1']}")
print(f"[content_path]: {torrent['content_path']}")
print(f"[magent_uri]: {torrent['magnet_uri'][0:80]}")
print(f"[completed_on]: {torrent['completed']}")
print(f"[trackers]: {len(trackers)}")
print(f"[file_count]: {len(files)}\n")
if args.debug:
print(f"[DEBUG]: {repr(torrent)}")
for elem in trackers:
print(f"[DEBUG]: Tracker {repr(elem)}")
print("\n", end="")
else: else:
print( print(
f'[ERROR]: Multiple clients ({len(clients)}) using "{args.endpoint}"' f"[ERROR]: Multiple clients with the same endpoint: {args.endpoint}"
) )
sys.exit(1) sys.exit(1)
else: elif len(clients) == 0:
print("[ERROR]: Must specify directory OR client endpoint") print(f'[ERROR]: Client using endpoint "{args.endpoint}" not found')
print("[ERROR]: Use --confirm-add to add a new endpoint")
sys.exit(1) sys.exit(1)
elif len(clients) == 1:
torrents = qb.torrents()
if __name__ == "__main__": print(f"[INFO]: There are {len(torrents)} torrents\n")
main() for torrent in torrents[:2]:
files = qb.get_torrent_files(torrent["hash"])
trackers = qb.get_torrent_trackers(torrent["hash"])
print(f"[name]: {torrent['name']}")
print(f"[infohash_v1]: {torrent['infohash_v1']}")
print(f"[content_path]: {torrent['content_path']}")
print(f"[magent_uri]: {torrent['magnet_uri'][0:80]}")
print(f"[completed_on]: {torrent['completed']}")
print(f"[trackers]: {len(trackers)}")
print(f"[file_count]: {len(files)}\n")
if args.debug:
print(f"[DEBUG]: {repr(torrent)}")
for elem in trackers:
print(f"[DEBUG]: Tracker {repr(elem)}")
print("\n", end="")
else:
print(f'[ERROR]: Multiple clients ({len(clients)}) using "{args.endpoint}"')
sys.exit(1)
else:
print("[ERROR]: Must specify directory OR client endpoint")
sys.exit(1)

View File

@@ -1,109 +0,0 @@
#!/usr/bin/env python3
"""
Database models for Torrent Archiver
"""
from sqlalchemy import Column, Integer, String, Boolean, DateTime, ForeignKey, UniqueConstraint
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
Base = declarative_base()
class Client(Base):
__tablename__ = 'clients'
id = Column(Integer, primary_key=True, autoincrement=True)
name = Column(String, nullable=False, unique=True)
uuid = Column(String, nullable=False, unique=True)
endpoint = Column(String, nullable=False)
last_seen = Column(DateTime, nullable=False)
torrent_clients = relationship("TorrentClient", back_populates="client")
torrent_trackers = relationship("TorrentTracker", back_populates="client")
torrent_files = relationship("TorrentFile", back_populates="client")
class Torrent(Base):
__tablename__ = 'torrents'
id = Column(Integer, primary_key=True, autoincrement=True)
info_hash_v1 = Column(String, nullable=False, unique=True)
info_hash_v2 = Column(String, unique=True)
file_count = Column(Integer, nullable=False)
completed_on = Column(DateTime, nullable=False)
torrent_clients = relationship("TorrentClient", back_populates="torrent")
torrent_trackers = relationship("TorrentTracker", back_populates="torrent")
torrent_files = relationship("TorrentFile", back_populates="torrent")
class TorrentClient(Base):
__tablename__ = 'torrent_clients'
id = Column(Integer, primary_key=True, autoincrement=True)
torrent_id = Column(Integer, ForeignKey('torrents.id'), nullable=False)
client_id = Column(Integer, ForeignKey('clients.id'), nullable=False)
name = Column(String, nullable=False)
content_path = Column(String, nullable=False)
last_seen = Column(DateTime, nullable=False)
torrent = relationship("Torrent", back_populates="torrent_clients")
client = relationship("Client", back_populates="torrent_clients")
__table_args__ = (UniqueConstraint('torrent_id', 'client_id'),)
class Tracker(Base):
__tablename__ = 'trackers'
id = Column(Integer, primary_key=True, autoincrement=True)
url = Column(String, nullable=False, unique=True)
last_seen = Column(DateTime, nullable=False)
torrent_trackers = relationship("TorrentTracker", back_populates="tracker")
class TorrentTracker(Base):
__tablename__ = 'torrent_trackers'
id = Column(Integer, primary_key=True, autoincrement=True)
client_id = Column(Integer, ForeignKey('clients.id'), nullable=False)
torrent_id = Column(Integer, ForeignKey('torrents.id'), nullable=False)
tracker_id = Column(Integer, ForeignKey('trackers.id'), nullable=False)
last_seen = Column(DateTime, nullable=False)
client = relationship("Client", back_populates="torrent_trackers")
torrent = relationship("Torrent", back_populates="torrent_trackers")
tracker = relationship("Tracker", back_populates="torrent_trackers")
__table_args__ = (UniqueConstraint('client_id', 'torrent_id', 'tracker_id'),)
class File(Base):
__tablename__ = 'files'
id = Column(Integer, primary_key=True, autoincrement=True)
size = Column(Integer, nullable=False)
oshash = Column(String, nullable=False, unique=True)
hash = Column(String, unique=True)
torrent_files = relationship("TorrentFile", back_populates="file")
class TorrentFile(Base):
__tablename__ = 'torrent_files'
id = Column(Integer, primary_key=True, autoincrement=True)
file_id = Column(Integer, ForeignKey('files.id'), nullable=False)
torrent_id = Column(Integer, ForeignKey('torrents.id'), nullable=False)
client_id = Column(Integer, ForeignKey('clients.id'), nullable=False)
file_index = Column(Integer, nullable=False)
file_path = Column(String, nullable=False)
is_downloaded = Column(Boolean, nullable=False)
last_checked = Column(DateTime, nullable=False)
file = relationship("File", back_populates="torrent_files")
torrent = relationship("Torrent", back_populates="torrent_files")
client = relationship("Client", back_populates="torrent_files")
__table_args__ = (UniqueConstraint('file_id', 'torrent_id', 'client_id', 'file_index'),)