Browse Source

Initial commit.

master
noirscape 2 years ago
commit
5310a2e825
Signed by: noirscape GPG Key ID: 38216818B8894B9F
5 changed files with 109 additions and 0 deletions
  1. +1
    -0
      .gitignore
  2. +8
    -0
      README.md
  3. +83
    -0
      main.py
  4. +2
    -0
      requirements.txt
  5. +15
    -0
      setup.py

+ 1
- 0
.gitignore View File

@ -0,0 +1 @@
venv

+ 8
- 0
README.md View File

@ -0,0 +1,8 @@
## Yiff-Party-DL
A clean and simple yiff.party content scraper.
## Usage
- Install with setup.py (can be installed with pip)
- Run `yiff-party-dl` from the command line for options

+ 83
- 0
main.py View File

@ -0,0 +1,83 @@
import click
import requests
import json
import os
import shutil
def download_file(url, filename):
with requests.get(url, stream=True) as r:
r.raw.decode_content = True
with open(filename, 'wb') as f:
shutil.copyfileobj(r.raw, f)
def progress_show(preface, data):
if data:
return f"{preface} - {data['id']}"
else:
return f"{preface} - No data yet."
def progress_show_post(data):
return progress_show('Post', data)
def progress_show_shared_file(data):
return progress_show('Shared File', data)
@click.group()
def cli():
pass
@cli.command()
@click.argument("CREATOR-ID", type=click.INT)
@click.option("--save-json", is_flag=True, help="Save the yiff.party JSON data. With one-dir, only stores if you pass this. With multiple-dirs, JSON data will always be made for individual posts and this just creates a global json file.")
@click.option("--one-dir/--multiple-dirs", help="Pass --one-dir to store everything in the same directory. --multiple-dirs is selected by default and creates a file structure.")
def download_creator(creator_id, save_json, one_dir):
"""Download posts by specified CREATOR-ID"""
click.echo("[Downloading json...]")
r = requests.get(f"https://yiff.party/{creator_id}.json")
json_data = r.json()
click.echo("[Resulting data]")
click.echo(f"[{len(json_data['posts'])} posts] and [{len(json_data['shared_files'])} shared files]")
click.echo(f"[Downloading posts...]")
with click.progressbar(json_data["posts"], item_show_func=progress_show_post) as posts:
for post in posts:
if one_dir:
out_path = f"{post['id']}-"
else:
out_path = f"posts/{post['id']}/"
os.makedirs(out_path, exist_ok=True)
with open(f"{out_path}{post['id']}.json", "w") as json_file:
json.dump(post, json_file)
download_file(post['post_file']['file_url'], out_path + post['post_file']['file_name'])
for attach in post['attachments']:
download_file(attach['file_url'], out_path + f"{attach['id']}-{attach['file_name']}")
click.echo("[Downloading shared files...]")
with click.progressbar(json_data["shared_files"], item_show_func=progress_show_shared_file) as shared_files:
for sfile in shared_files:
if one_dir:
out_path = f"{sfile['id']}-"
else:
out_path = f"shared_files/{sfile['id']}/"
with open(f"{out_path}{sdata['id']}.json", "w") as json_file:
json.dump(post, json_file)
download_file(sfile['file_url'], out_path + f"{sfile['file_name']}")
if save_json:
click.echo("[Storing creator json]")
with open(f"{creator_id}.json", "w") as json_file:
json.dump(json_data, json_file)
click.echo("[Done.]")
@cli.command()
@click.argument("CREATOR_IDs", nargs=-1, metavar='CREATOR_ID', required=True)
@click.pass_context
def download_creators(ctx, creator_ids):
"""Download all CREATOR-IDs you pass in."""
cur_cwd = os.getcwd()
for creator_id in creator_ids:
os.makedirs(creator_id, exist_ok=True)
os.chdir(cur_cwd + f"/{creator_id}")
ctx.invoke(download_creator, one_dir=False, creator_id=creator_id, save_json=True)
os.chdir(cur_cwd)
if __name__ == "__main__":
cli()

+ 2
- 0
requirements.txt View File

@ -0,0 +1,2 @@
requests
click

+ 15
- 0
setup.py View File

@ -0,0 +1,15 @@
from setuptools import setup
setup(
name='yiff-party-dl',
version='1.0.0',
py_modules=['main'],
install_requires=[
'click',
'requests'
],
entry_points='''
[console_scripts]
yiff-party-dl=main:cli
''',
)