Browse Source

Initial commit

master 1.0.0
noirscape 1 year ago
commit
4589c9c1b5
7 changed files with 716 additions and 0 deletions
  1. +353
    -0
      .gitignore
  2. +25
    -0
      yiff.party.dl.sln
  3. +42
    -0
      yiff.party.dl/ContentExporter.cs
  4. +175
    -0
      yiff.party.dl/CreatorDownloader.cs
  5. +55
    -0
      yiff.party.dl/JsonSchema.cs
  6. +52
    -0
      yiff.party.dl/Program.cs
  7. +14
    -0
      yiff.party.dl/yiff.party.dl.csproj

+ 353
- 0
.gitignore View File

@ -0,0 +1,353 @@
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/master/VisualStudio.gitignore
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Mono auto generated files
mono_crash.*
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUnit
*.VisualState.xml
TestResult.xml
nunit-*.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Coverlet is a free, cross platform Code Coverage Tool
coverage*[.json, .xml, .info]
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# NuGet Symbol Packages
*.snupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
*.appxbundle
*.appxupload
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
*- [Bb]ackup.rdl
*- [Bb]ackup ([0-9]).rdl
*- [Bb]ackup ([0-9][0-9]).rdl
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# BeatPulse healthcheck temp database
healthchecksdb
# Backup folder for Package Reference Convert tool in Visual Studio 2017
MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/

+ 25
- 0
yiff.party.dl.sln View File

@ -0,0 +1,25 @@

Microsoft Visual Studio Solution File, Format Version 12.00
# Visual Studio Version 16
VisualStudioVersion = 16.0.29613.14
MinimumVisualStudioVersion = 10.0.40219.1
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "yiff.party.dl", "yiff.party.dl\yiff.party.dl.csproj", "{D55DF598-706D-4FD7-941A-EB720695907C}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
Release|Any CPU = Release|Any CPU
EndGlobalSection
GlobalSection(ProjectConfigurationPlatforms) = postSolution
{D55DF598-706D-4FD7-941A-EB720695907C}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{D55DF598-706D-4FD7-941A-EB720695907C}.Debug|Any CPU.Build.0 = Debug|Any CPU
{D55DF598-706D-4FD7-941A-EB720695907C}.Release|Any CPU.ActiveCfg = Release|Any CPU
{D55DF598-706D-4FD7-941A-EB720695907C}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
GlobalSection(SolutionProperties) = preSolution
HideSolutionNode = FALSE
EndGlobalSection
GlobalSection(ExtensibilityGlobals) = postSolution
SolutionGuid = {317DA2C4-C93A-4801-89F8-943A69BB3B1A}
EndGlobalSection
EndGlobal

+ 42
- 0
yiff.party.dl/ContentExporter.cs View File

@ -0,0 +1,42 @@
using Konsole;
using System;
using System.ComponentModel;
using System.IO;
using System.Net;
using System.Text.Json;
public class ContentExporter
{
public static void DownloadURLToFile(string url, string out_path)
{
static bool DownloadTry(string url, string out_path)
{
try
{
using (WebClient client = new WebClient())
{
client.DownloadFile(new Uri(url), out_path);
}
}
catch (Exception)
{
return false;
}
return true;
}
bool success = false;
while (!success)
{
success = DownloadTry(url, out_path);
}
}
public static void SerializeObjectToJson(object obj, string out_path)
{
string jsonString = JsonSerializer.Serialize(obj);
File.WriteAllText(out_path, jsonString);
}
}

+ 175
- 0
yiff.party.dl/CreatorDownloader.cs View File

@ -0,0 +1,175 @@
using Konsole;
using Newtonsoft.Json;
using System;
using System.IO;
using System.Net;
using yiff.party.dl.json;
public class CreatorDownloader
{
private bool one_dir;
private bool save_json;
private string creator_id;
private string out_dir;
private Rootobject json;
private string json_data;
public CreatorDownloader(bool one_dir, bool save_json, string creator_id, string out_dir)
{
this.one_dir = one_dir;
this.save_json = save_json;
this.creator_id = creator_id;
if (out_dir == "out")
{
this.out_dir = creator_id;
}
else
{
this.out_dir = out_dir;
}
}
public void DownloadCreator()
{
FetchJson();
if (json.shared_files != null && json.shared_files.Length != 0)
FetchSharedFiles();
if (json.posts != null && json.posts.Length != 0)
FetchPosts();
ExportJson();
}
private void FetchJson()
{
using (var w = new WebClient())
{
string json_data = string.Empty;
string url = $"https://yiff.party/{creator_id}.json";
Console.WriteLine($"Downloading JSON: {url}");
try
{
json_data = w.DownloadString(url);
}
catch (Exception) { }
this.json_data = json_data;
json = !string.IsNullOrEmpty(json_data) ? JsonConvert.DeserializeObject<Rootobject>(json_data) : new Rootobject();
}
}
private string MakeOutDirectory()
{
DirectoryInfo directory = Directory.CreateDirectory(this.out_dir);
return directory.FullName;
}
private string MakeSubOutdirectory(string subdir)
{
DirectoryInfo directory = Directory.CreateDirectory(this.out_dir + "/" + subdir);
return directory.FullName;
}
private void FetchSharedFiles()
{
ProgressBarSlim overall_progress = new ProgressBarSlim(json.shared_files.Length);
int overal_progress_cnt = 0;
foreach (Shared_Files shared_file in json.shared_files) {
overall_progress.Refresh(overal_progress_cnt, "Saving shared files...");
overal_progress_cnt++;
SaveIndividualSharedFile(shared_file);
}
overall_progress.Refresh(overal_progress_cnt, "Finished downloading shared files.");
}
private void SaveIndividualSharedFile(Shared_Files shared_file)
{
string path = GetPath(PossibleResponseTypes.SharedFile, shared_file.id.ToString());
string file_path = path + $"/{shared_file.id}-{shared_file.file_name}";
string json_path = path + $"/{shared_file.id}.json";
ContentExporter.DownloadURLToFile(shared_file.file_url, file_path);
ContentExporter.SerializeObjectToJson(shared_file, json_path);
}
private void FetchPosts()
{
ProgressBarSlim overall_progress = new ProgressBarSlim(json.posts.Length);
int overal_progress_cnt = 0;
foreach (Post post in json.posts)
{
overall_progress.Refresh(overal_progress_cnt, "Saving posts...");
overal_progress_cnt++;
SaveIndividualPost(post);
}
overall_progress.Refresh(overal_progress_cnt, "Finished downloading posts.");
}
private void SaveIndividualPost(Post post)
{
// Setting up total progress length
int total_progress = post.attachments.Length;
// Add one for the post file
total_progress += 1;
// Add one more for the json
if (!one_dir)
total_progress += 1;
// Common base path
string path = GetPath(PossibleResponseTypes.Post, post.id.ToString());
ProgressBarSlim post_progress = new ProgressBarSlim(total_progress);
int post_progress_cnt = 0;
foreach (Attachment attachment in post.attachments)
{
post_progress.Refresh(post_progress_cnt, $"Saving post attachment [ID:{post.id}] [F:{attachment.file_name}]");
ContentExporter.DownloadURLToFile(attachment.file_url, path + $"/{attachment.id}-{attachment.file_name}");
post_progress_cnt++;
}
post_progress.Refresh(post_progress_cnt, $"Saving post file [ID:{post.id}] [F:{post.post_file.file_name}]");
ContentExporter.DownloadURLToFile(post.post_file.file_url, path + $"/{post.post_file.file_name}");
post_progress_cnt++;
if (!one_dir)
{
post_progress.Refresh(post_progress_cnt, $"Saving post json [ID:{post.id}]");
ContentExporter.SerializeObjectToJson(post, path + $"/{post.id}.json");
post_progress_cnt++;
}
post_progress.Refresh(post_progress_cnt, $"Done saving post [ID:{post.id}]");
}
private void ExportJson()
{
MakeOutDirectory();
ContentExporter.SerializeObjectToJson(json, this.out_dir + $"/{this.creator_id}.json");
}
private string GetPath(PossibleResponseTypes type, string suffix)
{
if (this.one_dir)
{
return MakeOutDirectory();
}
string base_element = "";
if (type == PossibleResponseTypes.Post)
{
base_element = "posts";
}
if (type == PossibleResponseTypes.SharedFile)
{
base_element = "shared-files";
}
return MakeSubOutdirectory(this.out_dir + "/" + base_element + "/" + suffix);
}
}
public enum PossibleResponseTypes
{
SharedFile,
Post
}

+ 55
- 0
yiff.party.dl/JsonSchema.cs View File

@ -0,0 +1,55 @@
using System;
namespace yiff.party.dl.json
{
public class Rootobject
{
public Post[] posts { get; set; }
public Shared_Files[] shared_files { get; set; }
}
public class Post
{
public Attachment[] attachments { get; set; }
public string body { get; set; }
public int created { get; set; }
public Embed embed { get; set; }
public int id { get; set; }
public Post_File post_file { get; set; }
public string title { get; set; }
}
public class Embed
{
public object description { get; set; }
public object html { get; set; }
public string provider { get; set; }
public string provider_url { get; set; }
public object subject { get; set; }
public string url { get; set; }
}
public class Post_File
{
public string file_name { get; set; }
public string file_url { get; set; }
}
public class Shared_Files
{
public string description { get; set; }
public string file_name { get; set; }
public string file_url { get; set; }
public long id { get; set; }
public string title { get; set; }
public int uploaded { get; set; }
}
public class Attachment
{
public string file_name { get; set; }
public string file_url { get; set; }
public long id { get; set; }
}
}

+ 52
- 0
yiff.party.dl/Program.cs View File

@ -0,0 +1,52 @@
using Konsole;
using System;
using System.Collections.Generic;
namespace yiff.party.dl
{
class Program
{
private const string usage = @"yiff.party.dl.exe
Usage:
yiff.party.dl.exe download-creator [--out-dir OUT | -o OUT] [--save-json | -j ] [--one-dir | -1] <creator_id>
yiff.party.dl.exe download-creators <creator_id>...
yiff.party.dl.exe --help
Options:
--out-dir OUT, -o OUT The directory to save to. If set to 'out' (default), use creator ID. [default: out]
--save-json, -j Whether to keep the JSON or not.
--one-dir, -1 Save everything into one directory (so no extra file structure will be created).
";
static void Main(string[] args)
{
IDictionary<string, DocoptNet.ValueObject> arguments = new DocoptNet.Docopt().Apply(usage, args, help: true, exit: true);
if (arguments["download-creator"].IsTrue)
{
Console.WriteLine("Downloading a single creator.");
CreatorDownloader downloader = new CreatorDownloader(
arguments["--one-dir"].IsTrue,
arguments["--save-json"].IsTrue,
arguments["<creator_id>"].AsList[0].ToString(),
arguments["--out-dir"].ToString());
downloader.DownloadCreator();
}
if (arguments["download-creators"].IsTrue)
{
Console.WriteLine("Downloading multiple creators.");
foreach (var cid in arguments["<creator_id>"].AsList)
{
string creator_id = cid.ToString();
CreatorDownloader downloader = new CreatorDownloader(
false, true, creator_id, "out");
downloader.DownloadCreator();
}
}
}
}
}

+ 14
- 0
yiff.party.dl/yiff.party.dl.csproj View File

@ -0,0 +1,14 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>netcoreapp3.1</TargetFramework>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="docopt.net" Version="0.6.1.10" />
<PackageReference Include="Goblinfactory.Konsole" Version="5.0.1" />
<PackageReference Include="Newtonsoft.Json" Version="12.0.3" />
</ItemGroup>
</Project>