Browse Source

Went a little overboard

existing-content-logger
noirscape 2 years ago
parent
commit
9bdea31ce5
Signed by: noirscape GPG Key ID: 38216818B8894B9F
9 changed files with 73 additions and 47 deletions
  1. +4
    -1
      .gitignore
  2. +0
    -4
      .vscode/settings.json
  3. +2
    -0
      existing_logger.py
  4. +14
    -0
      freeze.py
  5. +1
    -1
      logger_modules/db_ops.py
  6. +6
    -5
      logger_modules/existing_db_ops.py
  7. +3
    -1
      requirements.txt
  8. +40
    -28
      viewer.py
  9. +3
    -7
      viewer_modules/session_filter.py

+ 4
- 1
.gitignore View File

@ -136,4 +136,7 @@ static/attachments
static/avatars
# Logs
providence_existing.log
providence_existing.log
# VS Code
.vscode

+ 0
- 4
.vscode/settings.json View File

@ -1,4 +0,0 @@
{
"python.pythonPath": "venv/bin/python",
"python.linting.pylintArgs": ["--load-plugins", "pylint_flask", "--max-line-length=500"]
}

+ 2
- 0
existing_logger.py View File

@ -16,6 +16,7 @@ from sqlalchemy.orm import sessionmaker, scoped_session
import db
from logger_modules.existing_log import ExistingArchiveHandler
from logger_modules.existing_db_ops import ExistingDatabaseOperations
import logger_modules.init_funcs
@click.command()
@click.argument("ID", type=int)
@ -29,6 +30,7 @@ def main(id, log_type):
with open("config.yml", "r") as configfile:
config = yaml.safe_load(configfile)
logger_modules.init_funcs.make_static_dirs(config)
engine = create_engine(config["database_url"], pool_size=30)
LOGGER.info("Connected to database.")
db.Base.metadata.bind = engine


+ 14
- 0
freeze.py View File

@ -0,0 +1,14 @@
import viewer
import click
from flask_frozen import Freezer
viewer.app.config["FREEZER_RELATIVE_URLS"] = True
freezer = Freezer(viewer.app)
if __name__ == '__main__':
with click.progressbar(
freezer.freeze_yield(),
item_show_func=lambda p: p.url if p else 'Done!') as urls:
for url in urls:
pass

+ 1
- 1
logger_modules/db_ops.py View File

@ -69,7 +69,7 @@ class DatabaseOperations:
r = requests.get(str(attachment.url))
with open(f"static/attachments/{attachment.id}-{attachment.filename}", 'wb') as attachmentfile:
attachmentfile.write(r.content)
attachment_url = f"/static/attachments/{attachment.id}-{attachment.filename}.png"
attachment_url = f"/static/attachments/{attachment.id}-{attachment.filename}"
else:
attachment_url = str(attachment.url)
new_attachment = db.PrivateMessageAttachments(attachment_id=attachment.id, message_id=message.id,


+ 6
- 5
logger_modules/existing_db_ops.py View File

@ -36,8 +36,9 @@ class ExistingDatabaseOperations:
self.create_text_channel_if_not_exist(channel)
# Stage 2: Go over messages with create_message_if_not_exist
async for message in channel.history(limit=None):
self.create_message_if_not_exist(message, False)
if channel.permissions_for(channel.guild.me).read_message_history and channel.permissions_for(channel.guild.me).read_messages:
async for message in channel.history(limit=None):
self.create_message_if_not_exist(message, False)
async def store_dm_channel(self, user: discord.User):
# Stage 1: Create user
@ -145,7 +146,7 @@ class ExistingDatabaseOperations:
msg_embed = embed.get_rich_embed(message)
if message.edited_at:
with session_scope(self.sessionmaker) as session:
edited_exists = session.query(exists().where((msg_edit.content == message.content) & (msg_edit.message_id == message.id))).scalar()
edited_exists = session.query(exists().where((msg_edit.content == message.clean_content) & (msg_edit.message_id == message.id))).scalar()
LOGGER.info("Message with ID %s stored edit message status: %s", message.id, edited_exists)
else:
edited_exists = True
@ -178,7 +179,7 @@ class ExistingDatabaseOperations:
if not edited_exists:
with session_scope(self.sessionmaker) as session:
LOGGER.info("Message with ID %s a dummy edit.", message.id)
session.add(msg_edit(message_id=message.id, content=message.content, embed=msg_embed,
session.add(msg_edit(message_id=message.id, content=message.clean_content, embed=msg_embed,
edit_time=message.edited_at))
def create_attachment_if_not_exist(self, attachment: discord.Attachment, is_dm: bool, message_id: int):
@ -196,7 +197,7 @@ class ExistingDatabaseOperations:
r = requests.get(str(attachment.url))
with open(f"static/attachments/{attachment.id}-{attachment.filename}", 'wb') as attachmentfile:
attachmentfile.write(r.content)
attachment_url = f"/static/attachments/{attachment.id}-{attachment.filename}.png"
attachment_url = f"/static/attachments/{attachment.id}-{attachment.filename}"
else:
LOGGER.info("Not downloading attachment with ID %s", attachment.id)
attachment_url = str(attachment.url)


+ 3
- 1
requirements.txt View File

@ -8,4 +8,6 @@ pyyaml
requests
flask-wtf
Flask-Misaka
click
click
Flask-Breadcrumbs
Frozen-Flask

+ 40
- 28
viewer.py View File

@ -11,9 +11,13 @@ import viewer_modules.jinja_formatters
import viewer_modules.forms as forms
import viewer_modules.search
import viewer_modules.session_filter as session_filter
from flask_breadcrumbs import Breadcrumbs, register_breadcrumb
app = Flask(__name__)
# Initialize flask extensions
Misaka(app)
Breadcrumbs(app=app)
with open("config.yml") as configfile:
config = yaml.safe_load(configfile)
@ -104,15 +108,14 @@ def request_days(message_cls, channel_id):
"""
days = db_session.query(cast(message_cls.created_at, Date)).filter_by(channel_id=channel_id).distinct().all()
days = [day[0] for day in days]
channel = db_session.query(message_cls).filter_by(channel_id=channel_id).first().channel
return channel, days
return days
@app.teardown_appcontext
def shutdown_session(exception=None):
db_session.remove()
@register_breadcrumb(app, '.', '#root')
@app.route('/')
def view_root():
total_users = db_session.query(db.User).count()
@ -122,19 +125,21 @@ def view_root():
return render_template("root.html", total_users=total_users, total_guilds=total_guilds, total_messages=total_messages, total_deletions=total_deletions)
@register_breadcrumb(app, '.guilds', 'Guilds')
@app.route('/guilds/')
def list_all_guilds():
all_guilds = db_session.query(db.Guild).all()
return render_template("guilds_list.html", guilds=all_guilds)
@app.route('/guilds/<guild_id>/')
@register_breadcrumb(app, '.guilds.guild_id', 'Individual guild')
@app.route('/guilds/<int:guild_id>/')
def list_guild_channels(guild_id):
channels = db_session.query(db.GuildChannel).filter_by(guild_id=guild_id).order_by(
db.GuildChannel.created_at.asc()).all()
return render_template("channel_list.html", channels=channels)
@app.route('/guilds/<guild_id>/info')
@register_breadcrumb(app, '.guilds.id.info', 'Guild info')
@app.route('/guilds/<int:guild_id>/info')
def show_single_guild(guild_id):
guild = db_session.query(db.Guild).filter_by(id=guild_id).one()
members = db_session.query(db.GuildMember).filter_by(guild_id=guild_id).all()
@ -142,13 +147,15 @@ def show_single_guild(guild_id):
return render_template("guild_details.html", guild=guild, members=members, roles=roles)
@app.route('/users/<user_id>/')
@register_breadcrumb(app, '.users.id', 'User ID')
@app.route('/users/<int:user_id>/')
def show_single_user(user_id):
user = db_session.query(db.User).filter_by(id=user_id).one()
members = db_session.query(db.GuildMember).filter_by(user_id=user_id).all()
return render_template("user_details.html", user=user, members=members)
@app.route('/users/<user_id>/<guild_id>')
@register_breadcrumb(app, '.users.id.guild_id', 'Membership info')
@app.route('/users/<int:user_id>/<int:guild_id>')
def show_single_member(user_id, guild_id):
guild = db_session.query(db.Guild).filter_by(id=guild_id).one()
member = db_session.query(db.GuildMember).filter_by(guild_id=guild.id, user_id=user_id).one()
@ -158,46 +165,53 @@ def show_single_member(user_id, guild_id):
current_roles = db_session.query(db.GuildMemberRoles).filter_by(member_id=member.id).all()
return render_template("user_guild_history.html", guild=guild, join_leave_audits=join_leave_audits, ban_audits=ban_audits, roles_audit=roles_audit, current_roles=current_roles)
@app.route('/channels/<channel_id>/')
def list_all_logged_days_for_channel(channel_id):
channel, days = request_days(db.GuildMessage, channel_id)
@register_breadcrumb(app, '.guilds.guild_id.channel_id', 'Channel days')
@app.route('/guilds/<int:guild_id>/<int:channel_id>/')
def list_all_logged_days_for_channel(guild_id, channel_id):
days = request_days(db.GuildMessage, channel_id)
days.sort()
return render_template("days_list.html", channel=channel, days=days)
channel = db_session.query(db.GuildChannel).filter_by(id=channel_id).one()
return render_template("guild_days_list.html", channel=channel, days=days)
@app.route('/channels/<channel_id>/info')
def list_channel_info(channel_id):
@register_breadcrumb(app, '.guilds.guild_id.channel_id.info', 'Channel info')
@app.route('/guilds/<int:guild_id>/<int:channel_id>/info')
def list_channel_info(guild_id, channel_id):
channel = db_session.query(db.GuildChannel).filter_by(id=channel_id).one()
channel_edits = db_session.query(db.GuildChannelEdit).filter_by(guild_channel_id=channel_id).all()
return render_template("channel_details.html", channel=channel, channel_edits=channel_edits)
@app.route('/channels/<channel_id>/<date>')
def list_all_messages_per_day(channel_id, date):
@app.route('/guilds/<int:guild_id>/<int:channel_id>/<date>')
def list_all_messages_per_day(guild_id, channel_id, date):
channel, messages, total_messages = process_messages(db.GuildMessage, db.GuildMessageAttachments, db.GuildMessageEdit, db.GuildMessageDeletion, channel_id, date)
return render_template("messages.html", channel=channel, all_messages_grouped=messages, message_length=total_messages)
@register_breadcrumb(app, '.dms', 'DMs')
@app.route('/dms/')
def list_all_dms():
all_dms = db_session.query(db.DMChannel).all()
return render_template("dm_list.html", dms=all_dms)
@app.route('/dms/<dm_id>/')
@register_breadcrumb(app, '.dms.id', 'Individual DM')
@app.route('/dms/<int:dm_id>/')
def list_all_logged_days_for_dm(dm_id):
channel, days = request_days(db.PrivateMessage, dm_id)
return render_template("days_list.html", channel=channel, days=days)
days = request_days(db.PrivateMessage, dm_id)
channel = db_session.query(db.DMChannel).filter_by(id=dm_id).one()
return render_template("dm_days_list.html", channel=channel, days=days)
@app.route('/dms/<dm_id>/<date>')
@app.route('/dms/<int:dm_id>/<date>')
def list_all_dms_per_day(dm_id, date):
channel, messages, total_messages = process_messages(db.PrivateMessage, db.PrivateMessageAttachments, db.PrivateMessageEdit, db.PrivateMessageDeletion, dm_id, date)
channel.guild = channel.remote_user # STUPID AND HACKY BUT IT WORKS!
return render_template("messages.html", channel=channel, all_messages_grouped=messages, message_length=total_messages)
@register_breadcrumb(app, '.roles', 'Roles list')
@app.route('/roles/')
def show_all_roles():
all_roles = db_session.query(db.Role).all()
@ -207,23 +221,22 @@ def show_all_roles():
return render_template("role_list.html", roles=all_roles)
@app.route('/roles/<role_id>')
@register_breadcrumb(app, '.roles.id', 'Individual roles')
@app.route('/roles/<int:role_id>')
def show_role_info(role_id):
role = db_session.query(db.Role).filter_by(id=role_id).one()
user_ids = session_filter.get_all_user_ids_with_role(db_session, role_id)
role_users = []
for user_id in user_ids:
role_users.append(db_session.query(db.User).filter_by(id=user_id).one())
role_users = session_filter.get_all_user_ids_with_role(db_session, role_id)
return render_template("role_details.html", role=role, role_users=role_users)
@register_breadcrumb(app, '.search', 'Search')
@app.route('/search/')
def search_overview():
return render_template("search.html")
@register_breadcrumb(app, '.search.messages', 'Search Messages')
@app.route('/search/messages', methods=['GET', 'POST'])
def message_search():
form = forms.GeneralSearchForm(request.form)
@ -233,6 +246,7 @@ def message_search():
return render_template("search_messages.html", form=form, results=results)
@register_breadcrumb(app, '.search.users', 'Search users')
@app.route('/search/users', methods=['GET', 'POST'])
def user_search():
form = forms.UserSearchForm(request.form)
@ -242,7 +256,6 @@ def user_search():
return render_template("search_user.html", form=form, results=results)
@app.route('/results/users')
def user_results(form):
results = []
results = viewer_modules.search.search_users_on_session(form, db_session)
@ -250,7 +263,6 @@ def user_results(form):
return render_template("search_user.html", form=form, results=results)
@app.route('/results/messages')
def message_results(form):
results = []
results = viewer_modules.search.search_messages_on_session(form, db_session)


+ 3
- 7
viewer_modules/session_filter.py View File

@ -3,15 +3,11 @@ import db
def get_all_user_ids_with_role(session, role_id) -> List[db.User]:
"""Returns a list of all users who currently have a role."""
all_audits_for_role = session.query(db.RoleAudit).filter_by(role_id=role_id).all()
all_current_roles_list = session.query(db.GuildMemberRoles).filter_by(role_id=role_id).all()
user_list = []
for audit in all_audits_for_role:
if audit.role_was_added:
user_list.append(audit.member.user_id)
else:
if audit.member.user_id in user_list:
user_list.remove(audit.member.user_id)
for role in all_current_roles_list:
user_list.append(role.member.user)
return user_list