Compare commits

...
2 Commits
Author SHA1 Message Date
jude a9cc0b8f0a Move to FKs 2026-10-18 20:06:55 +01:00
jude 96bfef4575 Add support for S3-backed image storage 2026-10-18 19:12:50 +01:00
51 changed files with 3465 additions and 310 deletions

No files matched your search

+1
View File
@@ -1,6 +1,7 @@
target
.env
/venv
__pycache__/
.cargo
.idea
static/index.html
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n reminders.`id` AS id,\n\n channels.`channel` AS channel_id,\n reminders.`thread_id` AS thread_id,\n channels.`webhook_id` AS webhook_id,\n channels.`webhook_token` AS webhook_token,\n\n channels.`paused` AS 'channel_paused',\n channels.`paused_until` AS 'channel_paused_until',\n reminders.`enabled` AS 'enabled',\n\n reminders.`tts` AS tts,\n reminders.`pin` AS pin,\n reminders.`content` AS content,\n reminders.`attachment` AS attachment,\n reminders.`attachment_name` AS attachment_name,\n\n reminders.`utc_time` AS 'utc_time',\n reminders.`timezone` AS timezone,\n reminders.`restartable` AS restartable,\n reminders.`expires` AS 'expires',\n reminders.`interval_seconds` AS 'interval_seconds',\n reminders.`interval_days` AS 'interval_days',\n reminders.`interval_months` AS 'interval_months',\n\n reminders.`avatar` AS avatar,\n reminders.`username` AS username\n FROM\n reminders\n INNER JOIN\n channels\n ON\n reminders.channel_id = channels.id\n WHERE\n reminders.`status` = 'pending' AND\n reminders.`id` IN (\n SELECT\n MIN(id)\n FROM\n reminders\n WHERE\n reminders.`utc_time` <= NOW() AND\n `status` = 'pending' AND\n (\n reminders.`interval_seconds` IS NOT NULL\n OR reminders.`interval_months` IS NOT NULL\n OR reminders.`interval_days` IS NOT NULL\n OR reminders.enabled\n )\n GROUP BY channel_id\n )\n ",
"query": "\n SELECT\n reminders.`id` AS id,\n\n channels.`channel` AS channel_id,\n reminders.`thread_id` AS thread_id,\n channels.`webhook_id` AS webhook_id,\n channels.`webhook_token` AS webhook_token,\n\n channels.`paused` AS 'channel_paused',\n channels.`paused_until` AS 'channel_paused_until',\n reminders.`enabled` AS 'enabled',\n\n reminders.`tts` AS tts,\n reminders.`pin` AS pin,\n reminders.`content` AS content,\n reminders.`attachment` AS attachment,\n reminders.`attachment_name` AS attachment_name,\n\n reminders.`utc_time` AS 'utc_time',\n reminders.`timezone` AS timezone,\n reminders.`restartable` AS restartable,\n reminders.`expires` AS 'expires',\n reminders.`interval_seconds` AS 'interval_seconds',\n reminders.`interval_days` AS 'interval_days',\n reminders.`interval_months` AS 'interval_months',\n\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,\n reminders.`username` AS username\n FROM\n reminders\n INNER JOIN\n channels\n ON\n reminders.channel_id = channels.id\n WHERE\n reminders.`status` = 'pending' AND\n reminders.`id` IN (\n SELECT\n MIN(id)\n FROM\n reminders\n WHERE\n reminders.`utc_time` <= NOW() AND\n `status` = 'pending' AND\n (\n reminders.`interval_seconds` IS NOT NULL\n OR reminders.`interval_months` IS NOT NULL\n OR reminders.`interval_days` IS NOT NULL\n OR reminders.enabled\n )\n GROUP BY channel_id\n )\n ",
"describe": {
"columns": [
{
@@ -230,5 +230,5 @@
true
]
},
"hash": "a33ef7167746183766b6a9b4e31784dd3712fa2b4f72482d4b681e0a9fdbaf1d"
"hash": "253f96647b53ee3dd2f00e6e12c5972ac69f48ca5ce40498bc6a0d7687d28c34"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "INSERT INTO reminder_template\n (guild_id,\n name,\n attachment,\n attachment_name,\n avatar,\n avatar_id,\n content,\n embed_author,\n embed_author_url,\n embed_author_id,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_footer_id,\n embed_image_url,\n embed_image_id,\n embed_thumbnail_url,\n embed_thumbnail_id,\n embed_title,\n embed_fields,\n interval_seconds,\n interval_days,\n interval_months,\n tts,\n username\n ) VALUES ((SELECT id FROM guilds WHERE guild = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,\n ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"describe": {
"columns": [],
"parameters": {
"Right": 26
},
"nullable": []
},
"hash": "2796be7484d8872eaae3655599c5071cfb0bdc1de4a557cd44441df3508a0bd2"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n reminders.attachment_name,\n reminders.avatar,\n channels.channel,\n reminders.content,\n reminders.embed_author,\n reminders.embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n reminders.embed_footer_url,\n reminders.embed_image_url,\n reminders.embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.restartable,\n reminders.tts,\n reminders.uid,\n reminders.username,\n reminders.utc_time\n FROM reminders\n LEFT JOIN channels ON channels.id = reminders.channel_id\n WHERE uid = ?",
"query": "\n SELECT\n reminders.attachment_name,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,\n channels.channel,\n reminders.content,\n reminders.embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.restartable,\n reminders.tts,\n reminders.uid,\n reminders.username,\n reminders.utc_time\n FROM reminders\n LEFT JOIN channels ON channels.id = reminders.channel_id\n WHERE uid = ?",
"describe": {
"columns": [
{
@@ -260,5 +260,5 @@
false
]
},
"hash": "fa58f079f00bec4c3f9cbb2ee0a607f63c5b586654c541bff6b99eff83ae0a44"
"hash": "332c615185b316e1b3dab4b81565af1fb15b0e235bc8d5a3b95f5ee8feeae055"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n reminders.attachment_name,\n reminders.avatar,\n channels.channel,\n reminders.content,\n reminders.embed_author,\n reminders.embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n reminders.embed_footer_url,\n reminders.embed_image_url,\n reminders.embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.restartable,\n reminders.tts,\n reminders.uid,\n reminders.username,\n reminders.utc_time\n FROM reminders\n LEFT JOIN channels ON channels.id = reminders.channel_id\n WHERE uid = ?",
"query": "\n SELECT\n reminders.attachment_name,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,\n channels.channel,\n reminders.content,\n reminders.embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.restartable,\n reminders.tts,\n reminders.uid,\n reminders.username,\n reminders.utc_time\n FROM reminders\n LEFT JOIN channels ON channels.id = reminders.channel_id\n WHERE uid = ?",
"describe": {
"columns": [
{
@@ -260,5 +260,5 @@
false
]
},
"hash": "53db321bc70b45800f696115b65da8550de208104017f8e0d0a163781488a534"
"hash": "45e6381b3b68ad03f6459db8472587cf36d11a61bc5f94ed173adfd4ea68565d"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "INSERT INTO reminders (\n uid,\n attachment,\n attachment_name,\n channel_id,\n avatar,\n content,\n embed_author,\n embed_author_url,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_image_url,\n embed_thumbnail_url,\n embed_title,\n embed_fields,\n enabled,\n expires,\n interval_seconds,\n interval_days,\n interval_months,\n name,\n restartable,\n tts,\n username,\n `utc_time`\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"describe": {
"columns": [],
"parameters": {
"Right": 26
},
"nullable": []
},
"hash": "4cc3ff2bbdf8502f6e739a002c7aa16df8c4f24de0966b26b268efab97e205e3"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n `embed_title` AS title,\n `embed_description` AS description,\n `embed_image_url` AS image_url,\n `embed_thumbnail_url` AS thumbnail_url,\n `embed_footer` AS footer,\n `embed_footer_url` AS footer_url,\n `embed_author` AS author,\n `embed_author_url` AS author_url,\n `embed_color` AS color,\n IFNULL(`embed_fields`, '[]') AS \"fields:_\"\n FROM reminders\n WHERE `id` = ?",
"query": "\n SELECT\n `embed_title` AS title,\n `embed_description` AS description,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), `embed_image_url`) AS image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), `embed_thumbnail_url`) AS thumbnail_url,\n `embed_footer` AS footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), `embed_footer_url`) AS footer_url,\n `embed_author` AS author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), `embed_author_url`) AS author_url,\n `embed_color` AS color,\n IFNULL(`embed_fields`, '[]') AS \"fields:_\"\n FROM reminders\n WHERE `id` = ?",
"describe": {
"columns": [
{
@@ -110,5 +110,5 @@
false
]
},
"hash": "98ce62a260f411b834ab4b5374b49ea79850cce49e23b71d0eee917609beabd2"
"hash": "7bbf5875b7e0edcf19f4fea7ad6f1685ed176c9b6979ea208d47e421b97540d3"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "INSERT INTO reminders (\n uid,\n attachment,\n attachment_name,\n channel_id,\n content,\n embed_author,\n embed_author_url,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_image_url,\n embed_thumbnail_url,\n embed_title,\n embed_fields,\n enabled,\n expires,\n interval_seconds,\n interval_days,\n interval_months,\n name,\n tts,\n `utc_time`\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"describe": {
"columns": [],
"parameters": {
"Right": 23
},
"nullable": []
},
"hash": "7f44451c441507ccd7f8d41633fb156f95a28389fcc9c489fdb228446f88342e"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "SELECT\n reminders.attachment,\n reminders.attachment_name,\n reminders.content,\n reminders.embed_author,\n reminders.embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n reminders.embed_footer_url,\n reminders.embed_image_url,\n reminders.embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.tts,\n reminders.uid,\n reminders.utc_time\n FROM reminders\n WHERE uid = ?",
"query": "SELECT\n reminders.attachment,\n reminders.attachment_name,\n reminders.content,\n reminders.embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.tts,\n reminders.uid,\n reminders.utc_time\n FROM reminders\n WHERE uid = ?",
"describe": {
"columns": [
{
@@ -230,5 +230,5 @@
false
]
},
"hash": "2613618c00941649726b31f7c50b5ef50d85323a73def5c87bab2a58c80bf6cd"
"hash": "89452b34238cb0d885044fdbfc5ecab967d17224ffe4929be16be4a7b5c8e3ea"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "INSERT INTO reminders (\n uid,\n attachment,\n attachment_name,\n channel_id,\n content,\n embed_author,\n embed_author_url,\n embed_author_id,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_footer_id,\n embed_image_url,\n embed_image_id,\n embed_thumbnail_url,\n embed_thumbnail_id,\n embed_title,\n embed_fields,\n enabled,\n expires,\n interval_seconds,\n interval_days,\n interval_months,\n name,\n tts,\n `utc_time`\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"describe": {
"columns": [],
"parameters": {
"Right": 27
},
"nullable": []
},
"hash": "91d6a89be74bc242e5fafbe8bbb701ef1ee2db4779af504858b9cd9d602df9df"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "SELECT * FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
"query": "SELECT id,\n guild_id,\n name,\n attachment,\n attachment_name,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.avatar_id), avatar) AS avatar,\n content,\n embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_author_id), embed_author_url) AS embed_author_url,\n embed_color,\n embed_description,\n embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_footer_id), embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_image_id), embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_thumbnail_id), embed_thumbnail_url) AS embed_thumbnail_url,\n embed_title,\n embed_fields,\n interval_seconds,\n interval_days,\n interval_months,\n tts,\n username FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
"describe": {
"columns": [
{
@@ -14,15 +14,6 @@
},
{
"ordinal": 1,
"name": "name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL",
"max_size": 400
}
},
{
"ordinal": 2,
"name": "guild_id",
"type_info": {
"type": "Long",
@@ -30,44 +21,17 @@
"max_size": 10
}
},
{
"ordinal": 2,
"name": "name",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL",
"max_size": 400
}
},
{
"ordinal": 3,
"name": "username",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 128
}
},
{
"ordinal": 4,
"name": "avatar",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 5,
"name": "content",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL",
"max_size": 8192
}
},
{
"ordinal": 6,
"name": "tts",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"max_size": 1
}
},
{
"ordinal": 7,
"name": "attachment",
"type_info": {
"type": "Blob",
@@ -76,7 +40,7 @@
}
},
{
"ordinal": 8,
"ordinal": 4,
"name": "attachment_name",
"type_info": {
"type": "VarString",
@@ -85,14 +49,50 @@
}
},
{
"ordinal": 9,
"name": "embed_title",
"ordinal": 5,
"name": "avatar",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 6,
"name": "content",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL",
"max_size": 8192
}
},
{
"ordinal": 7,
"name": "embed_author",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL",
"max_size": 1024
}
},
{
"ordinal": 8,
"name": "embed_author_url",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 9,
"name": "embed_color",
"type_info": {
"type": "Long",
"flags": "NOT_NULL | UNSIGNED",
"max_size": 10
}
},
{
"ordinal": 10,
"name": "embed_description",
@@ -104,24 +104,6 @@
},
{
"ordinal": 11,
"name": "embed_image_url",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 12,
"name": "embed_thumbnail_url",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 13,
"name": "embed_footer",
"type_info": {
"type": "VarString",
@@ -130,7 +112,7 @@
}
},
{
"ordinal": 14,
"ordinal": 12,
"name": "embed_footer_url",
"type_info": {
"type": "VarString",
@@ -138,9 +120,27 @@
"max_size": 2048
}
},
{
"ordinal": 13,
"name": "embed_image_url",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 14,
"name": "embed_thumbnail_url",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 15,
"name": "embed_author",
"name": "embed_title",
"type_info": {
"type": "VarString",
"flags": "NOT_NULL",
@@ -149,24 +149,6 @@
},
{
"ordinal": 16,
"name": "embed_author_url",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 2048
}
},
{
"ordinal": 17,
"name": "embed_color",
"type_info": {
"type": "Long",
"flags": "NOT_NULL | UNSIGNED",
"max_size": 10
}
},
{
"ordinal": 18,
"name": "embed_fields",
"type_info": {
"type": "Json",
@@ -175,7 +157,7 @@
}
},
{
"ordinal": 19,
"ordinal": 17,
"name": "interval_seconds",
"type_info": {
"type": "Long",
@@ -184,7 +166,7 @@
}
},
{
"ordinal": 20,
"ordinal": 18,
"name": "interval_days",
"type_info": {
"type": "Long",
@@ -193,13 +175,31 @@
}
},
{
"ordinal": 21,
"ordinal": 19,
"name": "interval_months",
"type_info": {
"type": "Long",
"flags": "UNSIGNED",
"max_size": 10
}
},
{
"ordinal": 20,
"name": "tts",
"type_info": {
"type": "Tiny",
"flags": "NOT_NULL",
"max_size": 1
}
},
{
"ordinal": 21,
"name": "username",
"type_info": {
"type": "VarString",
"flags": "",
"max_size": 128
}
}
],
"parameters": {
@@ -211,24 +211,24 @@
false,
true,
true,
false,
false,
true,
true,
false,
false,
true,
true,
false,
false,
false,
true,
false,
true,
true,
false,
true,
true,
true,
true,
false,
true
]
},
"hash": "a8b3563796f18e8ac28166771892226dbf8f730535e7f2d1225ff9eb6b1d9361"
"hash": "bb845163adb4f6dc61c068f58bcc4dda9609c22b41507a652408669bd17fe6ea"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT\n reminders.attachment,\n reminders.attachment_name,\n reminders.content,\n reminders.embed_author,\n reminders.embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n reminders.embed_footer_url,\n reminders.embed_image_url,\n reminders.embed_thumbnail_url,\n reminders.embed_title,\n IFNULL(reminders.embed_fields, '[]') AS embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.tts,\n reminders.uid,\n reminders.utc_time\n FROM reminders\n INNER JOIN channels ON channels.id = reminders.channel_id\n WHERE `status` = 'pending' AND channels.channel = ?\n ",
"query": "\n SELECT\n reminders.attachment,\n reminders.attachment_name,\n reminders.content,\n reminders.embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,\n reminders.embed_title,\n IFNULL(reminders.embed_fields, '[]') AS embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.tts,\n reminders.uid,\n reminders.utc_time\n FROM reminders\n INNER JOIN channels ON channels.id = reminders.channel_id\n WHERE `status` = 'pending' AND channels.channel = ?\n ",
"describe": {
"columns": [
{
@@ -230,5 +230,5 @@
false
]
},
"hash": "48a9268e9106a3a05e86ab36551944ba855a07b0241e912c6b1adee779c6f0ea"
"hash": "e01fef8182e50d65459b03e1e93ed4198e3b581f3dfe1df76c250cafa2b9ce0f"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "\n SELECT reminders.attachment,\n reminders.attachment_name,\n reminders.content,\n reminders.embed_author,\n reminders.embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n reminders.embed_footer_url,\n reminders.embed_image_url,\n reminders.embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.tts,\n reminders.uid,\n reminders.utc_time\n FROM reminders\n LEFT JOIN channels ON channels.id = reminders.channel_id\n WHERE uid = ?\n ",
"query": "\n SELECT reminders.attachment,\n reminders.attachment_name,\n reminders.content,\n reminders.embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,\n reminders.embed_color,\n reminders.embed_description,\n reminders.embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,\n reminders.embed_title,\n reminders.embed_fields,\n reminders.enabled,\n reminders.expires,\n reminders.interval_seconds,\n reminders.interval_days,\n reminders.interval_months,\n reminders.name,\n reminders.tts,\n reminders.uid,\n reminders.utc_time\n FROM reminders\n LEFT JOIN channels ON channels.id = reminders.channel_id\n WHERE uid = ?\n ",
"describe": {
"columns": [
{
@@ -230,5 +230,5 @@
false
]
},
"hash": "ecbea4acb61da864a4f83004b1163391068456a366fee95daff5f46413bcf494"
"hash": "e32fca1af5eb399768aa1ca0f0073d919a2c29f5d7c86cf1a23866e110dc8f33"
}
@@ -1,6 +1,6 @@
{
"db_name": "MySQL",
"query": "SELECT\n name,\n attachment,\n attachment_name,\n avatar,\n content,\n embed_author,\n embed_author_url,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_image_url,\n embed_thumbnail_url,\n embed_title,\n embed_fields,\n interval_seconds,\n interval_days,\n interval_months,\n tts,\n username\n FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
"query": "SELECT\n name,\n attachment,\n attachment_name,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.avatar_id), avatar) AS avatar,\n content,\n embed_author,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_author_id), embed_author_url) AS embed_author_url,\n embed_color,\n embed_description,\n embed_footer,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_footer_id), embed_footer_url) AS embed_footer_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_image_id), embed_image_url) AS embed_image_url,\n COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_thumbnail_id), embed_thumbnail_url) AS embed_thumbnail_url,\n embed_title,\n embed_fields,\n interval_seconds,\n interval_days,\n interval_months,\n tts,\n username\n FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
"describe": {
"columns": [
{
@@ -210,5 +210,5 @@
true
]
},
"hash": "be4c9a0fffa3ecaad4eb8ae59229ebfed685cf069b309ab874db4db069d14a3c"
"hash": "efac2582e5b7de7e0923f9ebc1bc8d643683baf65e94dcc7f431290099de95d7"
}
@@ -0,0 +1,12 @@
{
"db_name": "MySQL",
"query": "INSERT INTO reminders (\n uid,\n attachment,\n attachment_name,\n channel_id,\n avatar,\n avatar_id,\n content,\n embed_author,\n embed_author_url,\n embed_author_id,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_footer_id,\n embed_image_url,\n embed_image_id,\n embed_thumbnail_url,\n embed_thumbnail_id,\n embed_title,\n embed_fields,\n enabled,\n expires,\n interval_seconds,\n interval_days,\n interval_months,\n name,\n restartable,\n tts,\n username,\n `utc_time`\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
"describe": {
"columns": [],
"parameters": {
"Right": 31
},
"nullable": []
},
"hash": "f3364e9792d1886eff3d2dec0c3e789d1aeb9209405013df0e81560a42ca39b2"
}
@@ -1,12 +0,0 @@
{
"db_name": "MySQL",
"query": "INSERT INTO reminder_template\n (guild_id,\n name,\n attachment,\n attachment_name,\n avatar,\n content,\n embed_author,\n embed_author_url,\n embed_color,\n embed_description,\n embed_footer,\n embed_footer_url,\n embed_image_url,\n embed_thumbnail_url,\n embed_title,\n embed_fields,\n interval_seconds,\n interval_days,\n interval_months,\n tts,\n username\n ) VALUES ((SELECT id FROM guilds WHERE guild = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,\n ?, ?, ?, ?, ?, ?, ?)",
"describe": {
"columns": [],
"parameters": {
"Right": 21
},
"nullable": []
},
"hash": "f64d9a37720b8459ffbdd66186194b35332cc37b09a176165a839c342ba5354c"
}
Generated
+1320 -81
View File
File diff suppressed because it is too large. Load diff
+10 -1
View File
@@ -36,6 +36,9 @@ oauth2 = "4"
csv = "1.2"
sd-notify = "0.4.1"
cron-parser = "0.10"
image = { version = "0.25", default-features = false, features = ["jpeg", "png", "gif", "webp"] }
nsfw = { version = "0.2", default-features = false }
rust-s3 = { version = "0.38", default-features = false, features = ["tokio-rustls-tls"] }
[dependencies.extract_derive]
path = "extract_derive"
@@ -68,6 +71,12 @@ conf-files = [
"/etc/reminder-rs/Rocket.toml",
]
[package.metadata.deb.systemd-units]
[[package.metadata.deb.systemd-units]]
unit-scripts = "systemd"
unit-name = "reminder-rs"
start = false
[[package.metadata.deb.systemd-units]]
unit-scripts = "systemd"
unit-name = "reminder-rs-clean-images"
start = false
+14
View File
@@ -76,6 +76,20 @@ __Other Variables__
* `THEME_COLOR` - default `8fb677`. Specifies the hex value of the color to use on info message
embeds
Images can be uploaded directly when all of these settings are supplied:
* `S3_BUCKET` - bucket
* `S3_REGION` - bucket region
* `S3_ACCESS_KEY_ID` and `S3_SECRET_ACCESS_KEY` - credentials
* `S3_PUBLIC_URL` - public base URL. Objects must be publicly readable through this URL.
* `NSFW_CHECK_ENABLED` - enables the use of a small model to block uploading
of NSFW images. Set it to `false` to skip NSFW moderation and model initialization.
Uploaded images are stored as bucket paths in `image_uploads`. Reminders and templates
reference uploads through indexed foreign keys; external image URLs remain supported.
Public URLs are resolved from `S3_PUBLIC_URL` for delivery, dashboard previews, and exports.
Changing that base URL does not require updating stored reminders.
## Running with Docker
A `compose.yml` file is provided to aid in running the bot agnostically using docker.
+7
View File
@@ -0,0 +1,7 @@
# Bundled NSFW model
`model.onnx` is the pretrained model distributed with
[Fyko/nsfw v0.2.0](https://github.com/fyko/nsfw/releases/tag/v0.2.0), based on
[GantMan/nsfw_model](https://github.com/GantMan/nsfw_model).
Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx
Binary file not shown.
+15
View File
@@ -20,3 +20,18 @@ REPORT_EMAIL=
SENT_CLEAN_AGE=
# Delete all reminders after this many months
TOTAL_CLEAN_AGE=
# Optional dashboard image uploads. Leave these blank to use image URLs only.
# All five S3 settings are required. The bucket/CDN must serve objects publicly.
S3_BUCKET=
S3_REGION=
S3_ACCESS_KEY_ID=
S3_SECRET_ACCESS_KEY=
S3_PUBLIC_URL=
# Optional S3-compatible endpoint (uses path-style addressing).
S3_ENDPOINT=
# NSFW moderation is enabled by default. Set false (or 0/off) to disable it.
NSFW_CHECK_ENABLED=true
# Seconds to retain an upload after cleanup first detects that it is orphaned.
# Must be positive. Defaults to seven days when absent or blank.
IMAGE_UPLOAD_ORPHAN_RETENTION_SECONDS=604800
@@ -0,0 +1,45 @@
CREATE TABLE image_uploads (
id BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY,
-- Discord user ID from the authenticated session, including guild-only users.
uploader BIGINT UNSIGNED NOT NULL,
path VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_bin NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
orphaned_at DATETIME DEFAULT NULL,
UNIQUE INDEX image_uploads_path (path),
INDEX image_uploads_uploader (uploader),
INDEX image_uploads_orphaned_at (orphaned_at)
);
ALTER TABLE reminders
ADD COLUMN avatar_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminders_avatar_id (avatar_id),
ADD FOREIGN KEY (avatar_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_image_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminders_embed_image_id (embed_image_id),
ADD FOREIGN KEY (embed_image_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_thumbnail_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminders_embed_thumbnail_id (embed_thumbnail_id),
ADD FOREIGN KEY (embed_thumbnail_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_author_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminders_embed_author_id (embed_author_id),
ADD FOREIGN KEY (embed_author_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_footer_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminders_embed_footer_id (embed_footer_id),
ADD FOREIGN KEY (embed_footer_id) REFERENCES image_uploads (id) ON DELETE RESTRICT;
ALTER TABLE reminder_template
ADD COLUMN avatar_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminder_template_avatar_id (avatar_id),
ADD FOREIGN KEY (avatar_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_image_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminder_template_embed_image_id (embed_image_id),
ADD FOREIGN KEY (embed_image_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_thumbnail_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminder_template_embed_thumbnail_id (embed_thumbnail_id),
ADD FOREIGN KEY (embed_thumbnail_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_author_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminder_template_embed_author_id (embed_author_id),
ADD FOREIGN KEY (embed_author_id) REFERENCES image_uploads (id) ON DELETE RESTRICT,
ADD COLUMN embed_footer_id BIGINT UNSIGNED DEFAULT NULL,
ADD INDEX reminder_template_embed_footer_id (embed_footer_id),
ADD FOREIGN KEY (embed_footer_id) REFERENCES image_uploads (id) ON DELETE RESTRICT;
+9 -1
View File
@@ -9,6 +9,7 @@ enum ColorScheme {
type UserInfo = {
name: string;
patreon: boolean;
image_uploads: boolean;
preferences: {
timezone: string | null;
reset_inputs_on_create: boolean;
@@ -126,6 +127,13 @@ export const fetchUserInfo = () => ({
staleTime: USER_INFO_STALE_TIME,
});
export const uploadImage = (file: File) =>
axios
.post<{ url: string }>("/dashboard/api/images", file, {
headers: { "Content-Type": "application/octet-stream" },
})
.then((response) => response.data.url);
export const patchUserInfo = () => ({
mutationFn: (userInfo: UpdateUserInfo) => axios.patch(`/dashboard/api/user`, userInfo),
});
@@ -236,7 +244,7 @@ export const fetchGuildTodos = (guild: string) => ({
});
export const patchGuildTodo = (guild: string) => ({
mutationFn: ({id, todo}) => axios.patch(`/dashboard/api/guild/${guild}/todos/${id}`, todo),
mutationFn: ({ id, todo }) => axios.patch(`/dashboard/api/guild/${guild}/todos/${id}`, todo),
});
export const postGuildTodo = (guild: string) => ({
@@ -2,14 +2,22 @@ import { JSX } from "preact";
import { createPortal } from "preact/compat";
type Props = {
setModalOpen: (open: boolean) => never;
setModalOpen: (open: boolean) => void;
title: string | JSX.Element;
onSubmitText?: string;
onSubmit?: () => void;
onSubmitDisabled?: boolean;
children: string | JSX.Element | JSX.Element[] | (() => JSX.Element);
};
export const Modal = ({ setModalOpen, title, onSubmit, onSubmitText, children }: Props) => {
export const Modal = ({
setModalOpen,
title,
onSubmit,
onSubmitText,
onSubmitDisabled,
children,
}: Props) => {
const body = document.querySelector("body");
return createPortal(
@@ -34,7 +42,11 @@ export const Modal = ({ setModalOpen, title, onSubmit, onSubmitText, children }:
<section class="modal-card-body">{children}</section>
{onSubmit && (
<footer class="modal-card-foot">
<button class="button is-success" onClick={onSubmit}>
<button
class="button is-success"
onClick={onSubmit}
disabled={onSubmitDisabled}
>
{onSubmitText || "Save"}
</button>
<button
@@ -1,13 +1,65 @@
import { Modal } from "../Modal";
import { useState } from "preact/hooks";
import { useEffect, useRef, useState } from "preact/hooks";
import { useQuery } from "react-query";
import axios from "axios";
import { fetchUserInfo, uploadImage } from "../../api";
const useImageDrop = (enabled: boolean, onFiles: (files: File[]) => void, busy = false) => {
const [dragging, setDragging] = useState(false);
const dragDepth = useRef(0);
const acceptsDrag = (event: DragEvent) =>
enabled && event.dataTransfer?.types.includes("Files");
return {
dragging,
handlers: {
onDragEnter: (event: DragEvent) => {
if (!acceptsDrag(event)) return;
event.preventDefault();
event.stopPropagation();
dragDepth.current += 1;
if (!busy) setDragging(true);
},
onDragOver: (event: DragEvent) => {
if (!acceptsDrag(event)) return;
event.preventDefault();
event.stopPropagation();
event.dataTransfer.dropEffect = busy ? "none" : "copy";
},
onDragLeave: () => {
dragDepth.current = Math.max(0, dragDepth.current - 1);
if (dragDepth.current === 0) setDragging(false);
},
onDrop: (event: DragEvent) => {
if (!acceptsDrag(event)) return;
event.preventDefault();
event.stopPropagation();
dragDepth.current = 0;
setDragging(false);
if (!busy) onFiles(Array.from(event.dataTransfer.files));
},
},
};
};
export const ImagePicker = ({ alt, url, setImage, ...props }) => {
const [modalOpen, setModalOpen] = useState(false);
const [droppedFiles, setDroppedFiles] = useState<File[] | null>(null);
const { data: userInfo } = useQuery(fetchUserInfo());
const uploadsEnabled = userInfo?.image_uploads === true;
const { dragging, handlers } = useImageDrop(uploadsEnabled, (files) => {
setDroppedFiles(files);
setModalOpen(true);
});
return (
<>
<a
{...handlers}
class={dragging ? "image-picker-dragging" : ""}
title={uploadsEnabled ? "Click to edit or drop an image to upload" : undefined}
onClick={() => {
setDroppedFiles(null);
setModalOpen(true);
}}
role={"button"}
@@ -18,14 +70,64 @@ export const ImagePicker = ({ alt, url, setImage, ...props }) => {
<ImagePickerModal
setModalOpen={setModalOpen}
setImage={setImage}
url={url}
uploadsEnabled={uploadsEnabled}
droppedFiles={droppedFiles}
></ImagePickerModal>
)}
</>
);
};
const ImagePickerModal = ({ setModalOpen, setImage }) => {
const [value, setValue] = useState("");
const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, droppedFiles }) => {
const [value, setValue] = useState(url || "");
const [uploading, setUploading] = useState(false);
const [error, setError] = useState("");
const [blocked, setBlocked] = useState(false);
const fileInput = useRef<HTMLInputElement>(null);
const uploadInProgress = useRef(false);
const upload = async (files: File[]) => {
setBlocked(false);
if (!uploadsEnabled || uploadInProgress.current) return;
setError("");
if (files.length !== 1) {
setError("Please upload only one image.");
return;
}
const file = files[0];
if (file.size > 8 * 1024 * 1024) {
setError("Images must be no larger than 8 MiB.");
return;
}
uploadInProgress.current = true;
setUploading(true);
try {
setValue(await uploadImage(file));
} catch (error) {
if (axios.isAxiosError(error) && error.response?.status === 451) {
setBlocked(true);
} else {
setError(
axios.isAxiosError(error) && typeof error.response?.data?.error === "string"
? error.response.data.error
: "Could not upload image. Please try again.",
);
}
} finally {
uploadInProgress.current = false;
setUploading(false);
}
};
const { dragging, handlers } = useImageDrop(
uploadsEnabled,
(files) => void upload(files),
uploading,
);
useEffect(() => {
if (droppedFiles) void upload(droppedFiles);
}, [droppedFiles]);
return (
<Modal
@@ -36,6 +138,7 @@ const ImagePickerModal = ({ setModalOpen, setImage }) => {
setModalOpen(false);
}}
onSubmitText={"Save"}
onSubmitDisabled={uploading}
>
<p>
Please note: if you attach an image directly from Discord, it will not be visible in
@@ -46,10 +149,65 @@ const ImagePickerModal = ({ setModalOpen, setImage }) => {
class="input"
id="urlInput"
placeholder="Image URL..."
value={value}
disabled={uploading}
onInput={(ev) => {
setValue(ev.currentTarget.value);
}}
></input>
{uploadsEnabled && (
<div
{...handlers}
class={`mt-4 image-upload-dropzone ${dragging ? "image-picker-dragging" : ""}`}
>
<p>or…</p>
<input
ref={fileInput}
type="file"
accept="image/png,image/jpeg,image/gif,image/webp"
hidden
disabled={uploading}
aria-label="Choose an image to upload"
onChange={(event) => {
const file = event.currentTarget.files?.[0];
event.currentTarget.value = "";
if (file) void upload([file]);
}}
/>
<button
type="button"
class={`button is-info ${uploading ? "is-loading" : ""}`}
disabled={uploading}
onClick={() => fileInput.current?.click()}
>
Upload image
</button>
<p class="help">Or drag and drop an image here.</p>
<p class="help">
Static PNG, JPEG, GIF or WebP, up to 8 MiB. All uploads are converted to
PNGs.
</p>
{blocked && (
<>
<p class="help" style="color: red;">
This image was flagged as potentially NSFW. Unfortunately, due to
the UK's Online Safety Act, we cannot handle explicit content.
Please defer to using Imgur or another image storage platform who
have the resources to deal with idiotic laws.
</p>
<p class="help" style="color: red;">
The image you attempted to upload has not been stored.
</p>
</>
)}
{uploading && <p role="status">Uploading image…</p>}
</div>
)}
{error && (
<p class="help is-danger" role="alert">
{error}
</p>
)}
</Modal>
);
};
@@ -34,3 +34,24 @@ div.reminderContent {
background-color: #f5f5f5;
border-radius: 8px;
}
.image-upload-dropzone {
border: 2px dashed #b5b5b5;
border-radius: 4px;
padding: 1rem;
text-align: center;
}
.image-picker-dragging {
outline: 3px solid #3e8ed0;
outline-offset: 2px;
img {
opacity: 0.6;
}
&.image-upload-dropzone {
background-color: #eff5fb;
border-color: #3e8ed0;
}
}
+155
View File
@@ -0,0 +1,155 @@
use std::env;
use log::{info, warn};
use sqlx::{Acquire, MySql, Pool};
use crate::{
image_upload::{is_upload_path, IMAGE_REFERENCES},
web::image_uploads::ImageUploadService,
Error,
};
const DEFAULT_RETENTION_SECONDS: u32 = 7 * 24 * 60 * 60;
const BATCH_SIZE: u32 = 1000;
fn retention_seconds(value: Option<&str>) -> Result<u32, Error> {
match value.map(str::trim).filter(|value| !value.is_empty()) {
None => Ok(DEFAULT_RETENTION_SECONDS),
Some(value) => value.parse::<u32>().ok().filter(|value| *value > 0).ok_or_else(|| {
"IMAGE_UPLOAD_ORPHAN_RETENTION_SECONDS must be a positive integer of seconds".into()
}),
}
}
fn unreferenced_predicate() -> String {
["reminders", "reminder_template"]
.into_iter()
.flat_map(|table| {
IMAGE_REFERENCES.into_iter().map(move |(_, column)| {
format!("NOT EXISTS (SELECT 1 FROM {table} WHERE {column} = upload.id)")
})
})
.collect::<Vec<_>>()
.join(" AND ")
}
async fn mark_orphans(pool: &Pool<MySql>) -> Result<(), Error> {
sqlx::query(&format!(
"UPDATE image_uploads AS upload SET orphaned_at = CASE
WHEN {} THEN COALESCE(upload.orphaned_at, UTC_TIMESTAMP(6)) ELSE NULL END",
unreferenced_predicate()
))
.execute(pool)
.await?;
Ok(())
}
pub(crate) async fn run(pool: &Pool<MySql>) -> Result<(), Error> {
let retention =
retention_seconds(env::var("IMAGE_UPLOAD_ORPHAN_RETENTION_SECONDS").ok().as_deref())?;
let Some(storage) = ImageUploadService::for_cleanup()? else {
info!("Image cleanup skipped: S3 uploads are not configured");
return Ok(());
};
cleanup(pool, &storage, retention).await
}
async fn cleanup(
pool: &Pool<MySql>,
storage: &ImageUploadService,
retention: u32,
) -> Result<(), Error> {
mark_orphans(pool).await?;
let candidates: Vec<u64> = sqlx::query_scalar(
"SELECT id FROM image_uploads
WHERE orphaned_at <= UTC_TIMESTAMP(6) - INTERVAL ? SECOND
ORDER BY orphaned_at, id LIMIT ?",
)
.bind(retention)
.bind(BATCH_SIZE)
.fetch_all(pool)
.await?;
let mut deleted = 0;
let mut failed = 0;
for id in candidates {
match delete_orphan(pool, storage, retention, id).await {
Ok(true) => deleted += 1,
Ok(false) => {}
Err(error) => {
failed += 1;
warn!("Could not clean up image upload {}: {}", id, error);
}
}
}
info!("Image cleanup deleted {} uploads; {} failed", deleted, failed);
if failed > 0 {
return Err(format!("Image cleanup failed for {failed} uploads; retained for retry").into());
}
Ok(())
}
async fn delete_orphan(
pool: &Pool<MySql>,
storage: &ImageUploadService,
retention: u32,
id: u64,
) -> Result<bool, Error> {
let mut connection = pool.acquire().await?;
sqlx::query("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ")
.execute(&mut *connection)
.await?;
let mut transaction = connection.begin().await?;
// Serialize overlapping cleanup jobs and recheck the retention time under the row lock.
let path: Option<String> = sqlx::query_scalar(
"SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads
WHERE id = ? AND orphaned_at <= UTC_TIMESTAMP(6) - INTERVAL ? SECOND
FOR UPDATE",
)
.bind(id)
.bind(retention)
.fetch_optional(&mut *transaction)
.await?;
let Some(path) = path else {
transaction.rollback().await?;
return Ok(false);
};
if !is_upload_path(&path) {
return Err("Upload path does not match a generated S3 object key".into());
}
for table in ["reminders", "reminder_template"] {
for (_, column) in IMAGE_REFERENCES {
// Each probe uses the FK index. Locking reads see committed references.
// The exclusive upload row lock blocks new FK references until commit.
let referenced: Option<u32> = sqlx::query_scalar(&format!(
"SELECT id FROM {table} WHERE {column} = ? LIMIT 1 FOR UPDATE"
))
.bind(id)
.fetch_optional(&mut *transaction)
.await?;
if referenced.is_some() {
sqlx::query("UPDATE image_uploads SET orphaned_at = NULL WHERE id = ?")
.bind(id)
.execute(&mut *transaction)
.await?;
transaction.commit().await?;
return Ok(false);
}
}
}
let response = storage.bucket.delete_object(&path).await?;
// A missing object is already cleaned up, including a retry after a DB commit failure.
if !(200..300).contains(&response.status_code()) && response.status_code() != 404 {
return Err(format!("S3 deletion returned HTTP {}", response.status_code()).into());
}
sqlx::query("DELETE FROM image_uploads WHERE id = ?")
.bind(id)
.execute(&mut *transaction)
.await?;
transaction.commit().await?;
Ok(true)
}
#[cfg(test)]
mod tests;
+310
View File
@@ -0,0 +1,310 @@
use std::{sync::Arc, time::Duration};
use chrono::NaiveDateTime;
use s3::{creds::Credentials, Bucket, Region};
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
sync::Semaphore,
task::JoinHandle,
};
use super::*;
const BASE_URL: &str = "https://images.example.com/bucket";
fn upload_path(number: u64) -> String {
format!("uploads/{number:064x}.png")
}
async fn upload(pool: &Pool<MySql>, number: u64) -> u64 {
sqlx::query("INSERT INTO image_uploads (uploader, path) VALUES (123, ?)")
.bind(upload_path(number))
.execute(pool)
.await
.unwrap()
.last_insert_id()
}
async fn orphaned_at(pool: &Pool<MySql>, id: u64) -> Option<NaiveDateTime> {
sqlx::query_scalar("SELECT orphaned_at FROM image_uploads WHERE id = ?")
.bind(id)
.fetch_one(pool)
.await
.unwrap()
}
async fn expire(pool: &Pool<MySql>, id: u64) {
sqlx::query(
"UPDATE image_uploads SET orphaned_at = UTC_TIMESTAMP(6) - INTERVAL 2 DAY WHERE id = ?",
)
.bind(id)
.execute(pool)
.await
.unwrap();
}
async fn reference_tables(pool: &Pool<MySql>) -> (u64, u64) {
let guild = sqlx::query("INSERT INTO guilds (guild) VALUES (123)")
.execute(pool)
.await
.unwrap()
.last_insert_id();
let channel = sqlx::query("INSERT INTO channels (channel, guild_id) VALUES (123, ?)")
.bind(guild)
.execute(pool)
.await
.unwrap()
.last_insert_id();
(guild, channel)
}
async fn storage(status: u16) -> (ImageUploadService, JoinHandle<String>) {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let task = tokio::spawn(async move {
let (mut socket, _) =
tokio::time::timeout(Duration::from_secs(5), listener.accept()).await.unwrap().unwrap();
let mut received = Vec::new();
let mut buffer = [0; 4096];
while !received.windows(4).any(|window| window == b"\r\n\r\n") {
let count = socket.read(&mut buffer).await.unwrap();
assert!(count > 0);
received.extend_from_slice(&buffer[..count]);
}
socket
.write_all(
format!("HTTP/1.1 {status} Test\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.as_bytes(),
)
.await
.unwrap();
String::from_utf8(received).unwrap()
});
let credentials = Credentials::new(Some("test"), Some("test"), None, None, None).unwrap();
let mut bucket =
Bucket::new("images", Region::Custom { region: "local".into(), endpoint }, credentials)
.unwrap()
.with_path_style();
bucket.set_request_timeout(Some(Duration::from_secs(2)));
(
ImageUploadService {
bucket,
public_url: BASE_URL.into(),
model: None,
slots: Arc::new(Semaphore::new(2)),
},
task,
)
}
#[test]
fn retention_defaults_and_rejects_invalid_settings() {
for value in [None, Some(""), Some(" ")] {
assert_eq!(retention_seconds(value).unwrap(), 604800);
}
assert_eq!(retention_seconds(Some(" 3600 ")).unwrap(), 3600);
for value in ["0", "-1", "1.5", "forever", "4294967296"] {
assert!(retention_seconds(Some(value)).is_err());
}
}
#[test]
fn restricts_deletion_to_generated_bucket_paths() {
assert!(is_upload_path(&upload_path(1)));
for path in [
format!("{BASE_URL}/{}", upload_path(1)),
upload_path(1).replace("uploads/", "private/"),
"uploads/../private.png".into(),
format!("uploads/{}.png", "g".repeat(64)),
format!("{}?download=true", upload_path(1)),
"uploads/short.png".into(),
] {
assert!(!is_upload_path(&path), "{path}");
}
}
#[sqlx::test]
async fn orphan_timestamp_survives_repeated_runs_and_resets_on_reuse(pool: Pool<MySql>) {
let id = upload(&pool, 1).await;
assert!(orphaned_at(&pool, id).await.is_none());
mark_orphans(&pool).await.unwrap();
let first = orphaned_at(&pool, id).await.unwrap();
mark_orphans(&pool).await.unwrap();
assert_eq!(orphaned_at(&pool, id).await, Some(first));
let (guild, _) = reference_tables(&pool).await;
let template =
sqlx::query("INSERT INTO reminder_template (guild_id, embed_image_id) VALUES (?, ?)")
.bind(guild)
.bind(id)
.execute(&pool)
.await
.unwrap()
.last_insert_id();
mark_orphans(&pool).await.unwrap();
assert!(orphaned_at(&pool, id).await.is_none());
sqlx::query("DELETE FROM reminder_template WHERE id = ?")
.bind(template)
.execute(&pool)
.await
.unwrap();
mark_orphans(&pool).await.unwrap();
assert!(orphaned_at(&pool, id).await.unwrap() >= first);
}
#[sqlx::test]
async fn protects_all_image_fields_in_reminders_and_templates(pool: Pool<MySql>) {
let (guild, channel) = reference_tables(&pool).await;
let mut ids = Vec::new();
for (index, (_, column)) in IMAGE_REFERENCES.into_iter().enumerate() {
for table in ["reminders", "reminder_template"] {
let number = ids.len() as u64 + 1;
let id = upload(&pool, number).await;
ids.push(id);
let query = if table == "reminders" {
// Archived/disabled reminders still retain their image references.
format!(
"INSERT INTO reminders (uid, channel_id, `utc_time`, status, enabled, {column})
VALUES ('field-{index}', {channel}, UTC_TIMESTAMP(), 'sent', 0, ?)"
)
} else {
format!("INSERT INTO reminder_template (guild_id, {column}) VALUES ({guild}, ?)")
};
sqlx::query(&query).bind(id).execute(&pool).await.unwrap();
}
}
let unused = upload(&pool, 100).await;
// Previously orphaned files are unmarked as soon as cleanup observes a reference.
for id in &ids {
expire(&pool, *id).await;
}
mark_orphans(&pool).await.unwrap();
for id in ids {
assert!(orphaned_at(&pool, id).await.is_none());
}
assert!(orphaned_at(&pool, unused).await.is_some());
}
#[sqlx::test]
async fn only_expired_orphans_are_deleted_and_rows_removed_after_success(pool: Pool<MySql>) {
let old = upload(&pool, 1).await;
let fresh = upload(&pool, 2).await;
expire(&pool, old).await;
let (storage, request) = storage(204).await;
cleanup(&pool, &storage, 86400).await.unwrap();
assert!(request.await.unwrap().starts_with(&format!("DELETE /images/uploads/{:064x}.png ", 1)));
let ids: Vec<u64> = sqlx::query_scalar("SELECT id FROM image_uploads ORDER BY id")
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(ids, vec![fresh]);
assert!(orphaned_at(&pool, fresh).await.is_some());
}
#[sqlx::test]
async fn failed_deletions_keep_the_original_timestamp_and_retry_missing_objects(pool: Pool<MySql>) {
let id = upload(&pool, 1).await;
expire(&pool, id).await;
let original = orphaned_at(&pool, id).await;
let (failed_storage, failed_request) = storage(403).await;
assert!(cleanup(&pool, &failed_storage, 86400).await.is_err());
failed_request.await.unwrap();
assert_eq!(orphaned_at(&pool, id).await, original);
let (retry_storage, retry_request) = storage(404).await;
cleanup(&pool, &retry_storage, 86400).await.unwrap();
retry_request.await.unwrap();
let count: i64 =
sqlx::query_scalar("SELECT COUNT(*) FROM image_uploads").fetch_one(&pool).await.unwrap();
assert_eq!(count, 0);
}
#[sqlx::test]
async fn rechecks_references_before_deleting_a_candidate(pool: Pool<MySql>) {
let id = upload(&pool, 1).await;
expire(&pool, id).await;
let (guild, _) = reference_tables(&pool).await;
sqlx::query("INSERT INTO reminder_template (guild_id, avatar_id) VALUES (?, ?)")
.bind(guild)
.bind(id)
.execute(&pool)
.await
.unwrap();
let (storage, request) = storage(204).await;
assert!(!delete_orphan(&pool, &storage, 86400, id).await.unwrap());
assert!(orphaned_at(&pool, id).await.is_none());
request.abort();
}
#[sqlx::test]
async fn invalid_paths_are_retained_without_contacting_s3(pool: Pool<MySql>) {
let id = upload(&pool, 1).await;
sqlx::query(
"UPDATE image_uploads SET path = 'https://elsewhere.example.com/file.png' WHERE id = ?",
)
.bind(id)
.execute(&pool)
.await
.unwrap();
expire(&pool, id).await;
let (storage, request) = storage(204).await;
assert!(cleanup(&pool, &storage, 86400).await.is_err());
assert!(orphaned_at(&pool, id).await.is_some());
request.abort();
}
#[sqlx::test]
async fn cleanup_blocks_new_references_until_deletion_finishes(pool: Pool<MySql>) {
let id = upload(&pool, 1).await;
expire(&pool, id).await;
let (guild, _) = reference_tables(&pool).await;
let (mut storage, unused_request) = storage(204).await;
unused_request.abort();
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
storage.bucket = Bucket::new(
"images",
Region::Custom { region: "local".into(), endpoint },
Credentials::new(Some("test"), Some("test"), None, None, None).unwrap(),
)
.unwrap()
.with_path_style();
let (request_started, wait_for_request) = tokio::sync::oneshot::channel();
let (finish_delete, wait_to_finish) = tokio::sync::oneshot::channel();
let server = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut received = Vec::new();
let mut buffer = [0; 4096];
while !received.windows(4).any(|window| window == b"\r\n\r\n") {
let count = socket.read(&mut buffer).await.unwrap();
assert!(count > 0);
received.extend_from_slice(&buffer[..count]);
}
request_started.send(()).unwrap();
wait_to_finish.await.unwrap();
socket
.write_all(b"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.unwrap();
});
let cleanup_pool = pool.clone();
let cleanup =
tokio::spawn(async move { delete_orphan(&cleanup_pool, &storage, 86400, id).await });
tokio::time::timeout(Duration::from_secs(5), wait_for_request).await.unwrap().unwrap();
let mut writer = tokio::spawn(async move {
sqlx::query("INSERT INTO reminder_template (guild_id, embed_image_id) VALUES (?, ?)")
.bind(guild)
.bind(id)
.execute(&pool)
.await
});
assert!(tokio::time::timeout(Duration::from_millis(100), &mut writer).await.is_err());
finish_delete.send(()).unwrap();
assert!(cleanup.await.unwrap().unwrap());
// The writer resumes after cleanup commits, and the FK rejects the deleted upload.
let error = writer.await.unwrap().unwrap_err();
assert_eq!(error.as_database_error().unwrap().code().as_deref(), Some("23000"));
server.await.unwrap();
}
+94
View File
@@ -0,0 +1,94 @@
use std::env;
use serde::{Serialize, Serializer};
use sqlx::Executor;
use crate::Database;
pub const IMAGE_REFERENCES: [(&str, &str); 5] = [
("avatar", "avatar_id"),
("embed_image_url", "embed_image_id"),
("embed_thumbnail_url", "embed_thumbnail_id"),
("embed_author_url", "embed_author_id"),
("embed_footer_url", "embed_footer_id"),
];
// Only paths generated by the upload route belong to managed storage.
pub fn is_upload_path(path: &str) -> bool {
path.strip_prefix("uploads/").and_then(|name| name.strip_suffix(".png")).is_some_and(|name| {
name.len() == 64
&& name.bytes().all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte))
})
}
fn public_url() -> String {
env::var("S3_PUBLIC_URL")
.ok()
.and_then(|value| reqwest::Url::parse(&value).ok())
.map(|url| url.as_str().trim_end_matches('/').to_string())
.unwrap_or_default()
}
fn resolve_with_base(value: &str, base: &str) -> Option<String> {
if is_upload_path(value) {
(!base.is_empty()).then(|| format!("{}/{value}", base.trim_end_matches('/')))
} else {
Some(value.to_string())
}
}
// Read queries select a bucket path for managed images and the original URL otherwise.
// They explicitly collate paths as text for SQLx and legacy URL column collations.
// Resolve at the boundary so changing S3_PUBLIC_URL never requires rewriting reminders.
pub fn resolve_url(value: Option<&str>) -> Option<String> {
value.and_then(|value| resolve_with_base(value, &public_url()))
}
pub fn serialize_url<S>(value: &Option<String>, serializer: S) -> Result<S::Ok, S::Error>
where
S: Serializer,
{
resolve_url(value.as_deref()).serialize(serializer)
}
pub struct ImageReference {
pub url: Option<String>,
pub id: Option<u64>,
}
impl ImageReference {
pub async fn from_url(
executor: impl Executor<'_, Database = Database>,
url: Option<&str>,
) -> Result<Self, sqlx::Error> {
Self::with_base(executor, url, &public_url()).await
}
async fn with_base(
executor: impl Executor<'_, Database = Database>,
url: Option<&str>,
base: &str,
) -> Result<Self, sqlx::Error> {
let path = url
.filter(|_| !base.is_empty())
.and_then(|url| url.strip_prefix(base.trim_end_matches('/')))
.and_then(|path| path.strip_prefix('/'))
.filter(|path| is_upload_path(path));
if let Some(path) = path {
// Hold a shared upload lock until the caller commits its reference. Cleanup
// locks that same row exclusively before checking references and deleting S3.
let id = sqlx::query_scalar(
"SELECT id FROM image_uploads WHERE path = ? LOCK IN SHARE MODE",
)
.bind(path)
.fetch_one(executor)
.await?;
Ok(Self { url: None, id: Some(id) })
} else {
Ok(Self { url: url.map(str::to_string), id: None })
}
}
}
#[cfg(test)]
mod tests;
+124
View File
@@ -0,0 +1,124 @@
use super::*;
use sqlx::{MySql, Pool};
#[test]
fn resolves_paths_using_the_current_base_and_preserves_external_urls() {
let path = format!("uploads/{:064x}.png", 1);
assert_eq!(
resolve_with_base(&path, "https://new.example/bucket/"),
Some(format!("https://new.example/bucket/{path}"))
);
assert_eq!(resolve_with_base(&path, ""), None);
assert_eq!(
resolve_with_base("https://external.example/a.png", "https://new.example"),
Some("https://external.example/a.png".into())
);
for invalid in ["uploads/../private.png", "uploads/short.png", "uploads/ABC.png"] {
assert!(!is_upload_path(invalid));
}
}
#[sqlx::test]
async fn maps_managed_urls_to_ids_and_rejects_deleted_uploads(pool: Pool<MySql>) {
let path = format!("uploads/{:064x}.png", 1);
let id = sqlx::query("INSERT INTO image_uploads (uploader, path) VALUES (123, ?)")
.bind(&path)
.execute(&pool)
.await
.unwrap()
.last_insert_id();
let base = "https://images.example/bucket";
let reference =
ImageReference::with_base(&pool, Some(&format!("{base}/{path}")), base).await.unwrap();
assert_eq!(reference.id, Some(id));
assert_eq!(reference.url, None);
for url in [
None,
Some("https://external.example/a.png"),
Some("https://images.example/bucket-other/a.png"),
] {
let reference = ImageReference::with_base(&pool, url, base).await.unwrap();
assert_eq!(reference.id, None);
assert_eq!(reference.url.as_deref(), url);
}
let missing = format!("{base}/uploads/{:064x}.png", 2);
assert!(matches!(
ImageReference::with_base(&pool, Some(&missing), base).await,
Err(sqlx::Error::RowNotFound)
));
}
#[sqlx::test(migrations = false)]
async fn image_migration_preserves_external_urls_and_adds_indexed_references(pool: Pool<MySql>) {
let migrations = sqlx::migrate!();
let mut connection = pool.acquire().await.unwrap();
for migration in migrations.iter().filter(|migration| migration.version < 20261011120000) {
sqlx::raw_sql(&migration.sql).execute(&mut *connection).await.unwrap();
}
drop(connection);
sqlx::query("INSERT INTO guilds (id, guild) VALUES (1, 123)").execute(&pool).await.unwrap();
sqlx::query("INSERT INTO channels (id, channel, guild_id) VALUES (1, 123, 1)")
.execute(&pool)
.await
.unwrap();
let external = "https://external.example/image.png";
for (index, (column, _)) in IMAGE_REFERENCES.into_iter().enumerate() {
for table in ["reminders", "reminder_template"] {
let sql = if table == "reminders" {
format!("INSERT INTO reminders (uid, channel_id, `utc_time`, {column}) VALUES ('field-{index}', 1, UTC_TIMESTAMP(), ?)")
} else {
format!("INSERT INTO reminder_template (guild_id, {column}) VALUES (1, ?)")
};
sqlx::query(&sql).bind(external).execute(&pool).await.unwrap();
}
}
sqlx::raw_sql(include_str!("../../migrations/20261011120000_image_uploads.sql"))
.execute(&pool)
.await
.unwrap();
let path = format!("uploads/{:064x}.png", 1);
let upload = sqlx::query("INSERT INTO image_uploads (uploader, path) VALUES (123, ?)")
.bind(&path)
.execute(&pool)
.await
.unwrap()
.last_insert_id();
let stored: String = sqlx::query_scalar(
"SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = ?",
)
.bind(upload)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(stored, path);
for (column, id_column) in IMAGE_REFERENCES {
for table in ["reminders", "reminder_template"] {
let values: (Option<String>, Option<u64>) = sqlx::query_as(&format!(
"SELECT {column}, {id_column} FROM {table} WHERE {column} = ?"
))
.bind(external)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(values, (Some(external.into()), None));
let indexed: i64 = sqlx::query_scalar(
"SELECT COUNT(*) FROM information_schema.statistics WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ? AND seq_in_index = 1"
).bind(table).bind(id_column).fetch_one(&pool).await.unwrap();
assert!(indexed > 0);
sqlx::query(&format!(
"UPDATE {table} SET {column} = NULL, {id_column} = ? WHERE {column} = ?"
))
.bind(upload)
.bind(external)
.execute(&pool)
.await
.unwrap();
}
}
// Referenced uploads cannot be removed even outside the cleanup code.
assert!(sqlx::query("DELETE FROM image_uploads WHERE id = ?")
.bind(upload)
.execute(&pool)
.await
.is_err());
}
+3
View File
@@ -9,6 +9,8 @@ mod consts;
mod event_handlers;
#[cfg(not(test))]
mod hooks;
mod image_cleanup;
mod image_upload;
mod interval_parser;
mod metrics;
#[cfg(not(test))]
@@ -121,6 +123,7 @@ async fn _main(tx: Sender<()>) -> Result<(), Box<dyn StdError + Send + Sync>> {
Pool::connect(&env::var("DATABASE_URL").expect("No database URL provided")).await.unwrap();
match cmd_word {
Some("clean-images") => image_cleanup::run(&database).await,
Some("clean") => {
let sent_clean_age = env::var("SENT_CLEAN_AGE").expect("No SENT_CLEAN_AGE provided");
if sent_clean_age.is_empty() {
+11 -5
View File
@@ -154,12 +154,12 @@ impl Embed {
SELECT
`embed_title` AS title,
`embed_description` AS description,
`embed_image_url` AS image_url,
`embed_thumbnail_url` AS thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), `embed_image_url`) AS image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), `embed_thumbnail_url`) AS thumbnail_url,
`embed_footer` AS footer,
`embed_footer_url` AS footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), `embed_footer_url`) AS footer_url,
`embed_author` AS author,
`embed_author_url` AS author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), `embed_author_url`) AS author_url,
`embed_color` AS color,
IFNULL(`embed_fields`, '[]') AS "fields:_"
FROM reminders
@@ -179,6 +179,11 @@ impl Embed {
field.value = substitute(&field.value);
});
embed.image_url = crate::image_upload::resolve_url(embed.image_url.as_deref());
embed.thumbnail_url = crate::image_upload::resolve_url(embed.thumbnail_url.as_deref());
embed.author_url = crate::image_upload::resolve_url(embed.author_url.as_deref());
embed.footer_url = crate::image_upload::resolve_url(embed.footer_url.as_deref());
if embed.has_content() {
Some(embed)
} else {
@@ -387,7 +392,7 @@ impl Reminder {
reminders.`interval_days` AS 'interval_days',
reminders.`interval_months` AS 'interval_months',
reminders.`avatar` AS avatar,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,
reminders.`username` AS username
FROM
reminders
@@ -422,6 +427,7 @@ impl Reminder {
.into_iter()
.map(|mut rem| {
rem.content = substitute(&rem.content);
rem.avatar = crate::image_upload::resolve_url(rem.avatar.as_deref());
rem
})
+295
View File
@@ -0,0 +1,295 @@
use std::{env, io::Cursor, sync::Arc};
use image::{ImageFormat, ImageReader, Limits};
use log::{info, warn};
use nsfw::model::{Classification, Metric};
use s3::{creds::Credentials, Bucket, Region};
use tokio::sync::Semaphore;
pub const MAX_UPLOAD_BYTES: u64 = 8 * 1024 * 1024;
const MAX_IMAGE_PIXELS: u64 = 16_000_000;
const NSFW_THRESHOLD: f32 = 0.5;
const MODEL_BYTES: &[u8] =
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/assets/nsfw/model.onnx"));
// An empty configuration keeps the dashboard's existing URL-only behavior.
pub struct ImageUploads(pub Option<Arc<ImageUploadService>>);
pub struct ImageUploadService {
pub bucket: Box<Bucket>,
pub public_url: String,
pub model: Option<nsfw::Model>,
pub slots: Arc<Semaphore>,
}
impl ImageUploads {
pub fn from_env() -> Self {
let names =
["S3_BUCKET", "S3_REGION", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_PUBLIC_URL"];
let values = names.map(|name| env::var(name).unwrap_or_default());
Self::from_settings(&values)
}
fn from_settings(values: &[String; 5]) -> Self {
if values.iter().all(|value| value.trim().is_empty()) {
return Self(None);
}
if values.iter().any(|value| value.trim().is_empty()) {
warn!("Image uploads disabled: all S3 upload settings must be supplied");
return Self(None);
}
let check_nsfw = nsfw_check_enabled(env::var("NSFW_CHECK_ENABLED").ok().as_deref());
match ImageUploadService::new(values, check_nsfw) {
Ok(service) => {
info!("Dashboard image uploads enabled");
Self(Some(Arc::new(service)))
}
Err(error) => {
warn!("Image uploads disabled: {}", error);
Self(None)
}
}
}
}
impl ImageUploadService {
// Cleanup uses the same storage configuration without loading the moderation model.
pub(crate) fn for_cleanup() -> Result<Option<Self>, Box<dyn std::error::Error + Send + Sync>> {
let values =
["S3_BUCKET", "S3_REGION", "S3_ACCESS_KEY_ID", "S3_SECRET_ACCESS_KEY", "S3_PUBLIC_URL"]
.map(|name| env::var(name).unwrap_or_default());
if values.iter().all(|value| value.trim().is_empty()) {
return Ok(None);
}
if values.iter().any(|value| value.trim().is_empty()) {
return Err("Image cleanup requires all S3 upload settings".into());
}
Self::new(&values, false).map(Some).map_err(|error| error.to_string().into())
}
fn new(values: &[String; 5], check_nsfw: bool) -> Result<Self, Box<dyn std::error::Error>> {
let url = reqwest::Url::parse(&values[4])?;
let public_url = url.as_str().trim_end_matches('/').to_string();
if !matches!(url.scheme(), "https" | "http")
|| url.host_str().is_none()
|| !url.username().is_empty()
|| url.password().is_some()
|| url.query().is_some()
|| url.fragment().is_some()
|| public_url.len() > 400
{
return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into());
}
let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty());
let region = match &endpoint {
Some(endpoint) => {
Region::Custom { region: values[1].clone(), endpoint: endpoint.clone() }
}
None => values[1].parse()?,
};
let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?;
let mut bucket = Bucket::new(&values[0], region, credentials)?;
if endpoint.is_some() {
bucket = bucket.with_path_style();
}
bucket.set_request_timeout(Some(std::time::Duration::from_secs(30)));
let model = if check_nsfw {
Some(nsfw::create_model(MODEL_BYTES)?)
} else {
info!("NSFW checks disabled for dashboard image uploads");
None
};
Ok(Self { bucket, public_url, model, slots: Arc::new(Semaphore::new(2)) })
}
}
fn nsfw_check_enabled(value: Option<&str>) -> bool {
match value.map(str::trim).map(str::to_ascii_lowercase).as_deref() {
None | Some("" | "true" | "1" | "on") => true,
Some("false" | "0" | "off") => false,
Some(_) => {
warn!("Unrecognized NSFW_CHECK_ENABLED value; keeping NSFW checks enabled");
true
}
}
}
#[derive(Debug, PartialEq)]
pub enum ImageError {
InvalidImage,
Nsfw,
ModerationFailed,
}
fn probable_nsfw(predictions: &[Classification]) -> Result<bool, ImageError> {
if predictions.len() != 5
|| predictions.iter().any(|prediction| {
!prediction.score.is_finite() || !(0.0..=1.0).contains(&prediction.score)
})
{
return Err(ImageError::ModerationFailed);
}
let score: f32 = predictions
.iter()
.filter(|prediction| {
matches!(prediction.metric, Metric::Porn | Metric::Hentai | Metric::Sexy)
})
.map(|prediction| prediction.score)
.sum();
Ok(score >= NSFW_THRESHOLD)
}
// Publish only the decoded pixels, which are moderated when NSFW checks are enabled.
// Static PNGs strip metadata and prevent unchecked animation frames from being published.
fn decode_image(bytes: &[u8]) -> Result<image::RgbaImage, ImageError> {
let mut reader = ImageReader::new(Cursor::new(bytes))
.with_guessed_format()
.map_err(|_| ImageError::InvalidImage)?;
if !matches!(
reader.format(),
Some(ImageFormat::Jpeg | ImageFormat::Png | ImageFormat::Gif | ImageFormat::WebP)
) {
return Err(ImageError::InvalidImage);
}
let mut limits = Limits::default();
limits.max_image_width = Some(8192);
limits.max_image_height = Some(8192);
limits.max_alloc = Some(128 * 1024 * 1024);
reader.limits(limits);
let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?;
use image::ImageDecoder;
let (width, height) = decoder.dimensions();
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS {
return Err(ImageError::InvalidImage);
}
image::DynamicImage::from_decoder(decoder)
.map(|image| image.to_rgba8())
.map_err(|_| ImageError::InvalidImage)
}
pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8>, ImageError> {
let image = decode_image(bytes)?;
// Flatten transparency so the classifier sees exactly the published image.
let mut visible = image::RgbImage::new(image.width(), image.height());
for (target, source) in visible.pixels_mut().zip(image.pixels()) {
for channel in 0..3 {
let alpha = u32::from(source[3]);
target[channel] =
((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8;
}
}
let visible = image::DynamicImage::ImageRgb8(visible);
if let Some(model) = model {
let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| {
warn!("Image moderation failed: {}", error);
ImageError::ModerationFailed
})?;
if probable_nsfw(&predictions)? {
return Err(ImageError::Nsfw);
}
}
let mut output = Cursor::new(Vec::new());
visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?;
Ok(output.into_inner())
}
#[cfg(test)]
mod tests {
use super::*;
fn predictions(scores: [f32; 5]) -> Vec<Classification> {
[Metric::Drawings, Metric::Hentai, Metric::Neutral, Metric::Porn, Metric::Sexy]
.into_iter()
.zip(scores)
.map(|(metric, score)| Classification { metric, score })
.collect()
}
#[test]
fn combines_unsafe_categories_and_rejects_invalid_scores() {
assert_eq!(probable_nsfw(&predictions([0.1, 0.2, 0.4, 0.2, 0.1])), Ok(true));
assert_eq!(probable_nsfw(&predictions([0.1, 0.1, 0.7, 0.05, 0.05])), Ok(false));
assert_eq!(
probable_nsfw(&predictions([0.0, 0.0, f32::NAN, 0.0, 0.0])),
Err(ImageError::ModerationFailed)
);
assert_eq!(probable_nsfw(&[]), Err(ImageError::ModerationFailed));
assert_eq!(
probable_nsfw(&predictions([0.0, -0.5, 1.0, 0.0, 0.0])),
Err(ImageError::ModerationFailed)
);
}
#[test]
fn absent_or_partial_configuration_disables_uploads() {
let mut settings = std::array::from_fn(|_| String::new());
assert!(ImageUploads::from_settings(&settings).0.is_none());
settings[0] = "images".into();
assert!(ImageUploads::from_settings(&settings).0.is_none());
}
#[test]
fn disabling_nsfw_checks_skips_model_initialization() {
let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"]
.map(str::to_string);
assert!(ImageUploadService::new(&settings, false).unwrap().model.is_none());
}
#[test]
fn disabled_moderation_allows_flagged_images_but_still_validates_files() {
let model = nsfw::create_model(
&include_bytes!("../../tests/fixtures/image-upload-unsafe.onnx")[..],
)
.unwrap();
let mut bytes = Cursor::new(Vec::new());
image::DynamicImage::new_rgb8(2, 2).write_to(&mut bytes, ImageFormat::Png).unwrap();
assert_eq!(prepare_image(Some(&model), bytes.get_ref()), Err(ImageError::Nsfw));
let published = prepare_image(None, bytes.get_ref()).unwrap();
assert_eq!(image::load_from_memory(&published).unwrap().width(), 2);
assert_eq!(prepare_image(None, b"not an image"), Err(ImageError::InvalidImage));
}
#[test]
fn bundled_model_loads_and_classifies_an_image() {
let model = nsfw::create_model(MODEL_BYTES).unwrap();
let image = image::RgbaImage::from_pixel(224, 224, image::Rgba([255, 255, 255, 255]));
let predictions = nsfw::examine(&model, &image).unwrap();
assert!(probable_nsfw(&predictions).is_ok());
}
#[test]
fn rejects_non_images_and_truncated_images() {
assert_eq!(decode_image(b"<svg></svg>"), Err(ImageError::InvalidImage));
assert_eq!(decode_image(b"\x89PNG\r\n\x1a\n"), Err(ImageError::InvalidImage));
}
#[test]
fn decodes_real_images_without_trusting_the_filename() {
let image = image::DynamicImage::new_rgb8(2, 2);
for format in [ImageFormat::Png, ImageFormat::Jpeg, ImageFormat::Gif, ImageFormat::WebP] {
let mut bytes = Cursor::new(Vec::new());
image.write_to(&mut bytes, format).unwrap();
assert_eq!(decode_image(bytes.get_ref()).unwrap().dimensions(), (2, 2));
}
}
#[test]
fn rejects_excessive_image_dimensions() {
let image = image::DynamicImage::new_rgb8(8193, 1);
let mut bytes = Cursor::new(Vec::new());
image.write_to(&mut bytes, ImageFormat::Png).unwrap();
assert_eq!(decode_image(bytes.get_ref()), Err(ImageError::InvalidImage));
}
}
+31
View File
@@ -109,3 +109,34 @@ macro_rules! path {
}
}};
}
macro_rules! update_image_field {
($pool:expr, $error:ident, $reminder:ident, $field:ident, $id:ident) => {
if let Some(value) = &$reminder.$field {
match crate::image_upload::ImageReference::from_url($pool, value.as_deref()).await {
Ok(image) => {
if let Err(e) = sqlx::query(concat!(
"UPDATE reminders SET `",
stringify!($field),
"` = ?, `",
stringify!($id),
"` = ? WHERE uid = ?"
))
.bind(image.url)
.bind(image.id)
.bind(&$reminder.uid)
.execute($pool)
.await
{
warn!("Could not update image reference: {:?}", e);
$error.push(format!("Error setting field {}", stringify!($field)));
}
}
Err(e) => {
warn!("Could not resolve image upload: {:?}", e);
$error.push(format!("Error setting field {}", stringify!($field)));
}
}
}
};
}
+7 -4
View File
@@ -4,6 +4,7 @@ mod macros;
mod catchers;
mod fairings;
mod guards;
pub(crate) mod image_uploads;
mod ip_blocking;
mod routes;
@@ -124,7 +125,8 @@ pub async fn initialize(
if Path::new("static").exists() { "static" } else { "/lib/reminder-rs/static" };
info!("Using static path: {}", static_path);
rocket::build()
let image_uploads = image_uploads::ImageUploads::from_env();
let rocket = rocket::build()
.attach(MetricProducer)
.attach(Template::fairing())
.register(
@@ -214,9 +216,10 @@ pub async fn initialize(
routes::dashboard::export::todos::export,
routes::dashboard::export::todos::import,
],
)
.launch()
.await?;
);
let rocket = routes::dashboard::api::images::mount(rocket, image_uploads);
rocket.launch().await?;
warn!("Exiting rocket runtime");
// distribute kill signal
+22 -15
View File
@@ -81,17 +81,17 @@ pub async fn get_reminders(
"
SELECT
reminders.attachment_name,
reminders.avatar,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,
channels.channel,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
IFNULL(reminders.embed_fields, '[]') AS embed_fields,
reminders.enabled,
@@ -169,12 +169,7 @@ pub async fn edit_reminder(
update_field!(transaction.executor(), error, reminder.[
attachment,
attachment_name,
avatar,
embed_author_url,
embed_color,
embed_footer_url,
embed_image_url,
embed_thumbnail_url,
enabled,
expires,
name,
@@ -183,6 +178,18 @@ pub async fn edit_reminder(
utc_time
]);
update_image_field!(transaction.executor(), error, reminder, avatar, avatar_id);
update_image_field!(transaction.executor(), error, reminder, embed_image_url, embed_image_id);
update_image_field!(
transaction.executor(),
error,
reminder,
embed_thumbnail_url,
embed_thumbnail_id
);
update_image_field!(transaction.executor(), error, reminder, embed_author_url, embed_author_id);
update_image_field!(transaction.executor(), error, reminder, embed_footer_url, embed_footer_id);
if reminder.interval_days.flatten().is_some()
|| reminder.interval_months.flatten().is_some()
|| reminder.interval_seconds.flatten().is_some()
@@ -347,17 +354,17 @@ pub async fn edit_reminder(
"
SELECT
reminders.attachment_name,
reminders.avatar,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,
channels.channel,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
reminders.embed_fields,
reminders.enabled,
@@ -31,7 +31,28 @@ pub async fn get_reminder_templates(
match sqlx::query_as_unchecked!(
ReminderTemplate,
"SELECT * FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
"SELECT id,
guild_id,
name,
attachment,
attachment_name,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.avatar_id), avatar) AS avatar,
content,
embed_author,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_author_id), embed_author_url) AS embed_author_url,
embed_color,
embed_description,
embed_footer,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_footer_id), embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_image_id), embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_thumbnail_id), embed_thumbnail_url) AS embed_thumbnail_url,
embed_title,
embed_fields,
interval_seconds,
interval_days,
interval_months,
tts,
username FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
id
)
.fetch_all(pool.inner())
+326
View File
@@ -0,0 +1,326 @@
use log::warn;
use rand::{rngs::OsRng, RngCore};
use rocket::{
data::{Data, ToByteUnit},
http::{CookieJar, Status},
post,
request::{FromRequest, Outcome},
serde::json::{json, Json, Value},
Request, State,
};
use sqlx::{MySql, Pool};
use crate::web::image_uploads::{prepare_image, ImageError, ImageUploads, MAX_UPLOAD_BYTES};
pub fn mount(
rocket: rocket::Rocket<rocket::Build>,
uploads: ImageUploads,
) -> rocket::Rocket<rocket::Build> {
let enabled = uploads.0.is_some();
let rocket = rocket.manage(uploads);
if enabled {
rocket.mount("/dashboard", rocket::routes![upload_image])
} else {
rocket
}
}
pub struct Uploader(u64);
#[rocket::async_trait]
impl<'r> FromRequest<'r> for Uploader {
type Error = ();
async fn from_request(request: &'r Request<'_>) -> Outcome<Self, Self::Error> {
match uploader_id(request.cookies()) {
Some(id) => Outcome::Success(Self(id)),
None => Outcome::Error((Status::Unauthorized, ())),
}
}
}
fn uploader_id(cookies: &CookieJar<'_>) -> Option<u64> {
cookies.get_private("userid")?.value().parse().ok().filter(|id| *id != 0)
}
type UploadResult = Result<Json<Value>, (Status, Json<Value>)>;
fn error(status: Status, message: &str) -> (Status, Json<Value>) {
(status, Json(json!({ "error": message })))
}
// Requiring a non-form content type prevents cross-site HTML form submissions.
// Authorization is a request guard, so it runs before the body is read.
#[post("/api/images", format = "application/octet-stream", data = "<data>")]
pub async fn upload_image(
uploader: Uploader,
uploads: &State<ImageUploads>,
pool: &State<Pool<MySql>>,
data: Data<'_>,
) -> UploadResult {
let service =
uploads.0.as_ref().ok_or_else(|| error(Status::NotFound, "Image uploads disabled"))?;
let _permit = service
.slots
.clone()
.try_acquire_owned()
.map_err(|_| error(Status::TooManyRequests, "Image uploads busy. Please try again."))?;
let bytes = data
.open(MAX_UPLOAD_BYTES.bytes())
.into_bytes()
.await
.map_err(|_| error(Status::BadRequest, "Could not read image"))?;
if !bytes.is_complete() {
return Err(error(Status::PayloadTooLarge, "Images must be no larger than 8 MiB"));
}
let service_for_moderation = service.clone();
let moderated = tokio::task::spawn_blocking(move || {
prepare_image(service_for_moderation.model.as_ref(), &bytes.into_inner())
})
.await
.map_err(|err| {
warn!("Image moderation task failed: {}", err);
error(Status::ServiceUnavailable, "Image moderation unavailable. Please try again.")
})?;
let image = moderated.map_err(|err| match err {
ImageError::InvalidImage => error(
Status::UnprocessableEntity,
"Please upload a valid PNG, JPEG, GIF or WebP image (at most 16 million pixels)",
),
ImageError::Nsfw => error(
Status::UnavailableForLegalReasons,
"This image was flagged as probably NSFW and cannot be uploaded.",
),
ImageError::ModerationFailed => {
error(Status::ServiceUnavailable, "Image moderation unavailable. Please try again.")
}
})?;
let mut random = [0u8; 32];
OsRng.fill_bytes(&mut random);
let key = format!(
"uploads/{}.png",
random.iter().map(|byte| format!("{:02x}", byte)).collect::<String>()
);
let url = format!("{}/{}", service.public_url, key);
let response =
service.bucket.put_object_with_content_type(&key, &image, "image/png").await.map_err(
|err| {
warn!("Could not upload image to S3: {}", err);
error(Status::BadGateway, "Could not store image. Please try again.")
},
)?;
if !(200..300).contains(&response.status_code()) {
warn!("S3 image upload returned {}", response.status_code());
return Err(error(Status::BadGateway, "Could not store image. Please try again."));
}
if let Err(err) = sqlx::query("INSERT INTO image_uploads (uploader, path) VALUES (?, ?)")
.bind(uploader.0)
.bind(&key)
.execute(pool.inner())
.await
{
warn!("Could not record image upload: {}", err);
match service.bucket.delete_object(&key).await {
Ok(response) if (200..300).contains(&response.status_code()) => {}
_ => warn!("Could not clean up untracked image object {}", key),
}
return Err(error(
Status::InternalServerError,
"Could not record image upload. Please try again.",
));
}
Ok(Json(json!({ "url": url })))
}
#[cfg(test)]
mod tests {
use std::{io::Cursor, sync::Arc};
use rocket::{http::ContentType, local::asynchronous::Client};
use s3::{creds::Credentials, Bucket, Region};
use tokio::sync::Semaphore;
use super::*;
use crate::web::image_uploads::ImageUploadService;
fn service(unsafe_image: bool, endpoint: &str) -> ImageUploads {
let model: &[u8] = if unsafe_image {
include_bytes!("../../../../../tests/fixtures/image-upload-unsafe.onnx")
} else {
include_bytes!("../../../../../tests/fixtures/image-upload-safe.onnx")
};
let bucket = Bucket::new(
"images",
Region::Custom { region: "eu-west-2".into(), endpoint: endpoint.into() },
Credentials::new(Some("test"), Some("test"), None, None, None).unwrap(),
)
.unwrap()
.with_path_style();
ImageUploads(Some(Arc::new(ImageUploadService {
bucket,
public_url: "https://images.example.com".into(),
model: Some(nsfw::create_model(model).unwrap()),
slots: Arc::new(Semaphore::new(2)),
})))
}
fn image_bytes() -> Vec<u8> {
let mut bytes = Cursor::new(Vec::new());
image::DynamicImage::new_rgb8(2, 2).write_to(&mut bytes, image::ImageFormat::Png).unwrap();
bytes.into_inner()
}
async fn client(uploads: ImageUploads) -> Client {
let pool = sqlx::mysql::MySqlPoolOptions::new()
.connect_lazy("mysql://unused:unused@127.0.0.1:1/unused")
.unwrap();
Client::tracked(mount(rocket::build().manage(pool), uploads)).await.unwrap()
}
#[rocket::async_test]
async fn disabled_upload_route_is_not_available() {
let client = client(ImageUploads(None)).await;
assert_eq!(
client
.post("/dashboard/api/images")
.header(ContentType::Binary)
.body(image_bytes())
.dispatch()
.await
.status(),
Status::NotFound
);
}
#[rocket::async_test]
async fn upload_requires_an_authenticated_private_cookie() {
let client = client(service(false, "http://127.0.0.1:1")).await;
for cookie in [None, Some("not-an-id"), Some("0")] {
let mut request = client
.post("/dashboard/api/images")
.header(ContentType::Binary)
.body(image_bytes());
if let Some(cookie) = cookie {
request = request.private_cookie(("userid", cookie));
}
assert_eq!(request.dispatch().await.status(), Status::Unauthorized);
}
}
#[rocket::async_test]
async fn unsafe_images_return_451_without_contacting_storage_or_database() {
let client = client(service(true, "http://127.0.0.1:1")).await;
let response = client
.post("/dashboard/api/images")
.header(ContentType::Binary)
.private_cookie(("userid", "123456789012345678"))
.body(image_bytes())
.dispatch()
.await;
assert_eq!(response.status(), Status::UnavailableForLegalReasons);
assert!(response.into_json::<Value>().await.unwrap()["error"]
.as_str()
.unwrap()
.contains("NSFW"));
}
#[rocket::async_test]
async fn rejects_invalid_oversized_and_form_uploads() {
let client = client(service(false, "http://127.0.0.1:1")).await;
for (body, status) in [
(b"this is not an image".to_vec(), Status::UnprocessableEntity),
(vec![0; MAX_UPLOAD_BYTES as usize + 1], Status::PayloadTooLarge),
] {
let response = client
.post("/dashboard/api/images")
.header(ContentType::Binary)
.private_cookie(("userid", "123456789012345678"))
.body(body)
.dispatch()
.await;
assert_eq!(response.status(), status);
}
let response = client
.post("/dashboard/api/images")
.header(ContentType::Form)
.private_cookie(("userid", "123456789012345678"))
.body("file=anything")
.dispatch()
.await;
assert_eq!(response.status(), Status::NotFound);
}
#[sqlx::test]
async fn stores_moderated_image_and_tracks_uploader(pool: Pool<MySql>) {
use tokio::{
io::{AsyncReadExt, AsyncWriteExt},
net::TcpListener,
};
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let endpoint = format!("http://{}", listener.local_addr().unwrap());
let storage = tokio::spawn(async move {
let (mut socket, _) = listener.accept().await.unwrap();
let mut received = Vec::new();
let mut buffer = [0; 4096];
let (headers, body) = loop {
let count = socket.read(&mut buffer).await.unwrap();
assert!(count > 0);
received.extend_from_slice(&buffer[..count]);
if let Some(end) = received.windows(4).position(|window| window == b"\r\n\r\n") {
let headers = String::from_utf8(received[..end].to_vec()).unwrap();
let length: usize = headers
.lines()
.find_map(|line| {
line.to_lowercase()
.strip_prefix("content-length:")
.map(|length| length.trim().parse().unwrap())
})
.unwrap();
if received.len() >= end + 4 + length {
break (headers, received[end + 4..end + 4 + length].to_vec());
}
}
};
socket
.write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nConnection: close\r\n\r\n")
.await
.unwrap();
(headers, body)
});
let client =
Client::tracked(mount(rocket::build().manage(pool.clone()), service(false, &endpoint)))
.await
.unwrap();
let response = client
.post("/dashboard/api/images")
.header(ContentType::Binary)
.private_cookie(("userid", "123456789012345678"))
.body(image_bytes())
.dispatch()
.await;
assert_eq!(response.status(), Status::Ok);
let response = response.into_json::<Value>().await.unwrap();
let url = response["url"].as_str().unwrap();
let row: (u64, String) =
sqlx::query_as("SELECT uploader, path COLLATE utf8mb4_unicode_ci FROM image_uploads")
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(
row,
(
123456789012345678,
url.strip_prefix("https://images.example.com/").unwrap().to_string()
)
);
let (headers, body) = storage.await.unwrap();
assert!(headers.starts_with("PUT /images/uploads/"));
assert!(headers.to_lowercase().contains("content-type: image/png"));
assert_eq!(image::load_from_memory(&body).unwrap().width(), 2);
assert!(url.starts_with("https://images.example.com/uploads/"));
assert!(url.ends_with(".png"));
}
}
+1
View File
@@ -1,4 +1,5 @@
pub mod guild;
pub mod images;
pub mod user;
use log::warn;
+5
View File
@@ -23,6 +23,7 @@ use serenity::{
use sqlx::{MySql, Pool};
use crate::web::guards::transaction::Transaction;
use crate::web::image_uploads::ImageUploads;
#[derive(Serialize)]
struct UserPreferences {
@@ -36,6 +37,7 @@ struct UserPreferences {
struct UserInfo {
name: String,
patreon: bool,
image_uploads: bool,
preferences: UserPreferences,
}
@@ -52,10 +54,12 @@ pub async fn get_user_info(
cookies: &CookieJar<'_>,
ctx: &State<Context>,
pool: &State<Pool<MySql>>,
uploads: &State<ImageUploads>,
) -> JsonValue {
offline!(json!(UserInfo {
name: "Discord".to_string(),
patreon: true,
image_uploads: uploads.0.is_some(),
preferences: UserPreferences {
timezone: "UTC".to_string(),
use_browser_timezone: true,
@@ -101,6 +105,7 @@ pub async fn get_user_info(
);
let user_info = UserInfo {
image_uploads: uploads.0.is_some(),
name: cookies
.get_private("username")
.map_or("Discord User".to_string(), |c| c.value().to_string()),
+40 -9
View File
@@ -6,7 +6,9 @@ use serde::{Deserialize, Serialize};
use serenity::{client::Context, model::id::UserId};
use sqlx::types::Json;
use crate::image_upload::ImageReference;
use crate::utils::check_subscription;
use crate::web::routes::dashboard::image_reference_error;
use crate::web::{
consts::{
DAY, MAX_CONTENT_LENGTH, MAX_EMBED_AUTHOR_LENGTH, MAX_EMBED_DESCRIPTION_LENGTH,
@@ -28,12 +30,16 @@ pub struct Reminder {
pub attachment_name: Option<String>,
pub content: String,
pub embed_author: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
pub embed_author_url: Option<String>,
pub embed_color: u32,
pub embed_description: String,
pub embed_footer: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
pub embed_footer_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
pub embed_image_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
pub embed_thumbnail_url: Option<String>,
pub embed_title: String,
pub embed_fields: Option<Json<Vec<EmbedField>>>,
@@ -139,6 +145,23 @@ pub async fn create_reminder(
let name = if reminder.name.is_empty() { name_default() } else { reminder.name.clone() };
let new_uid = generate_uid();
let embed_image_url =
ImageReference::from_url(transaction.executor(), reminder.embed_image_url.as_deref())
.await
.map_err(image_reference_error)?;
let embed_thumbnail_url =
ImageReference::from_url(transaction.executor(), reminder.embed_thumbnail_url.as_deref())
.await
.map_err(image_reference_error)?;
let embed_author_url =
ImageReference::from_url(transaction.executor(), reminder.embed_author_url.as_deref())
.await
.map_err(image_reference_error)?;
let embed_footer_url =
ImageReference::from_url(transaction.executor(), reminder.embed_footer_url.as_deref())
.await
.map_err(image_reference_error)?;
// write to db
match sqlx::query!(
"INSERT INTO reminders (
@@ -149,12 +172,16 @@ pub async fn create_reminder(
content,
embed_author,
embed_author_url,
embed_author_id,
embed_color,
embed_description,
embed_footer,
embed_footer_url,
embed_footer_id,
embed_image_url,
embed_image_id,
embed_thumbnail_url,
embed_thumbnail_id,
embed_title,
embed_fields,
enabled,
@@ -165,20 +192,24 @@ pub async fn create_reminder(
name,
tts,
`utc_time`
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
new_uid,
reminder.attachment,
reminder.attachment_name,
channel,
reminder.content,
reminder.embed_author,
reminder.embed_author_url,
embed_author_url.url,
embed_author_url.id,
reminder.embed_color,
reminder.embed_description,
reminder.embed_footer,
reminder.embed_footer_url,
reminder.embed_image_url,
reminder.embed_thumbnail_url,
embed_footer_url.url,
embed_footer_url.id,
embed_image_url.url,
embed_image_url.id,
embed_thumbnail_url.url,
embed_thumbnail_url.id,
reminder.embed_title,
reminder.embed_fields,
reminder.enabled,
@@ -200,13 +231,13 @@ pub async fn create_reminder(
reminders.attachment_name,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
reminders.embed_fields,
reminders.enabled,
+19 -12
View File
@@ -71,13 +71,13 @@ pub async fn get_reminders(
reminders.attachment_name,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
IFNULL(reminders.embed_fields, '[]') AS embed_fields,
reminders.enabled,
@@ -146,11 +146,7 @@ pub async fn edit_reminder(
update_field!(transaction.executor(), error, reminder.[
attachment,
attachment_name,
embed_author_url,
embed_color,
embed_footer_url,
embed_image_url,
embed_thumbnail_url,
enabled,
expires,
name,
@@ -158,6 +154,17 @@ pub async fn edit_reminder(
utc_time
]);
update_image_field!(transaction.executor(), error, reminder, embed_image_url, embed_image_id);
update_image_field!(
transaction.executor(),
error,
reminder,
embed_thumbnail_url,
embed_thumbnail_id
);
update_image_field!(transaction.executor(), error, reminder, embed_author_url, embed_author_id);
update_image_field!(transaction.executor(), error, reminder, embed_footer_url, embed_footer_id);
if reminder.interval_days.flatten().is_some()
|| reminder.interval_months.flatten().is_some()
|| reminder.interval_seconds.flatten().is_some()
@@ -248,13 +255,13 @@ pub async fn edit_reminder(
reminders.attachment_name,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
reminders.embed_fields,
reminders.enabled,
@@ -37,16 +37,16 @@ pub async fn export(
name,
attachment,
attachment_name,
avatar,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.avatar_id), avatar) AS avatar,
content,
embed_author,
embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_author_id), embed_author_url) AS embed_author_url,
embed_color,
embed_description,
embed_footer,
embed_footer_url,
embed_image_url,
embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_footer_id), embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_image_id), embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminder_template.embed_thumbnail_id), embed_thumbnail_url) AS embed_thumbnail_url,
embed_title,
embed_fields,
interval_seconds,
+5 -5
View File
@@ -46,17 +46,17 @@ pub async fn export(
SELECT
reminders.attachment,
reminders.attachment_name,
reminders.avatar,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,
CONCAT('#', channels.channel) AS channel,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
CONVERT(CAST(embed_fields AS BINARY) USING utf8mb4) AS embed_fields,
reminders.enabled,
+176 -17
View File
@@ -20,6 +20,7 @@ use serenity::{
use sqlx::types::Json;
use sqlx::FromRow;
use crate::image_upload::ImageReference;
use crate::utils::check_subscription;
use crate::web::{
catchers::internal_server_error,
@@ -39,6 +40,11 @@ pub mod export;
type Unset<T> = Option<T>;
fn image_reference_error(error: sqlx::Error) -> serde_json::Value {
warn!("Could not resolve image upload: {:?}", error);
json!({"error": "Could not resolve image upload"})
}
fn name_default() -> String {
"Reminder".to_string()
}
@@ -92,15 +98,20 @@ pub struct ReminderTemplate {
name: String,
attachment: Option<Attachment>,
attachment_name: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
avatar: Option<String>,
content: String,
embed_author: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_author_url: Option<String>,
embed_color: u32,
embed_description: String,
embed_footer: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_footer_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_image_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_thumbnail_url: Option<String>,
embed_title: String,
embed_fields: Option<Json<Vec<EmbedField>>>,
@@ -117,15 +128,20 @@ pub struct ReminderTemplateCsv {
name: String,
attachment: Option<Attachment>,
attachment_name: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
avatar: Option<String>,
content: String,
embed_author: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_author_url: Option<String>,
embed_color: u32,
embed_description: String,
embed_footer: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_footer_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_image_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_thumbnail_url: Option<String>,
embed_title: String,
embed_fields: Option<String>,
@@ -182,17 +198,22 @@ pub struct CreateReminder {
#[derive(Serialize, FromRow)]
pub struct GetReminder {
attachment_name: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
avatar: Option<String>,
#[serde(with = "string")]
channel: u64,
content: String,
embed_author: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_author_url: Option<String>,
embed_color: u32,
embed_description: String,
embed_footer: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_footer_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_image_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_thumbnail_url: Option<String>,
embed_title: String,
embed_fields: Option<Json<Vec<EmbedField>>>,
@@ -215,16 +236,21 @@ pub struct GetReminder {
pub struct ReminderCsv {
attachment: Option<Attachment>,
attachment_name: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
avatar: Option<String>,
channel: String,
content: String,
embed_author: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_author_url: Option<String>,
embed_color: u32,
embed_description: String,
embed_footer: String,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_footer_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_image_url: Option<String>,
#[serde(serialize_with = "crate::image_upload::serialize_url")]
embed_thumbnail_url: Option<String>,
embed_title: String,
embed_fields: Option<String>,
@@ -491,6 +517,26 @@ pub(crate) async fn create_reminder(
let new_uid = generate_uid();
let avatar = ImageReference::from_url(transaction.executor(), reminder.avatar.as_deref())
.await
.map_err(image_reference_error)?;
let embed_image_url =
ImageReference::from_url(transaction.executor(), reminder.embed_image_url.as_deref())
.await
.map_err(image_reference_error)?;
let embed_thumbnail_url =
ImageReference::from_url(transaction.executor(), reminder.embed_thumbnail_url.as_deref())
.await
.map_err(image_reference_error)?;
let embed_author_url =
ImageReference::from_url(transaction.executor(), reminder.embed_author_url.as_deref())
.await
.map_err(image_reference_error)?;
let embed_footer_url =
ImageReference::from_url(transaction.executor(), reminder.embed_footer_url.as_deref())
.await
.map_err(image_reference_error)?;
// write to db
match sqlx::query!(
"INSERT INTO reminders (
@@ -499,15 +545,20 @@ pub(crate) async fn create_reminder(
attachment_name,
channel_id,
avatar,
avatar_id,
content,
embed_author,
embed_author_url,
embed_author_id,
embed_color,
embed_description,
embed_footer,
embed_footer_url,
embed_footer_id,
embed_image_url,
embed_image_id,
embed_thumbnail_url,
embed_thumbnail_id,
embed_title,
embed_fields,
enabled,
@@ -520,21 +571,26 @@ pub(crate) async fn create_reminder(
tts,
username,
`utc_time`
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
new_uid,
reminder.attachment,
reminder.attachment_name,
channel,
reminder.avatar,
avatar.url,
avatar.id,
reminder.content,
reminder.embed_author,
reminder.embed_author_url,
embed_author_url.url,
embed_author_url.id,
reminder.embed_color,
reminder.embed_description,
reminder.embed_footer,
reminder.embed_footer_url,
reminder.embed_image_url,
reminder.embed_thumbnail_url,
embed_footer_url.url,
embed_footer_url.id,
embed_image_url.url,
embed_image_url.id,
embed_thumbnail_url.url,
embed_thumbnail_url.id,
reminder.embed_title,
reminder.embed_fields,
reminder.enabled,
@@ -556,17 +612,17 @@ pub(crate) async fn create_reminder(
"
SELECT
reminders.attachment_name,
reminders.avatar,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.avatar_id), reminders.avatar) AS avatar,
channels.channel,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_author_id), reminders.embed_author_url) AS embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_footer_id), reminders.embed_footer_url) AS embed_footer_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_image_id), reminders.embed_image_url) AS embed_image_url,
COALESCE((SELECT path COLLATE utf8mb4_unicode_ci FROM image_uploads WHERE id = reminders.embed_thumbnail_id), reminders.embed_thumbnail_url) AS embed_thumbnail_url,
reminders.embed_title,
reminders.embed_fields,
reminders.enabled,
@@ -645,6 +701,35 @@ pub(crate) async fn create_reminder_template(
reminder_template.name.clone()
};
let avatar =
ImageReference::from_url(transaction.executor(), reminder_template.avatar.as_deref())
.await
.map_err(image_reference_error)?;
let embed_image_url = ImageReference::from_url(
transaction.executor(),
reminder_template.embed_image_url.as_deref(),
)
.await
.map_err(image_reference_error)?;
let embed_thumbnail_url = ImageReference::from_url(
transaction.executor(),
reminder_template.embed_thumbnail_url.as_deref(),
)
.await
.map_err(image_reference_error)?;
let embed_author_url = ImageReference::from_url(
transaction.executor(),
reminder_template.embed_author_url.as_deref(),
)
.await
.map_err(image_reference_error)?;
let embed_footer_url = ImageReference::from_url(
transaction.executor(),
reminder_template.embed_footer_url.as_deref(),
)
.await
.map_err(image_reference_error)?;
match sqlx::query!(
"INSERT INTO reminder_template
(guild_id,
@@ -652,15 +737,20 @@ pub(crate) async fn create_reminder_template(
attachment,
attachment_name,
avatar,
avatar_id,
content,
embed_author,
embed_author_url,
embed_author_id,
embed_color,
embed_description,
embed_footer,
embed_footer_url,
embed_footer_id,
embed_image_url,
embed_image_id,
embed_thumbnail_url,
embed_thumbnail_id,
embed_title,
embed_fields,
interval_seconds,
@@ -669,21 +759,26 @@ pub(crate) async fn create_reminder_template(
tts,
username
) VALUES ((SELECT id FROM guilds WHERE guild = ?), ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?, ?, ?, ?)",
?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
guild_id.get(),
name,
reminder_template.attachment,
reminder_template.attachment_name,
reminder_template.avatar,
avatar.url,
avatar.id,
reminder_template.content,
reminder_template.embed_author,
reminder_template.embed_author_url,
embed_author_url.url,
embed_author_url.id,
reminder_template.embed_color,
reminder_template.embed_description,
reminder_template.embed_footer,
reminder_template.embed_footer_url,
reminder_template.embed_image_url,
reminder_template.embed_thumbnail_url,
embed_footer_url.url,
embed_footer_url.id,
embed_image_url.url,
embed_image_url.id,
embed_thumbnail_url.url,
embed_thumbnail_url.id,
reminder_template.embed_title,
reminder_template.embed_fields,
reminder_template.interval_seconds,
@@ -869,3 +964,67 @@ async fn render_dashboard(cookies: &CookieJar<'_>) -> DashboardPage {
DashboardPage::Unauthorised(Redirect::to("/login/discord"))
}
}
#[cfg(test)]
mod tests {
use super::*;
use sqlx::{MySql, Pool};
#[sqlx::test]
async fn image_edits_distinguish_omission_replacement_and_removal(pool: Pool<MySql>) {
sqlx::query("INSERT INTO channels (id, channel) VALUES (1, 123)")
.execute(&pool)
.await
.unwrap();
let image = sqlx::query("INSERT INTO image_uploads (uploader, path) VALUES (123, ?)")
.bind(format!("uploads/{:064x}.png", 1))
.execute(&pool)
.await
.unwrap()
.last_insert_id();
sqlx::query("INSERT INTO reminders (uid, channel_id, `utc_time`, embed_image_id) VALUES ('image-edit', 1, UTC_TIMESTAMP(), ?)")
.bind(image).execute(&pool).await.unwrap();
let mut transaction = pool.begin().await.unwrap();
let mut errors = vec![];
let reminder: PatchReminder = serde_json::from_value(json!({"uid": "image-edit"})).unwrap();
update_image_field!(&mut *transaction, errors, reminder, embed_image_url, embed_image_id);
let stored: (Option<String>, Option<u64>) = sqlx::query_as(
"SELECT embed_image_url, embed_image_id FROM reminders WHERE uid = 'image-edit'",
)
.fetch_one(&mut *transaction)
.await
.unwrap();
assert_eq!(stored, (None, Some(image)));
let external = "https://external.example/new.png";
let reminder: PatchReminder = serde_json::from_value(json!({
"uid": "image-edit", "embed_image_url": external
}))
.unwrap();
update_image_field!(&mut *transaction, errors, reminder, embed_image_url, embed_image_id);
let stored: (Option<String>, Option<u64>) = sqlx::query_as(
"SELECT embed_image_url, embed_image_id FROM reminders WHERE uid = 'image-edit'",
)
.fetch_one(&mut *transaction)
.await
.unwrap();
assert_eq!(stored, (Some(external.into()), None));
sqlx::query("UPDATE reminders SET embed_image_url = NULL, embed_image_id = ? WHERE uid = 'image-edit'")
.bind(image).execute(&mut *transaction).await.unwrap();
let reminder: PatchReminder = serde_json::from_value(json!({
"uid": "image-edit", "embed_image_url": null
}))
.unwrap();
update_image_field!(&mut *transaction, errors, reminder, embed_image_url, embed_image_id);
let stored: (Option<String>, Option<u64>) = sqlx::query_as(
"SELECT embed_image_url, embed_image_id FROM reminders WHERE uid = 'image-edit'",
)
.fetch_one(&mut *transaction)
.await
.unwrap();
assert_eq!(stored, (None, None));
assert!(errors.is_empty());
transaction.commit().await.unwrap();
}
}
+11
View File
@@ -0,0 +1,11 @@
[Unit]
Description=Clean orphaned image uploads from Reminder Bot
Wants=network-online.target
After=network-online.target
[Service]
User=reminder
Type=oneshot
ExecStart=/usr/bin/reminder-rs clean-images
WorkingDirectory=/etc/reminder-rs
Environment="RUST_LOG=reminder_rs=info"
+10
View File
@@ -0,0 +1,10 @@
[Unit]
Description=Clean orphaned Reminder Bot image uploads hourly
[Timer]
OnCalendar=hourly
Persistent=true
Unit=reminder-rs-clean-images.service
[Install]
WantedBy=timers.target
+27
View File
@@ -0,0 +1,27 @@
The two tiny ONNX models are deterministic classifiers used by the image upload API tests.
They accept the same `[1, 224, 224, 3]` float input as the production nsfw model and return
`[Drawings, Hentai, Neutral, Porn, Sexy]` scores. One always returns Neutral = 1 and the
other always returns Porn = 1. They exercise the real nsfw inference path without requiring
image fixtures.
## Deterministic model fixtures
To regenerate them with Python's `onnx` package:
```python
from pathlib import Path
from onnx import TensorProto, checker, helper
for name, scores in [("safe", [0., 0., 1., 0., 0.]), ("unsafe", [0., 0., 0., 1., 0.])]:
tensor = helper.make_tensor("probabilities", TensorProto.FLOAT, [1, 5], scores)
graph = helper.make_graph(
[helper.make_node("Identity", ["probabilities"], ["output"])],
"constant classifier for upload endpoint tests",
[helper.make_tensor_value_info("input", TensorProto.FLOAT, [1, 224, 224, 3])],
[helper.make_tensor_value_info("output", TensorProto.FLOAT, [1, 5])],
[tensor],
)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)], ir_version=8)
checker.check_model(model)
Path(f"tests/fixtures/image-upload-{name}.onnx").write_bytes(model.SerializeToString())
```
Binary file not shown.
Binary file not shown.