Import todo lists. Export other data.

This commit is contained in:
jude 2022-07-22 23:30:45 +01:00
parent f4213c6a83
commit e19af54caf
21 changed files with 899 additions and 353 deletions

621
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -23,7 +23,7 @@ rmp-serde = "0.15"
rand = "0.7"
levenshtein = "1.0"
sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "macros", "mysql", "bigdecimal", "chrono"]}
base64 = "0.13.0"
base64 = "0.13"
[dependencies.postman]
path = "postman"

View File

@ -7,12 +7,10 @@ edition = "2021"
tokio = { version = "1", features = ["process", "full"] }
regex = "1.4"
log = "0.4"
env_logger = "0.8"
chrono = "0.4"
chrono-tz = { version = "0.5", features = ["serde"] }
lazy_static = "1.4"
num-integer = "0.1"
serde = "1.0"
serde_json = "1.0"
sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "macros", "mysql", "bigdecimal", "chrono", "json"]}
serenity = { version = "0.11.1", default-features = false, features = ["builder", "cache", "client", "gateway", "http", "model", "utils", "rustls_backend"] }

View File

@ -226,7 +226,6 @@ impl Into<CreateEmbed> for Embed {
}
}
#[derive(Debug)]
pub struct Reminder {
id: u32,
@ -566,7 +565,7 @@ UPDATE `channels` SET paused = 0, paused_until = NULL WHERE `channel` = ?
};
if let Err(e) = result {
error!("Error sending {:?}: {:?}", self, e);
error!("Error sending reminder {}: {:?}", self.id, e);
if let Error::Http(error) = e {
if error.status_code() == Some(StatusCode::NOT_FOUND) {

View File

@ -12,10 +12,10 @@ oauth2 = "4"
log = "0.4"
reqwest = "0.11"
serde = { version = "1.0", features = ["derive"] }
serde_json = "1.0"
sqlx = { version = "0.5", features = ["runtime-tokio-rustls", "macros", "mysql", "chrono", "json"] }
chrono = "0.4"
chrono-tz = "0.5"
lazy_static = "1.4.0"
rand = "0.7"
base64 = "0.13"
csv = "1.1"

View File

@ -26,12 +26,8 @@ use serenity::model::prelude::AttachmentType;
lazy_static! {
pub static ref DEFAULT_AVATAR: AttachmentType<'static> = (
include_bytes!(concat!(
env!("CARGO_MANIFEST_DIR"),
"/../assets/",
env!("WEBHOOK_AVATAR", "WEBHOOK_AVATAR not provided for compilation")
)) as &[u8],
env!("WEBHOOK_AVATAR"),
include_bytes!(concat!(env!("CARGO_MANIFEST_DIR"), "/../assets/webhook.jpg")) as &[u8],
"webhook.jpg",
)
.into();
pub static ref SUBSCRIPTION_ROLES: HashSet<u64> = HashSet::from_iter(

View File

@ -150,6 +150,11 @@ pub async fn initialize(
routes::dashboard::guild::get_reminders,
routes::dashboard::guild::edit_reminder,
routes::dashboard::guild::delete_reminder,
routes::dashboard::export::export_reminders,
routes::dashboard::export::export_reminder_templates,
routes::dashboard::export::export_todos,
routes::dashboard::export::import_reminders,
routes::dashboard::export::import_todos,
],
)
.launch()

View File

@ -0,0 +1,375 @@
use csv::{QuoteStyle, WriterBuilder};
use rocket::{
http::CookieJar,
serde::json::{json, Json, Value as JsonValue},
State,
};
use serenity::{
client::Context,
model::id::{ChannelId, GuildId},
};
use sqlx::{MySql, Pool};
use crate::routes::dashboard::{ImportBody, ReminderCsv, ReminderTemplateCsv, TodoCsv};
#[get("/api/guild/<id>/export/reminders")]
pub async fn export_reminders(
id: u64,
cookies: &CookieJar<'_>,
ctx: &State<Context>,
pool: &State<Pool<MySql>>,
) -> JsonValue {
check_authorization!(cookies, ctx.inner(), id);
let mut csv_writer = WriterBuilder::new().quote_style(QuoteStyle::Always).from_writer(vec![]);
let channels_res = GuildId(id).channels(&ctx.inner()).await;
match channels_res {
Ok(channels) => {
let channels = channels
.keys()
.into_iter()
.map(|k| k.as_u64().to_string())
.collect::<Vec<String>>()
.join(",");
let result = sqlx::query_as_unchecked!(
ReminderCsv,
"SELECT
reminders.attachment,
reminders.attachment_name,
reminders.avatar,
channels.channel,
reminders.content,
reminders.embed_author,
reminders.embed_author_url,
reminders.embed_color,
reminders.embed_description,
reminders.embed_footer,
reminders.embed_footer_url,
reminders.embed_image_url,
reminders.embed_thumbnail_url,
reminders.embed_title,
reminders.embed_fields,
reminders.enabled,
reminders.expires,
reminders.interval_seconds,
reminders.interval_months,
reminders.name,
reminders.restartable,
reminders.tts,
reminders.username,
reminders.utc_time
FROM reminders
LEFT JOIN channels ON channels.id = reminders.channel_id
WHERE FIND_IN_SET(channels.channel, ?)",
channels
)
.fetch_all(pool.inner())
.await;
match result {
Ok(reminders) => {
reminders.iter().for_each(|reminder| {
csv_writer.serialize(reminder).unwrap();
});
match csv_writer.into_inner() {
Ok(inner) => match String::from_utf8(inner) {
Ok(encoded) => {
json!({ "body": encoded })
}
Err(e) => {
warn!("Failed to write UTF-8: {:?}", e);
json!({"error": "Failed to write UTF-8"})
}
},
Err(e) => {
warn!("Failed to extract CSV: {:?}", e);
json!({"error": "Failed to extract CSV"})
}
}
}
Err(e) => {
warn!("Failed to complete SQL query: {:?}", e);
json!({"error": "Failed to query reminders"})
}
}
}
Err(e) => {
warn!("Could not fetch channels from {}: {:?}", id, e);
json!({"error": "Failed to get guild channels"})
}
}
}
#[put("/api/guild/<id>/export/reminders", data = "<body>")]
pub async fn import_reminders(
id: u64,
cookies: &CookieJar<'_>,
body: Json<ImportBody>,
ctx: &State<Context>,
pool: &State<Pool<MySql>>,
) -> JsonValue {
check_authorization!(cookies, ctx.inner(), id);
match base64::decode(&body.body) {
Ok(body) => {
let mut reader = csv::Reader::from_reader(body.as_slice());
for result in reader.deserialize::<ReminderCsv>() {
match result {
Ok(record) => {}
Err(e) => {
warn!("Couldn't deserialize CSV row: {:?}", e);
}
}
}
json!({"error": "Not implemented"})
}
Err(_) => {
json!({"error": "Malformed base64"})
}
}
}
#[get("/api/guild/<id>/export/todos")]
pub async fn export_todos(
id: u64,
cookies: &CookieJar<'_>,
ctx: &State<Context>,
pool: &State<Pool<MySql>>,
) -> JsonValue {
check_authorization!(cookies, ctx.inner(), id);
let mut csv_writer = WriterBuilder::new().quote_style(QuoteStyle::Always).from_writer(vec![]);
match sqlx::query_as_unchecked!(
TodoCsv,
"SELECT value, CONCAT('#', channels.channel) AS channel_id FROM todos
LEFT JOIN channels ON todos.channel_id = channels.id
INNER JOIN guilds ON todos.guild_id = guilds.id
WHERE guilds.guild = ?",
id
)
.fetch_all(pool.inner())
.await
{
Ok(todos) => {
todos.iter().for_each(|todo| {
csv_writer.serialize(todo).unwrap();
});
match csv_writer.into_inner() {
Ok(inner) => match String::from_utf8(inner) {
Ok(encoded) => {
json!({ "body": encoded })
}
Err(e) => {
warn!("Failed to write UTF-8: {:?}", e);
json!({"error": "Failed to write UTF-8"})
}
},
Err(e) => {
warn!("Failed to extract CSV: {:?}", e);
json!({"error": "Failed to extract CSV"})
}
}
}
Err(e) => {
warn!("Could not fetch templates from {}: {:?}", id, e);
json!({"error": "Failed to query templates"})
}
}
}
#[put("/api/guild/<id>/export/todos", data = "<body>")]
pub async fn import_todos(
id: u64,
cookies: &CookieJar<'_>,
body: Json<ImportBody>,
ctx: &State<Context>,
pool: &State<Pool<MySql>>,
) -> JsonValue {
check_authorization!(cookies, ctx.inner(), id);
let channels_res = GuildId(id).channels(&ctx.inner()).await;
match channels_res {
Ok(channels) => match base64::decode(&body.body) {
Ok(body) => {
let mut reader = csv::Reader::from_reader(body.as_slice());
let query_placeholder = "(?, (SELECT id FROM channels WHERE channel = ?), (SELECT id FROM guilds WHERE guild = ?))";
let mut query_params = vec![];
for result in reader.deserialize::<TodoCsv>() {
match result {
Ok(record) => match record.channel_id {
Some(channel_id) => {
let channel_id = channel_id.split_at(1).1;
match channel_id.parse::<u64>() {
Ok(channel_id) => {
if channels.contains_key(&ChannelId(channel_id)) {
query_params.push((record.value, Some(channel_id), id));
} else {
return json!({
"error":
format!("Invalid channel ID {}", channel_id)
});
}
}
Err(_) => {
return json!({
"error": format!("Invalid channel ID {}", channel_id)
});
}
}
}
None => {
query_params.push((record.value, None, id));
}
},
Err(e) => {
warn!("Couldn't deserialize CSV row: {:?}", e);
return json!({"error": "Deserialize error. Aborted"});
}
}
}
let _ = sqlx::query!(
"DELETE FROM todos WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
id
)
.execute(pool.inner())
.await;
let query_str = format!(
"INSERT INTO todos (value, channel_id, guild_id) VALUES {}",
vec![query_placeholder].repeat(query_params.len()).join(",")
);
let mut query = sqlx::query(&query_str);
for param in query_params {
query = query.bind(param.0).bind(param.1).bind(param.2);
}
let res = query.execute(pool.inner()).await;
match res {
Ok(_) => {
json!({})
}
Err(e) => {
warn!("Couldn't execute todo query: {:?}", e);
json!({"error": "An unexpected error occured."})
}
}
}
Err(_) => {
json!({"error": "Malformed base64"})
}
},
Err(e) => {
warn!("Couldn't fetch channels for guild {}: {:?}", id, e);
json!({"error": "Couldn't fetch channels."})
}
}
}
#[get("/api/guild/<id>/export/reminder_templates")]
pub async fn export_reminder_templates(
id: u64,
cookies: &CookieJar<'_>,
ctx: &State<Context>,
pool: &State<Pool<MySql>>,
) -> JsonValue {
check_authorization!(cookies, ctx.inner(), id);
let mut csv_writer = WriterBuilder::new().quote_style(QuoteStyle::Always).from_writer(vec![]);
match sqlx::query_as_unchecked!(
ReminderTemplateCsv,
"SELECT
name,
attachment,
attachment_name,
avatar,
content,
embed_author,
embed_author_url,
embed_color,
embed_description,
embed_footer,
embed_footer_url,
embed_image_url,
embed_thumbnail_url,
embed_title,
embed_fields,
tts,
username
FROM reminder_template WHERE guild_id = (SELECT id FROM guilds WHERE guild = ?)",
id
)
.fetch_all(pool.inner())
.await
{
Ok(templates) => {
templates.iter().for_each(|template| {
csv_writer.serialize(template).unwrap();
});
match csv_writer.into_inner() {
Ok(inner) => match String::from_utf8(inner) {
Ok(encoded) => {
json!({ "body": encoded })
}
Err(e) => {
warn!("Failed to write UTF-8: {:?}", e);
json!({"error": "Failed to write UTF-8"})
}
},
Err(e) => {
warn!("Failed to extract CSV: {:?}", e);
json!({"error": "Failed to extract CSV"})
}
}
}
Err(e) => {
warn!("Could not fetch templates from {}: {:?}", id, e);
json!({"error": "Failed to query templates"})
}
}
}

View File

@ -13,6 +13,7 @@ use crate::{
Database, Error,
};
pub mod export;
pub mod guild;
pub mod user;
@ -60,6 +61,28 @@ pub struct ReminderTemplate {
username: Option<String>,
}
#[derive(Serialize, Deserialize)]
pub struct ReminderTemplateCsv {
#[serde(default = "template_name_default")]
name: String,
attachment: Option<Vec<u8>>,
attachment_name: Option<String>,
avatar: Option<String>,
content: String,
embed_author: String,
embed_author_url: Option<String>,
embed_color: u32,
embed_description: String,
embed_footer: String,
embed_footer_url: Option<String>,
embed_image_url: Option<String>,
embed_thumbnail_url: Option<String>,
embed_title: String,
embed_fields: Option<String>,
tts: bool,
username: Option<String>,
}
#[derive(Deserialize)]
pub struct DeleteReminderTemplate {
id: u32,
@ -105,6 +128,36 @@ pub struct Reminder {
utc_time: NaiveDateTime,
}
#[derive(Serialize, Deserialize)]
pub struct ReminderCsv {
#[serde(with = "base64s")]
attachment: Option<Vec<u8>>,
attachment_name: Option<String>,
avatar: Option<String>,
channel: u64,
content: String,
embed_author: String,
embed_author_url: Option<String>,
embed_color: u32,
embed_description: String,
embed_footer: String,
embed_footer_url: Option<String>,
embed_image_url: Option<String>,
embed_thumbnail_url: Option<String>,
embed_title: String,
embed_fields: Option<String>,
enabled: bool,
expires: Option<NaiveDateTime>,
interval_seconds: Option<u32>,
interval_months: Option<u32>,
#[serde(default = "name_default")]
name: String,
restartable: bool,
tts: bool,
username: Option<String>,
utc_time: NaiveDateTime,
}
#[derive(Deserialize)]
pub struct PatchReminder {
uid: String,
@ -220,13 +273,22 @@ pub struct DeleteReminder {
uid: String,
}
#[derive(Deserialize)]
pub struct ImportBody {
body: String,
}
#[derive(Serialize, Deserialize)]
pub struct TodoCsv {
value: String,
channel_id: Option<String>,
}
async fn create_database_channel(
ctx: impl AsRef<Http>,
channel: ChannelId,
pool: impl Executor<'_, Database = Database> + Copy,
) -> Result<u32, crate::Error> {
println!("{:?}", channel);
let row =
sqlx::query!("SELECT webhook_token, webhook_id FROM channels WHERE channel = ?", channel.0)
.fetch_one(pool)

View File

@ -25,7 +25,6 @@ pub async fn discord_login(
// Set the desired scopes.
.add_scope(Scope::new("identify".to_string()))
.add_scope(Scope::new("guilds".to_string()))
.add_scope(Scope::new("email".to_string()))
// Set the PKCE code challenge.
.set_pkce_challenge(pkce_challenge)
.url();

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 18 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 20 KiB

View File

@ -12,6 +12,10 @@ const $createTemplateBtn = $createReminder.querySelector("button#createTemplate"
const $loadTemplateBtn = document.querySelector("button#load-template");
const $deleteTemplateBtn = document.querySelector("button#delete-template");
const $templateSelect = document.querySelector("select#templateSelect");
const $exportBtn = document.querySelector("button#export-data");
const $importBtn = document.querySelector("button#import-data");
const $downloader = document.querySelector("a#downloader");
const $uploader = document.querySelector("input#uploader");
let channels = [];
let guildNames = {};
@ -670,6 +674,39 @@ function has_source(string) {
}
}
$uploader.addEventListener("change", (ev) => {
const urlTail = document.querySelector('input[name="exportSelect"]:checked').value;
new Promise((resolve) => {
let fileReader = new FileReader();
fileReader.onload = (e) => resolve(fileReader.result);
fileReader.readAsDataURL($uploader.files[0]);
}).then((dataUrl) => {
fetch(`/dashboard/api/guild/${guildId()}/export/${urlTail}`, {
method: "PUT",
body: JSON.stringify({ body: dataUrl.split(",")[1] }),
}).then(() => {
delete $uploader.files[0];
});
});
});
$importBtn.addEventListener("click", () => {
$uploader.click();
});
$exportBtn.addEventListener("click", () => {
const urlTail = document.querySelector('input[name="exportSelect"]:checked').value;
fetch(`/dashboard/api/guild/${guildId()}/export/${urlTail}`)
.then((response) => response.json())
.then((data) => {
$downloader.href =
"data:text/plain;charset=utf-8," + encodeURIComponent(data.body);
$downloader.click();
});
});
$createReminderBtn.addEventListener("click", async () => {
$createReminderBtn.querySelector("span.icon > i").classList = [
"fas fa-spinner fa-spin",
@ -834,7 +871,7 @@ document.addEventListener("remindersLoaded", () => {
});
});
const fileInput = document.querySelectorAll("input[type=file]");
const fileInput = document.querySelectorAll("input.file-input[type=file]");
fileInput.forEach((element) => {
element.addEventListener("change", () => {

View File

@ -177,38 +177,41 @@
<section class="modal-card-body">
<div class="control">
<div class="field">
<input type="checkbox" class="default-width">
<label>Reminders</label>
<label>
<input type="radio" class="default-width" name="exportSelect" value="reminders" checked>
Reminders
</label>
</div>
</div>
<div class="control">
<div class="field">
<input type="checkbox" class="default-width">
<label>Todo Lists</label>
<label>
<input type="radio" class="default-width" name="exportSelect" value="todos">
Todo Lists
</label>
</div>
</div>
<div class="control">
<div class="field">
<input type="checkbox" class="default-width">
<label>Timers</label>
</div>
</div>
<div class="control">
<div class="field">
<input type="checkbox" class="default-width">
<label>Reminder templates</label>
</div>
</div>
<div class="control">
<div class="field">
<input type="checkbox" class="default-width">
<label>Macros</label>
<label>
<input type="radio" class="default-width" name="exportSelect" value="reminder_templates">
Reminder templates
</label>
</div>
</div>
<br>
<div class="has-text-centered">
<div style="color: red; font-weight: bold;">
By selecting "Import", you understand that this will overwrite existing data.
</div>
<div style="color: red">
Please first read the <a href="/help/iemanager">support page</a>
</div>
<button class="button is-success is-outlined" id="import-data">Import Data</button>
<button class="button is-success" id="export-data">Export Data</button>
</div>
<a id="downloader" download="export.csv" class="is-hidden"></a>
<input id="uploader" type="file" hidden></input>
</section>
</div>
<button class="modal-close is-large close-modal" aria-label="close"></button>

View File

@ -5,5 +5,5 @@
{% set show_contact = True %}
{% set page_title = "An Error Has Occurred" %}
{% set page_subtitle = "A server error has occurred. Please contact me and I will try and resolve this" %}
{% set page_subtitle = "A server error has occurred. Please retry, or ask in our Discord." %}
{% endblock %}

View File

@ -49,7 +49,7 @@
<div class="container">
<h2 class="title">Who your data is shared with</h2>
<p class="is-size-5 pl-6">
Your data may also be guarded by the privacy policies of <strong>MEGA</strong>, our backup provider, and
Your data is also guarded by the privacy policies of <strong>MEGA</strong>, our backup provider, and
<strong>Hetzner</strong>, our hosting provider.
</p>
</div>
@ -68,7 +68,7 @@
<br>
<br>
Reminders deleted with <strong>/del</strong> or via the dashboard are removed from the live database
instantly, but may persist in backups.
instantly, but may persist in backups for up to a year.
</p>
</div>
</section>

View File

@ -14,13 +14,75 @@
<div class="container">
<p class="title">Export your data</p>
<p class="content">
You can create reminders with the <code>/remind</code> command.
<br>
Fill out the "time" and "content" fields. If you wish, press on "Optional" to view other options
for the reminder.
You can export data associated with your server from the dashboard. The data will export as a CSV
file. The CSV file can then be edited and imported to bulk edit server data.
</p>
</div>
</div>
</section>
<section class="hero is-small">
<div class="hero-body">
<div class="container">
<p class="title">Import data</p>
<p class="content">
You can import previous exports or modified exports. When importing a file, <strong>existing data
will be overwritten</strong>.
</p>
</div>
</div>
</section>
<section class="hero is-small">
<div class="hero-body">
<div class="container content">
<p class="title">Edit your data</p>
<p>
The CSV can be edited either as a text file or in a spreadsheet editor such as LibreOffice Calc. To
set up LibreOffice Calc for editing, do the following:
</p>
<ol>
<li>
Export data from dashboard.
<figure>
<img src="/static/img/support/iemanager/select_export.png" alt="Selecting export button">
</figure>
</li>
<li>
Open the file in LibreOffice. <strong>During the import dialogue, select "Format quoted field as text".</strong>
<figure>
<img src="/static/img/support/iemanager/format_text.png" alt="Selecting format button">
</figure>
</li>
<li>
Make edits to the spreadsheet. You can add, edit, and remove rows for reminders. Don't remove the title row.
<figure>
<img src="/static/img/support/iemanager/edit_spreadsheet.png" alt="Editing spreadsheet">
</figure>
</li>
<li>
Save the edited CSV file and import it on the dashboard.
<figure>
<img src="/static/img/support/iemanager/import.png" alt="Import new reminders">
</figure>
</li>
</ol>
Other spreadsheet tools can also be used to edit exports, as long as they are properly configured:
<ul>
<li>
<strong>Google Sheets</strong>: Create a new blank spreadsheet. Select <strong>File >> Import >> Upload >> export.csv</strong>.
Use the following import settings:
<figure>
<img src="/static/img/support/iemanager/sheets_settings.png" alt="Google sheets import settings">
</figure>
</li>
<li>
<strong>Excel (including Excel Online)</strong>: Avoid using Excel. Excel will not correctly import channels, or give
clear options to correct imports.
</li>
</ul>
</div>
</div>
</section>
{% endblock %}

View File

@ -20,11 +20,12 @@
<br>
Violating the Terms of Service may result in receiving a permanent ban from the Discord server,
permanent restriction on your usage of Reminder Bot, or removal of some or all of your content on
Reminder Bot or the Discord server.
Reminder Bot or the Discord server. None of these will necessarily be preceded or succeeded by a warning
or notice.
<br>
<br>
The Terms of Service may be updated at any time. Notice will be provided via the Discord server. You
should consider the Terms of Service to be a guideline for appropriate behaviour.
The Terms of Service may be updated. Notice will be provided via the Discord server. You
should consider the Terms of Service to be a strong for appropriate behaviour.
</p>
</div>
</section>
@ -37,6 +38,12 @@
<li>Do not use the bot to harass other Discord users</li>
<li>Do not use the bot to transmit malware or other illegal content</li>
<li>Do not use the bot to send more than 15 messages during a 60 second period</li>
<li>
Do not attempt to circumvent restrictions imposed by the bot or website, including trying to access
data of other users, circumvent Patreon restrictions, or uploading files and creating reminders that
are too large for the bot to send or process. Some or all of these actions may be illegal in your
country
</li>
</ul>
</div>
</section>