This commit is contained in:
jude committed 2026-10-14 20:16:39 +01:00
1 parent c3bb015d2f
commit e1e49904a8
8 files changed
+32 -68

No files matched your search

-10
View File
@@ -5,13 +5,3 @@
[GantMan/nsfw_model](https://github.com/GantMan/nsfw_model). [GantMan/nsfw_model](https://github.com/GantMan/nsfw_model).
Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx
Size: 17,320,596 bytes (approximately 16.5 MiB).
SHA-256: `0dca163d933af88e03fa0f9d233655857e9cc9f38bfd2bc28ba57c486d70feaf`.
The application embeds these bytes using `include_bytes!` and initializes the model only
when dashboard S3 uploads are configured and `NSFW_CHECK_ENABLED` is enabled (the default).
Set `NSFW_CHECK_ENABLED=false` to skip model initialization and moderation.
Keep this file in the source checkout and build
context; it does not need to be installed separately alongside the compiled binary.
+3 -1
View File
@@ -4,5 +4,7 @@ CREATE TABLE image_uploads (
uploader BIGINT UNSIGNED NOT NULL, uploader BIGINT UNSIGNED NOT NULL,
url VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL UNIQUE, url VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL UNIQUE,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX image_uploads_uploader (uploader) orphaned_at DATETIME DEFAULT NULL,
INDEX image_uploads_uploader (uploader),
INDEX image_uploads_orphaned_at (orphaned_at)
); );
@@ -1,4 +0,0 @@
ALTER TABLE image_uploads
-- UTC time at which cleanup first observed that this upload had no references.
ADD COLUMN orphaned_at DATETIME(6) NULL DEFAULT NULL,
ADD INDEX image_uploads_orphaned_at (orphaned_at);
@@ -88,10 +88,11 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
const uploadInProgress = useRef(false); const uploadInProgress = useRef(false);
const upload = async (files: File[]) => { const upload = async (files: File[]) => {
setBlocked(false);
if (!uploadsEnabled || uploadInProgress.current) return; if (!uploadsEnabled || uploadInProgress.current) return;
setError(""); setError("");
if (files.length !== 1) { if (files.length !== 1) {
setError("Please upload one image at a time."); setError("Please upload only one image.");
return; return;
} }
const file = files[0]; const file = files[0];
@@ -128,27 +129,6 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
if (droppedFiles) void upload(droppedFiles); if (droppedFiles) void upload(droppedFiles);
}, [droppedFiles]); }, [droppedFiles]);
if (blocked) {
return (
<Modal
setModalOpen={() => setBlocked(false)}
title="Image upload blocked"
onSubmit={() => setBlocked(false)}
onSubmitText="OK"
>
<p>
This image was flagged as potentially NSFW. Unfortunately, due to the UK's
Online Safety Act, we cannot handle explicit content. Please defer to using
Imgur or another image storage platform who have the resources to deal with
idiotic laws.
</p>
<p>
The image you attempted to upload <strong>has not been stored.</strong>
</p>
</Modal>
);
}
return ( return (
<Modal <Modal
setModalOpen={setModalOpen} setModalOpen={setModalOpen}
@@ -180,6 +160,7 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
{...handlers} {...handlers}
class={`mt-4 image-upload-dropzone ${dragging ? "image-picker-dragging" : ""}`} class={`mt-4 image-upload-dropzone ${dragging ? "image-picker-dragging" : ""}`}
> >
<p>or…</p>
<input <input
ref={fileInput} ref={fileInput}
type="file" type="file"
@@ -203,8 +184,22 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
</button> </button>
<p class="help">Or drag and drop an image here.</p> <p class="help">Or drag and drop an image here.</p>
<p class="help"> <p class="help">
PNG, JPEG, GIF or WebP, up to 8 MiB. All uploads are converted to PNGs. Static PNG, JPEG, GIF or WebP, up to 8 MiB. All uploads are converted to
PNGs.
</p> </p>
{blocked && (
<>
<p class="help" style="color: red;">
This image was flagged as potentially NSFW. Unfortunately, due to
the UK's Online Safety Act, we cannot handle explicit content.
Please defer to using Imgur or another image storage platform who
have the resources to deal with idiotic laws.
</p>
<p class="help" style="color: red;">
The image you attempted to upload has not been stored.
</p>
</>
)}
{uploading && <p role="status">Uploading image…</p>} {uploading && <p role="status">Uploading image…</p>}
</div> </div>
)} )}
+10 -10
View File
@@ -84,6 +84,7 @@ impl ImageUploadService {
{ {
return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into()); return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into());
} }
let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty()); let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty());
let region = match &endpoint { let region = match &endpoint {
Some(endpoint) => { Some(endpoint) => {
@@ -91,11 +92,13 @@ impl ImageUploadService {
} }
None => values[1].parse()?, None => values[1].parse()?,
}; };
let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?; let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?;
let mut bucket = Bucket::new(&values[0], region, credentials)?; let mut bucket = Bucket::new(&values[0], region, credentials)?;
if endpoint.is_some() { if endpoint.is_some() {
bucket = bucket.with_path_style(); bucket = bucket.with_path_style();
} }
bucket.set_request_timeout(Some(std::time::Duration::from_secs(30))); bucket.set_request_timeout(Some(std::time::Duration::from_secs(30)));
let model = if check_nsfw { let model = if check_nsfw {
Some(nsfw::create_model(MODEL_BYTES)?) Some(nsfw::create_model(MODEL_BYTES)?)
@@ -156,17 +159,21 @@ fn decode_image(bytes: &[u8]) -> Result<image::RgbaImage, ImageError> {
) { ) {
return Err(ImageError::InvalidImage); return Err(ImageError::InvalidImage);
} }
let mut limits = Limits::default(); let mut limits = Limits::default();
limits.max_image_width = Some(8192); limits.max_image_width = Some(8192);
limits.max_image_height = Some(8192); limits.max_image_height = Some(8192);
limits.max_alloc = Some(128 * 1024 * 1024); limits.max_alloc = Some(128 * 1024 * 1024);
reader.limits(limits); reader.limits(limits);
let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?; let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?;
use image::ImageDecoder; use image::ImageDecoder;
let (width, height) = decoder.dimensions(); let (width, height) = decoder.dimensions();
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS { if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS {
return Err(ImageError::InvalidImage); return Err(ImageError::InvalidImage);
} }
image::DynamicImage::from_decoder(decoder) image::DynamicImage::from_decoder(decoder)
.map(|image| image.to_rgba8()) .map(|image| image.to_rgba8())
.map_err(|_| ImageError::InvalidImage) .map_err(|_| ImageError::InvalidImage)
@@ -176,6 +183,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
let image = decode_image(bytes)?; let image = decode_image(bytes)?;
// Flatten transparency so the classifier sees exactly the published image. // Flatten transparency so the classifier sees exactly the published image.
let mut visible = image::RgbImage::new(image.width(), image.height()); let mut visible = image::RgbImage::new(image.width(), image.height());
for (target, source) in visible.pixels_mut().zip(image.pixels()) { for (target, source) in visible.pixels_mut().zip(image.pixels()) {
for channel in 0..3 { for channel in 0..3 {
let alpha = u32::from(source[3]); let alpha = u32::from(source[3]);
@@ -183,6 +191,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8; ((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8;
} }
} }
let visible = image::DynamicImage::ImageRgb8(visible); let visible = image::DynamicImage::ImageRgb8(visible);
if let Some(model) = model { if let Some(model) = model {
let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| { let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| {
@@ -193,6 +202,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
return Err(ImageError::Nsfw); return Err(ImageError::Nsfw);
} }
} }
let mut output = Cursor::new(Vec::new()); let mut output = Cursor::new(Vec::new());
visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?; visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?;
Ok(output.into_inner()) Ok(output.into_inner())
@@ -233,16 +243,6 @@ mod tests {
assert!(ImageUploads::from_settings(&settings).0.is_none()); assert!(ImageUploads::from_settings(&settings).0.is_none());
} }
#[test]
fn nsfw_check_defaults_to_enabled_and_accepts_explicit_toggles() {
for value in [None, Some(""), Some("true"), Some("1"), Some("ON"), Some("invalid")] {
assert!(nsfw_check_enabled(value));
}
for value in [Some("false"), Some("0"), Some("OFF"), Some(" false ")] {
assert!(!nsfw_check_enabled(value));
}
}
#[test] #[test]
fn disabling_nsfw_checks_skips_model_initialization() { fn disabling_nsfw_checks_skips_model_initialization() {
let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"] let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"]
+1 -20
View File
@@ -19,31 +19,12 @@ python3 scripts/split_nsfw_test_image.py /path/to/input.jpg --output-dir tests/f
python3 scripts/split_nsfw_test_image.py /path/to/safe.jpg --output-dir tests/fixtures/non_nsfw python3 scripts/split_nsfw_test_image.py /path/to/safe.jpg --output-dir tests/fixtures/non_nsfw
``` ```
Each command automatically selects the first unused positive index, starting with To run the tests:
`image_1a.png` and `image_1b.png`. Indices with either part already present are skipped.
You can override the selection with `--index 2`; existing parts are never overwritten.
Each part contains random-looking
RGBA pixels. The original decoded pixels, including alpha, are recovered by XORing the two
parts. Only the first frame of animated input is used. Keep both files unchanged: resizing,
re-encoding with loss or flattening their alpha channels prevents correct reconstruction.
Both parts together reconstruct the original image, so splitting is not access control.
Run the splitter's round-trip and overwrite checks with
`python3 -m unittest discover -s scripts -p 'test_*.py'`.
The directories initially contain no images. The two dataset tests are explicitly ignored
until you provide fixtures. To run them after adding pairs to both directories:
```sh ```sh
SQLX_OFFLINE=true cargo test --locked directory_images -- --ignored SQLX_OFFLINE=true cargo test --locked directory_images -- --ignored
``` ```
The tests load all numbered pairs, recombine them in memory and pass the reconstructed PNG
through the production upload moderation function using the bundled model and its current
threshold. They also check that disabling moderation accepts the reconstructed images.
Empty directories, missing partners, mismatched dimensions, unexpected names and unexpected
moderation results fail the tests. Recombined images are never written to disk.
## Deterministic model fixtures ## Deterministic model fixtures
To regenerate them with Python's `onnx` package: To regenerate them with Python's `onnx` package:
View File
Whitespace-only changes.
View File
Whitespace-only changes.