Compare commits

...
Author SHA1 Message Date
jude 1ec2e8cf4a Cleanup 2026-10-14 19:47:30 +01:00
7 changed files with 14 additions and 45 deletions

No files matched your search

-10
View File
@@ -5,13 +5,3 @@
[GantMan/nsfw_model](https://github.com/GantMan/nsfw_model).
Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx
Size: 17,320,596 bytes (approximately 16.5 MiB).
SHA-256: `0dca163d933af88e03fa0f9d233655857e9cc9f38bfd2bc28ba57c486d70feaf`.
The application embeds these bytes using `include_bytes!` and initializes the model only
when dashboard S3 uploads are configured and `NSFW_CHECK_ENABLED` is enabled (the default).
Set `NSFW_CHECK_ENABLED=false` to skip model initialization and moderation.
Keep this file in the source checkout and build
context; it does not need to be installed separately alongside the compiled binary.
+3 -1
View File
@@ -4,5 +4,7 @@ CREATE TABLE image_uploads (
uploader BIGINT UNSIGNED NOT NULL,
url VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL UNIQUE,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
INDEX image_uploads_uploader (uploader)
orphaned_at DATETIME DEFAULT NULL,
INDEX image_uploads_uploader (uploader),
INDEX image_uploads_orphaned_at (orphaned_at)
);
@@ -1,4 +0,0 @@
ALTER TABLE image_uploads
-- UTC time at which cleanup first observed that this upload had no references.
ADD COLUMN orphaned_at DATETIME(6) NULL DEFAULT NULL,
ADD INDEX image_uploads_orphaned_at (orphaned_at);
+10 -10
View File
@@ -84,6 +84,7 @@ impl ImageUploadService {
{
return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into());
}
let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty());
let region = match &endpoint {
Some(endpoint) => {
@@ -91,11 +92,13 @@ impl ImageUploadService {
}
None => values[1].parse()?,
};
let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?;
let mut bucket = Bucket::new(&values[0], region, credentials)?;
if endpoint.is_some() {
bucket = bucket.with_path_style();
}
bucket.set_request_timeout(Some(std::time::Duration::from_secs(30)));
let model = if check_nsfw {
Some(nsfw::create_model(MODEL_BYTES)?)
@@ -156,17 +159,21 @@ fn decode_image(bytes: &[u8]) -> Result<image::RgbaImage, ImageError> {
) {
return Err(ImageError::InvalidImage);
}
let mut limits = Limits::default();
limits.max_image_width = Some(8192);
limits.max_image_height = Some(8192);
limits.max_alloc = Some(128 * 1024 * 1024);
reader.limits(limits);
let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?;
use image::ImageDecoder;
let (width, height) = decoder.dimensions();
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS {
return Err(ImageError::InvalidImage);
}
image::DynamicImage::from_decoder(decoder)
.map(|image| image.to_rgba8())
.map_err(|_| ImageError::InvalidImage)
@@ -176,6 +183,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
let image = decode_image(bytes)?;
// Flatten transparency so the classifier sees exactly the published image.
let mut visible = image::RgbImage::new(image.width(), image.height());
for (target, source) in visible.pixels_mut().zip(image.pixels()) {
for channel in 0..3 {
let alpha = u32::from(source[3]);
@@ -183,6 +191,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8;
}
}
let visible = image::DynamicImage::ImageRgb8(visible);
if let Some(model) = model {
let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| {
@@ -193,6 +202,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
return Err(ImageError::Nsfw);
}
}
let mut output = Cursor::new(Vec::new());
visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?;
Ok(output.into_inner())
@@ -233,16 +243,6 @@ mod tests {
assert!(ImageUploads::from_settings(&settings).0.is_none());
}
#[test]
fn nsfw_check_defaults_to_enabled_and_accepts_explicit_toggles() {
for value in [None, Some(""), Some("true"), Some("1"), Some("ON"), Some("invalid")] {
assert!(nsfw_check_enabled(value));
}
for value in [Some("false"), Some("0"), Some("OFF"), Some(" false ")] {
assert!(!nsfw_check_enabled(value));
}
}
#[test]
fn disabling_nsfw_checks_skips_model_initialization() {
let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"]
+1 -20
View File
@@ -19,31 +19,12 @@ python3 scripts/split_nsfw_test_image.py /path/to/input.jpg --output-dir tests/f
python3 scripts/split_nsfw_test_image.py /path/to/safe.jpg --output-dir tests/fixtures/non_nsfw
```
Each command automatically selects the first unused positive index, starting with
`image_1a.png` and `image_1b.png`. Indices with either part already present are skipped.
You can override the selection with `--index 2`; existing parts are never overwritten.
Each part contains random-looking
RGBA pixels. The original decoded pixels, including alpha, are recovered by XORing the two
parts. Only the first frame of animated input is used. Keep both files unchanged: resizing,
re-encoding with loss or flattening their alpha channels prevents correct reconstruction.
Both parts together reconstruct the original image, so splitting is not access control.
Run the splitter's round-trip and overwrite checks with
`python3 -m unittest discover -s scripts -p 'test_*.py'`.
The directories initially contain no images. The two dataset tests are explicitly ignored
until you provide fixtures. To run them after adding pairs to both directories:
To run the tests:
```sh
SQLX_OFFLINE=true cargo test --locked directory_images -- --ignored
```
The tests load all numbered pairs, recombine them in memory and pass the reconstructed PNG
through the production upload moderation function using the bundled model and its current
threshold. They also check that disabling moderation accepts the reconstructed images.
Empty directories, missing partners, mismatched dimensions, unexpected names and unexpected
moderation results fail the tests. Recombined images are never written to disk.
## Deterministic model fixtures
To regenerate them with Python's `onnx` package:
View File
Whitespace-only changes.
View File
Whitespace-only changes.