Compare commits
1
Commits
e1e49904a8
...
1ec2e8cf4a
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1ec2e8cf4a |
No files matched your search
@@ -5,13 +5,3 @@
|
||||
[GantMan/nsfw_model](https://github.com/GantMan/nsfw_model).
|
||||
|
||||
Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx
|
||||
|
||||
Size: 17,320,596 bytes (approximately 16.5 MiB).
|
||||
|
||||
SHA-256: `0dca163d933af88e03fa0f9d233655857e9cc9f38bfd2bc28ba57c486d70feaf`.
|
||||
|
||||
The application embeds these bytes using `include_bytes!` and initializes the model only
|
||||
when dashboard S3 uploads are configured and `NSFW_CHECK_ENABLED` is enabled (the default).
|
||||
Set `NSFW_CHECK_ENABLED=false` to skip model initialization and moderation.
|
||||
Keep this file in the source checkout and build
|
||||
context; it does not need to be installed separately alongside the compiled binary.
|
||||
@@ -4,5 +4,7 @@ CREATE TABLE image_uploads (
|
||||
uploader BIGINT UNSIGNED NOT NULL,
|
||||
url VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL UNIQUE,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
INDEX image_uploads_uploader (uploader)
|
||||
orphaned_at DATETIME DEFAULT NULL,
|
||||
INDEX image_uploads_uploader (uploader),
|
||||
INDEX image_uploads_orphaned_at (orphaned_at)
|
||||
);
|
||||
@@ -1,4 +0,0 @@
|
||||
ALTER TABLE image_uploads
|
||||
-- UTC time at which cleanup first observed that this upload had no references.
|
||||
ADD COLUMN orphaned_at DATETIME(6) NULL DEFAULT NULL,
|
||||
ADD INDEX image_uploads_orphaned_at (orphaned_at);
|
||||
+10
-10
@@ -84,6 +84,7 @@ impl ImageUploadService {
|
||||
{
|
||||
return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into());
|
||||
}
|
||||
|
||||
let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty());
|
||||
let region = match &endpoint {
|
||||
Some(endpoint) => {
|
||||
@@ -91,11 +92,13 @@ impl ImageUploadService {
|
||||
}
|
||||
None => values[1].parse()?,
|
||||
};
|
||||
|
||||
let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?;
|
||||
let mut bucket = Bucket::new(&values[0], region, credentials)?;
|
||||
if endpoint.is_some() {
|
||||
bucket = bucket.with_path_style();
|
||||
}
|
||||
|
||||
bucket.set_request_timeout(Some(std::time::Duration::from_secs(30)));
|
||||
let model = if check_nsfw {
|
||||
Some(nsfw::create_model(MODEL_BYTES)?)
|
||||
@@ -156,17 +159,21 @@ fn decode_image(bytes: &[u8]) -> Result<image::RgbaImage, ImageError> {
|
||||
) {
|
||||
return Err(ImageError::InvalidImage);
|
||||
}
|
||||
|
||||
let mut limits = Limits::default();
|
||||
limits.max_image_width = Some(8192);
|
||||
limits.max_image_height = Some(8192);
|
||||
limits.max_alloc = Some(128 * 1024 * 1024);
|
||||
reader.limits(limits);
|
||||
|
||||
let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?;
|
||||
|
||||
use image::ImageDecoder;
|
||||
let (width, height) = decoder.dimensions();
|
||||
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS {
|
||||
return Err(ImageError::InvalidImage);
|
||||
}
|
||||
|
||||
image::DynamicImage::from_decoder(decoder)
|
||||
.map(|image| image.to_rgba8())
|
||||
.map_err(|_| ImageError::InvalidImage)
|
||||
@@ -176,6 +183,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
|
||||
let image = decode_image(bytes)?;
|
||||
// Flatten transparency so the classifier sees exactly the published image.
|
||||
let mut visible = image::RgbImage::new(image.width(), image.height());
|
||||
|
||||
for (target, source) in visible.pixels_mut().zip(image.pixels()) {
|
||||
for channel in 0..3 {
|
||||
let alpha = u32::from(source[3]);
|
||||
@@ -183,6 +191,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
|
||||
((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8;
|
||||
}
|
||||
}
|
||||
|
||||
let visible = image::DynamicImage::ImageRgb8(visible);
|
||||
if let Some(model) = model {
|
||||
let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| {
|
||||
@@ -193,6 +202,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
|
||||
return Err(ImageError::Nsfw);
|
||||
}
|
||||
}
|
||||
|
||||
let mut output = Cursor::new(Vec::new());
|
||||
visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?;
|
||||
Ok(output.into_inner())
|
||||
@@ -233,16 +243,6 @@ mod tests {
|
||||
assert!(ImageUploads::from_settings(&settings).0.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nsfw_check_defaults_to_enabled_and_accepts_explicit_toggles() {
|
||||
for value in [None, Some(""), Some("true"), Some("1"), Some("ON"), Some("invalid")] {
|
||||
assert!(nsfw_check_enabled(value));
|
||||
}
|
||||
for value in [Some("false"), Some("0"), Some("OFF"), Some(" false ")] {
|
||||
assert!(!nsfw_check_enabled(value));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn disabling_nsfw_checks_skips_model_initialization() {
|
||||
let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"]
|
||||
|
||||
Vendored
+1
-20
@@ -19,31 +19,12 @@ python3 scripts/split_nsfw_test_image.py /path/to/input.jpg --output-dir tests/f
|
||||
python3 scripts/split_nsfw_test_image.py /path/to/safe.jpg --output-dir tests/fixtures/non_nsfw
|
||||
```
|
||||
|
||||
Each command automatically selects the first unused positive index, starting with
|
||||
`image_1a.png` and `image_1b.png`. Indices with either part already present are skipped.
|
||||
You can override the selection with `--index 2`; existing parts are never overwritten.
|
||||
Each part contains random-looking
|
||||
RGBA pixels. The original decoded pixels, including alpha, are recovered by XORing the two
|
||||
parts. Only the first frame of animated input is used. Keep both files unchanged: resizing,
|
||||
re-encoding with loss or flattening their alpha channels prevents correct reconstruction.
|
||||
Both parts together reconstruct the original image, so splitting is not access control.
|
||||
|
||||
Run the splitter's round-trip and overwrite checks with
|
||||
`python3 -m unittest discover -s scripts -p 'test_*.py'`.
|
||||
|
||||
The directories initially contain no images. The two dataset tests are explicitly ignored
|
||||
until you provide fixtures. To run them after adding pairs to both directories:
|
||||
To run the tests:
|
||||
|
||||
```sh
|
||||
SQLX_OFFLINE=true cargo test --locked directory_images -- --ignored
|
||||
```
|
||||
|
||||
The tests load all numbered pairs, recombine them in memory and pass the reconstructed PNG
|
||||
through the production upload moderation function using the bundled model and its current
|
||||
threshold. They also check that disabling moderation accepts the reconstructed images.
|
||||
Empty directories, missing partners, mismatched dimensions, unexpected names and unexpected
|
||||
moderation results fail the tests. Recombined images are never written to disk.
|
||||
|
||||
## Deterministic model fixtures
|
||||
|
||||
To regenerate them with Python's `onnx` package:
|
||||
|
||||
Whitespace-only changes.
Vendored
Whitespace-only changes.
Reference in new issue
Block a user