Compare commits
1
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e1e49904a8 |
No files matched your search
@@ -5,13 +5,3 @@
|
|||||||
[GantMan/nsfw_model](https://github.com/GantMan/nsfw_model).
|
[GantMan/nsfw_model](https://github.com/GantMan/nsfw_model).
|
||||||
|
|
||||||
Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx
|
Source: https://github.com/fyko/nsfw/releases/download/v0.2.0/model.onnx
|
||||||
|
|
||||||
Size: 17,320,596 bytes (approximately 16.5 MiB).
|
|
||||||
|
|
||||||
SHA-256: `0dca163d933af88e03fa0f9d233655857e9cc9f38bfd2bc28ba57c486d70feaf`.
|
|
||||||
|
|
||||||
The application embeds these bytes using `include_bytes!` and initializes the model only
|
|
||||||
when dashboard S3 uploads are configured and `NSFW_CHECK_ENABLED` is enabled (the default).
|
|
||||||
Set `NSFW_CHECK_ENABLED=false` to skip model initialization and moderation.
|
|
||||||
Keep this file in the source checkout and build
|
|
||||||
context; it does not need to be installed separately alongside the compiled binary.
|
|
||||||
@@ -4,5 +4,7 @@ CREATE TABLE image_uploads (
|
|||||||
uploader BIGINT UNSIGNED NOT NULL,
|
uploader BIGINT UNSIGNED NOT NULL,
|
||||||
url VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL UNIQUE,
|
url VARCHAR(512) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci NOT NULL UNIQUE,
|
||||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||||
INDEX image_uploads_uploader (uploader)
|
orphaned_at DATETIME DEFAULT NULL,
|
||||||
|
INDEX image_uploads_uploader (uploader),
|
||||||
|
INDEX image_uploads_orphaned_at (orphaned_at)
|
||||||
);
|
);
|
||||||
@@ -1,4 +0,0 @@
|
|||||||
ALTER TABLE image_uploads
|
|
||||||
-- UTC time at which cleanup first observed that this upload had no references.
|
|
||||||
ADD COLUMN orphaned_at DATETIME(6) NULL DEFAULT NULL,
|
|
||||||
ADD INDEX image_uploads_orphaned_at (orphaned_at);
|
|
||||||
@@ -88,10 +88,11 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
|
|||||||
const uploadInProgress = useRef(false);
|
const uploadInProgress = useRef(false);
|
||||||
|
|
||||||
const upload = async (files: File[]) => {
|
const upload = async (files: File[]) => {
|
||||||
|
setBlocked(false);
|
||||||
if (!uploadsEnabled || uploadInProgress.current) return;
|
if (!uploadsEnabled || uploadInProgress.current) return;
|
||||||
setError("");
|
setError("");
|
||||||
if (files.length !== 1) {
|
if (files.length !== 1) {
|
||||||
setError("Please upload one image at a time.");
|
setError("Please upload only one image.");
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const file = files[0];
|
const file = files[0];
|
||||||
@@ -128,27 +129,6 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
|
|||||||
if (droppedFiles) void upload(droppedFiles);
|
if (droppedFiles) void upload(droppedFiles);
|
||||||
}, [droppedFiles]);
|
}, [droppedFiles]);
|
||||||
|
|
||||||
if (blocked) {
|
|
||||||
return (
|
|
||||||
<Modal
|
|
||||||
setModalOpen={() => setBlocked(false)}
|
|
||||||
title="Image upload blocked"
|
|
||||||
onSubmit={() => setBlocked(false)}
|
|
||||||
onSubmitText="OK"
|
|
||||||
>
|
|
||||||
<p>
|
|
||||||
This image was flagged as potentially NSFW. Unfortunately, due to the UK's
|
|
||||||
Online Safety Act, we cannot handle explicit content. Please defer to using
|
|
||||||
Imgur or another image storage platform who have the resources to deal with
|
|
||||||
idiotic laws.
|
|
||||||
</p>
|
|
||||||
<p>
|
|
||||||
The image you attempted to upload <strong>has not been stored.</strong>
|
|
||||||
</p>
|
|
||||||
</Modal>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
return (
|
||||||
<Modal
|
<Modal
|
||||||
setModalOpen={setModalOpen}
|
setModalOpen={setModalOpen}
|
||||||
@@ -180,6 +160,7 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
|
|||||||
{...handlers}
|
{...handlers}
|
||||||
class={`mt-4 image-upload-dropzone ${dragging ? "image-picker-dragging" : ""}`}
|
class={`mt-4 image-upload-dropzone ${dragging ? "image-picker-dragging" : ""}`}
|
||||||
>
|
>
|
||||||
|
<p>or…</p>
|
||||||
<input
|
<input
|
||||||
ref={fileInput}
|
ref={fileInput}
|
||||||
type="file"
|
type="file"
|
||||||
@@ -203,8 +184,22 @@ const ImagePickerModal = ({ setModalOpen, setImage, url, uploadsEnabled, dropped
|
|||||||
</button>
|
</button>
|
||||||
<p class="help">Or drag and drop an image here.</p>
|
<p class="help">Or drag and drop an image here.</p>
|
||||||
<p class="help">
|
<p class="help">
|
||||||
PNG, JPEG, GIF or WebP, up to 8 MiB. All uploads are converted to PNGs.
|
Static PNG, JPEG, GIF or WebP, up to 8 MiB. All uploads are converted to
|
||||||
|
PNGs.
|
||||||
</p>
|
</p>
|
||||||
|
{blocked && (
|
||||||
|
<>
|
||||||
|
<p class="help" style="color: red;">
|
||||||
|
This image was flagged as potentially NSFW. Unfortunately, due to
|
||||||
|
the UK's Online Safety Act, we cannot handle explicit content.
|
||||||
|
Please defer to using Imgur or another image storage platform who
|
||||||
|
have the resources to deal with idiotic laws.
|
||||||
|
</p>
|
||||||
|
<p class="help" style="color: red;">
|
||||||
|
The image you attempted to upload has not been stored.
|
||||||
|
</p>
|
||||||
|
</>
|
||||||
|
)}
|
||||||
{uploading && <p role="status">Uploading image…</p>}
|
{uploading && <p role="status">Uploading image…</p>}
|
||||||
</div>
|
</div>
|
||||||
)}
|
)}
|
||||||
|
|||||||
+10
-10
@@ -84,6 +84,7 @@ impl ImageUploadService {
|
|||||||
{
|
{
|
||||||
return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into());
|
return Err("S3_PUBLIC_URL must be an HTTP(S) base URL of at most 400 bytes".into());
|
||||||
}
|
}
|
||||||
|
|
||||||
let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty());
|
let endpoint = env::var("S3_ENDPOINT").ok().filter(|value| !value.trim().is_empty());
|
||||||
let region = match &endpoint {
|
let region = match &endpoint {
|
||||||
Some(endpoint) => {
|
Some(endpoint) => {
|
||||||
@@ -91,11 +92,13 @@ impl ImageUploadService {
|
|||||||
}
|
}
|
||||||
None => values[1].parse()?,
|
None => values[1].parse()?,
|
||||||
};
|
};
|
||||||
|
|
||||||
let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?;
|
let credentials = Credentials::new(Some(&values[2]), Some(&values[3]), None, None, None)?;
|
||||||
let mut bucket = Bucket::new(&values[0], region, credentials)?;
|
let mut bucket = Bucket::new(&values[0], region, credentials)?;
|
||||||
if endpoint.is_some() {
|
if endpoint.is_some() {
|
||||||
bucket = bucket.with_path_style();
|
bucket = bucket.with_path_style();
|
||||||
}
|
}
|
||||||
|
|
||||||
bucket.set_request_timeout(Some(std::time::Duration::from_secs(30)));
|
bucket.set_request_timeout(Some(std::time::Duration::from_secs(30)));
|
||||||
let model = if check_nsfw {
|
let model = if check_nsfw {
|
||||||
Some(nsfw::create_model(MODEL_BYTES)?)
|
Some(nsfw::create_model(MODEL_BYTES)?)
|
||||||
@@ -156,17 +159,21 @@ fn decode_image(bytes: &[u8]) -> Result<image::RgbaImage, ImageError> {
|
|||||||
) {
|
) {
|
||||||
return Err(ImageError::InvalidImage);
|
return Err(ImageError::InvalidImage);
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut limits = Limits::default();
|
let mut limits = Limits::default();
|
||||||
limits.max_image_width = Some(8192);
|
limits.max_image_width = Some(8192);
|
||||||
limits.max_image_height = Some(8192);
|
limits.max_image_height = Some(8192);
|
||||||
limits.max_alloc = Some(128 * 1024 * 1024);
|
limits.max_alloc = Some(128 * 1024 * 1024);
|
||||||
reader.limits(limits);
|
reader.limits(limits);
|
||||||
|
|
||||||
let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?;
|
let decoder = reader.into_decoder().map_err(|_| ImageError::InvalidImage)?;
|
||||||
|
|
||||||
use image::ImageDecoder;
|
use image::ImageDecoder;
|
||||||
let (width, height) = decoder.dimensions();
|
let (width, height) = decoder.dimensions();
|
||||||
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS {
|
if width == 0 || height == 0 || u64::from(width) * u64::from(height) > MAX_IMAGE_PIXELS {
|
||||||
return Err(ImageError::InvalidImage);
|
return Err(ImageError::InvalidImage);
|
||||||
}
|
}
|
||||||
|
|
||||||
image::DynamicImage::from_decoder(decoder)
|
image::DynamicImage::from_decoder(decoder)
|
||||||
.map(|image| image.to_rgba8())
|
.map(|image| image.to_rgba8())
|
||||||
.map_err(|_| ImageError::InvalidImage)
|
.map_err(|_| ImageError::InvalidImage)
|
||||||
@@ -176,6 +183,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
|
|||||||
let image = decode_image(bytes)?;
|
let image = decode_image(bytes)?;
|
||||||
// Flatten transparency so the classifier sees exactly the published image.
|
// Flatten transparency so the classifier sees exactly the published image.
|
||||||
let mut visible = image::RgbImage::new(image.width(), image.height());
|
let mut visible = image::RgbImage::new(image.width(), image.height());
|
||||||
|
|
||||||
for (target, source) in visible.pixels_mut().zip(image.pixels()) {
|
for (target, source) in visible.pixels_mut().zip(image.pixels()) {
|
||||||
for channel in 0..3 {
|
for channel in 0..3 {
|
||||||
let alpha = u32::from(source[3]);
|
let alpha = u32::from(source[3]);
|
||||||
@@ -183,6 +191,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
|
|||||||
((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8;
|
((u32::from(source[channel]) * alpha + 255 * (255 - alpha)) / 255) as u8;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let visible = image::DynamicImage::ImageRgb8(visible);
|
let visible = image::DynamicImage::ImageRgb8(visible);
|
||||||
if let Some(model) = model {
|
if let Some(model) = model {
|
||||||
let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| {
|
let predictions = nsfw::examine(model, &visible.to_rgba8()).map_err(|error| {
|
||||||
@@ -193,6 +202,7 @@ pub fn prepare_image(model: Option<&nsfw::Model>, bytes: &[u8]) -> Result<Vec<u8
|
|||||||
return Err(ImageError::Nsfw);
|
return Err(ImageError::Nsfw);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
let mut output = Cursor::new(Vec::new());
|
let mut output = Cursor::new(Vec::new());
|
||||||
visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?;
|
visible.write_to(&mut output, ImageFormat::Png).map_err(|_| ImageError::InvalidImage)?;
|
||||||
Ok(output.into_inner())
|
Ok(output.into_inner())
|
||||||
@@ -233,16 +243,6 @@ mod tests {
|
|||||||
assert!(ImageUploads::from_settings(&settings).0.is_none());
|
assert!(ImageUploads::from_settings(&settings).0.is_none());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
|
||||||
fn nsfw_check_defaults_to_enabled_and_accepts_explicit_toggles() {
|
|
||||||
for value in [None, Some(""), Some("true"), Some("1"), Some("ON"), Some("invalid")] {
|
|
||||||
assert!(nsfw_check_enabled(value));
|
|
||||||
}
|
|
||||||
for value in [Some("false"), Some("0"), Some("OFF"), Some(" false ")] {
|
|
||||||
assert!(!nsfw_check_enabled(value));
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn disabling_nsfw_checks_skips_model_initialization() {
|
fn disabling_nsfw_checks_skips_model_initialization() {
|
||||||
let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"]
|
let settings = ["images", "eu-west-2", "test", "test", "https://images.example.com"]
|
||||||
|
|||||||
Vendored
+1
-20
@@ -19,31 +19,12 @@ python3 scripts/split_nsfw_test_image.py /path/to/input.jpg --output-dir tests/f
|
|||||||
python3 scripts/split_nsfw_test_image.py /path/to/safe.jpg --output-dir tests/fixtures/non_nsfw
|
python3 scripts/split_nsfw_test_image.py /path/to/safe.jpg --output-dir tests/fixtures/non_nsfw
|
||||||
```
|
```
|
||||||
|
|
||||||
Each command automatically selects the first unused positive index, starting with
|
To run the tests:
|
||||||
`image_1a.png` and `image_1b.png`. Indices with either part already present are skipped.
|
|
||||||
You can override the selection with `--index 2`; existing parts are never overwritten.
|
|
||||||
Each part contains random-looking
|
|
||||||
RGBA pixels. The original decoded pixels, including alpha, are recovered by XORing the two
|
|
||||||
parts. Only the first frame of animated input is used. Keep both files unchanged: resizing,
|
|
||||||
re-encoding with loss or flattening their alpha channels prevents correct reconstruction.
|
|
||||||
Both parts together reconstruct the original image, so splitting is not access control.
|
|
||||||
|
|
||||||
Run the splitter's round-trip and overwrite checks with
|
|
||||||
`python3 -m unittest discover -s scripts -p 'test_*.py'`.
|
|
||||||
|
|
||||||
The directories initially contain no images. The two dataset tests are explicitly ignored
|
|
||||||
until you provide fixtures. To run them after adding pairs to both directories:
|
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
SQLX_OFFLINE=true cargo test --locked directory_images -- --ignored
|
SQLX_OFFLINE=true cargo test --locked directory_images -- --ignored
|
||||||
```
|
```
|
||||||
|
|
||||||
The tests load all numbered pairs, recombine them in memory and pass the reconstructed PNG
|
|
||||||
through the production upload moderation function using the bundled model and its current
|
|
||||||
threshold. They also check that disabling moderation accepts the reconstructed images.
|
|
||||||
Empty directories, missing partners, mismatched dimensions, unexpected names and unexpected
|
|
||||||
moderation results fail the tests. Recombined images are never written to disk.
|
|
||||||
|
|
||||||
## Deterministic model fixtures
|
## Deterministic model fixtures
|
||||||
|
|
||||||
To regenerate them with Python's `onnx` package:
|
To regenerate them with Python's `onnx` package:
|
||||||
|
|||||||
Whitespace-only changes.
Vendored
Whitespace-only changes.
Reference in new issue
Block a user