diff --git a/documentation/colormap.md b/documentation/colormap.md new file mode 100644 index 0000000..621b048 --- /dev/null +++ b/documentation/colormap.md @@ -0,0 +1,63 @@ +# Colormap Examples +Keeping everything the same, but changing the color map to demonstrate visual differences. + +## understanding Colormaps +These apply a color gradient to the greyscale saliency values. Low values (less important) get one end of the gradient, high values (more important) get the other. +inferno — Black → purple → orange → yellow. The default. Dramatic, high contrast. +plasma — Purple → pink → yellow. Vibrant and synthetic-looking. Feels digital, almost neon. +magma — Black → deep purple → pink-white. Darker and contrast than inferno. +viridis — Purple → teal → yellow-green. The standard scientific colormap, designed to be perceptually uniform. +hot — Black → red → orange → white. Classic heat map / thermal camera. +cool — Cyan → magenta. Cold, clinical. +spring — Magenta → yellow. Neon, synthetic, light greens. +copper — Black → dark brown → copper-gold. Archival, almost daguerreotype-y. +twilight — purple → white → orange → purple. Atmospheric brown. +ocean — Black-blue → cyan → white. Deep and watery blue + +## What is it? + +Model_name: vgg16 +method: guided IG +composition: inverse +colormap: THIS IS WHAT CHANGES +intensity: 0.75 +contrast: 1.5 +threshold: 0.25 +class_index: -1 +input_size: 224 +uses_smooth_grad: true +snoothgrad_samples: 25 + + +## listed + + +cool +![cool](method_demo/cool.png) + +hot +![hot](method_demo/hot.png) + +copper +![copper](method_demo/copper.png) + +inferno +![inferno](method_demo/inferno.png) + +magma +![magma](method_demo/magma.png) + +ocean +![ocean](method_demo/ocean.png) + +plasma +![plasma](method_demo/plasma.png) + +spring +![spring](method_demo/spring.png) + +twilight +![twilight](method_demo/twilight.png) + +viridis +![viridis](method_demo/viridis.png) \ No newline at end of file diff --git a/documentation/colormap_demo/cool.png b/documentation/colormap_demo/cool.png new file mode 100644 index 0000000..09897c7 Binary files /dev/null and b/documentation/colormap_demo/cool.png differ diff --git a/documentation/colormap_demo/copper.png b/documentation/colormap_demo/copper.png new file mode 100644 index 0000000..59c37c1 Binary files /dev/null and b/documentation/colormap_demo/copper.png differ diff --git a/documentation/colormap_demo/hot.png b/documentation/colormap_demo/hot.png new file mode 100644 index 0000000..26d2bfd Binary files /dev/null and b/documentation/colormap_demo/hot.png differ diff --git a/documentation/colormap_demo/inferno.png b/documentation/colormap_demo/inferno.png new file mode 100644 index 0000000..b048da8 Binary files /dev/null and b/documentation/colormap_demo/inferno.png differ diff --git a/documentation/colormap_demo/magma.png b/documentation/colormap_demo/magma.png new file mode 100644 index 0000000..a7c51ef Binary files /dev/null and b/documentation/colormap_demo/magma.png differ diff --git a/documentation/colormap_demo/ocean.png b/documentation/colormap_demo/ocean.png new file mode 100644 index 0000000..44a16c5 Binary files /dev/null and b/documentation/colormap_demo/ocean.png differ diff --git a/documentation/colormap_demo/plasma.png b/documentation/colormap_demo/plasma.png new file mode 100644 index 0000000..5da7d79 Binary files /dev/null and b/documentation/colormap_demo/plasma.png differ diff --git a/documentation/colormap_demo/spring.png b/documentation/colormap_demo/spring.png new file mode 100644 index 0000000..f138e22 Binary files /dev/null and b/documentation/colormap_demo/spring.png differ diff --git a/documentation/colormap_demo/twilight.png b/documentation/colormap_demo/twilight.png new file mode 100644 index 0000000..e0274df Binary files /dev/null and b/documentation/colormap_demo/twilight.png differ diff --git a/documentation/colormap_demo/viridis.png b/documentation/colormap_demo/viridis.png new file mode 100644 index 0000000..62bb370 Binary files /dev/null and b/documentation/colormap_demo/viridis.png differ diff --git a/documentation/composition.md b/documentation/composition.md index a9d5fd9..7e55276 100644 --- a/documentation/composition.md +++ b/documentation/composition.md @@ -1,7 +1,18 @@ -# Methods Examples -Keeping everything the same, but changing the model to demonstrate visual differences. - +# Composition Examples +Keeping everything the same, but changing the composition to demonstrate visual differences. +## Understanding compostion modes +These control how the saliency map is composited back onto the original image. This is where the artistic intent lives. +isolation — The default and most conceptually pointed for your project. Salient regions show in full color using the chosen colormap; everything else is converted to greyscale. The effect: the machine's gaze burns in color while everything it ignores fades to grey. Very direct visual commentary on algorithmic attention. +overlay — Classic heatmap blend. The colormap is alpha-composited over the original at the intensity level. Familiar, readable, less subversive. +spotlight — The salient regions are bright, everything else darkens dramatically. Feels like a searchlight or surveillance camera — forensic, clinical. +ghost — The original image fades to 25% opacity and the saliency map glows over it. The subject becomes spectral, haunted by its own classification. Good for an uncanny effect. +invert — Shows what the algorithm ignores instead of what it sees. The saliency map is flipped before compositing. Conceptually this is the most subversive mode — it visualizes the blind spots, the parts of a face that are invisible to the machine. +cutout — Hard binary mask. Pixels above the threshold value show the original image; everything below shows a dim version of the heatmap. Creates a stark, graphic effect — the subject literally cut out by the algorithm's attention boundary. +multiply — Photoshop-style multiply blend. Attention dims non-salient areas. Produces a dark, moody result where ignored regions sink into shadow. +screen — Photoshop-style screen blend. Attention brightens salient regions. Lighter and more luminous than multiply — glowing rather than shadowed. +triptych — Outputs three panels side by side: original | saliency map alone | composite. Designed for gallery or exhibition display — shows the full chain of the visualization in one image. +mask_only — Just the colored saliency map with no original image. Pure algorithmic output, no human context. ## What is it? diff --git a/documentation/extras.md b/documentation/extras.md new file mode 100644 index 0000000..b964585 --- /dev/null +++ b/documentation/extras.md @@ -0,0 +1,85 @@ +# Colormap Examples +Keeping everything the same, but changing the color map to demonstrate visual differences. + + +Other Parameters +intensity — How strongly the saliency visualization is blended onto the original. 0 = invisible, 1 = full. Most compositions look best between 0.6–0.85. +contrast (gamma) — Applies a power function to the saliency values before visualization. Values above 1.0 compress the low end and expand the high end — making the most salient regions pop dramatically while less important regions fade further. Cranking this to 2.5–3.0 produces extreme, high-drama results where only the very top attention regions survive. +threshold — Used primarily by isolation and cutout modes. Sets the minimum saliency value to be considered "attended to." Higher values make the visualization sparser and more selective — only the regions the model is most confident about survive. +class_index — ImageNet has 1000 classes. By default (-1) the node auto-detects what the model predicted and visualizes attention for that class. You can override this to force the visualization to show what the model would attend to if it were trying to classify the image as a specific thing — for example, forcing class 207 (golden retriever) on a human portrait produces attention patterns for a question the model was never meant to answer, which has interesting artistic implications. +input_size — All these models were trained on fixed-size inputs. 224×224 is standard for most; InceptionV3 requires 299×299. The image is resized to this before processing, which is why the output appears at that resolution. +use_smoothgrad — When enabled on top of any method (except SmoothGrad which already does this), applies the SmoothGrad averaging process to smooth the result. Generally produces more aesthetically pleasing outputs at the cost of being slower. +smoothgrad_samples — How many noisy samples to average when SmoothGrad is active. 25 is a good balance; 50+ produces very smooth results but takes proportionally longer. + + +## What is it? + +Model_name: vgg16 +method: guided IG +composition: inverse +colormap: magma +[the following IS WHAT CHANGES] +default values: + +intensity: 0.75 +contrast: 1.5 +threshold: 0.25 +class_index: -1 +input_size: 224 +uses_smooth_grad: true +snoothgrad_samples: 25 + + +## listed out +should explain why i picked the default values i did. + +### Intensity + +intensity = 0.5 +(lower) +![low_intensity](method_demo/intensity_0.5.png) + +intensity = 1.0 +(higher) +![high_intensity](method_demo/intensity_1.0.png) + +### Contrast + +contrast = 1.0 +(lower) +![low_contrast](method_demo/contrast_1.0.png) + +contrast = 2.0 +(higher) +![high_contrast](method_demo/contrast_2.0.png) + +### threshold +threshold = 0.0 +(lower) +![low_threshold](method_demo/threshold_0.png) + +threshold = 0.5 +(higher) +![high_threshold](method_demo/threshold_0.5.png) + + ### class index +class index = 1 +(a little change) +![index-1](method_demo/class_index_1.png) + +class index = 10 +(a larger change) +![index-10](method_demo/class_index_10.png) + + ### input size + Didn't adjust this because most models require this size (224) to run properly. + + ### uses smooth grad +Without (uses smooth grad = false) +![false](method_demo/smooth_grad_false.png.png) + + + + ### smooth grad samples +samples = 50 + ![samples_50](method_demo/samples_50.png) \ No newline at end of file diff --git a/documentation/extras_demo/class_index_1.png b/documentation/extras_demo/class_index_1.png new file mode 100644 index 0000000..f880271 Binary files /dev/null and b/documentation/extras_demo/class_index_1.png differ diff --git a/documentation/extras_demo/class_index_10.png b/documentation/extras_demo/class_index_10.png new file mode 100644 index 0000000..9e24210 Binary files /dev/null and b/documentation/extras_demo/class_index_10.png differ diff --git a/documentation/extras_demo/contrast_1.0.png b/documentation/extras_demo/contrast_1.0.png new file mode 100644 index 0000000..e43975c Binary files /dev/null and b/documentation/extras_demo/contrast_1.0.png differ diff --git a/documentation/extras_demo/contrast_2.0.png b/documentation/extras_demo/contrast_2.0.png new file mode 100644 index 0000000..1d4feb4 Binary files /dev/null and b/documentation/extras_demo/contrast_2.0.png differ diff --git a/documentation/extras_demo/intensity_0.5.png b/documentation/extras_demo/intensity_0.5.png new file mode 100644 index 0000000..1a73fc0 Binary files /dev/null and b/documentation/extras_demo/intensity_0.5.png differ diff --git a/documentation/extras_demo/intensity_1.0.png b/documentation/extras_demo/intensity_1.0.png new file mode 100644 index 0000000..c603079 Binary files /dev/null and b/documentation/extras_demo/intensity_1.0.png differ diff --git a/documentation/extras_demo/samples_50.png b/documentation/extras_demo/samples_50.png new file mode 100644 index 0000000..bb4e0a5 Binary files /dev/null and b/documentation/extras_demo/samples_50.png differ diff --git a/documentation/extras_demo/smooth_grad_false.png b/documentation/extras_demo/smooth_grad_false.png new file mode 100644 index 0000000..5d06b41 Binary files /dev/null and b/documentation/extras_demo/smooth_grad_false.png differ diff --git a/documentation/extras_demo/threshold_0.5.png b/documentation/extras_demo/threshold_0.5.png new file mode 100644 index 0000000..cf3ea51 Binary files /dev/null and b/documentation/extras_demo/threshold_0.5.png differ diff --git a/documentation/extras_demo/threshold_0.png b/documentation/extras_demo/threshold_0.png new file mode 100644 index 0000000..9d58bd2 Binary files /dev/null and b/documentation/extras_demo/threshold_0.png differ diff --git a/documentation/methods.md b/documentation/methods.md index d8a6ad4..2863f82 100644 --- a/documentation/methods.md +++ b/documentation/methods.md @@ -1,6 +1,14 @@ # Methods Examples Keeping everything the same, but changing the model to demonstrate visual differences. +## Understanding methods +These are the mathematical techniques used to ask "what pixels mattered to this model's decision?" Each has a different visual character. +Vanilla Gradients — The simplest approach: take the derivative of the output with respect to each input pixel. Fast and raw. The result is noisy and looks almost like static or grain — shows every pixel that had any influence at all, including spurious ones. Visually the most aggressive and chaotic. +SmoothGrad — Takes Vanilla Gradients but runs it many times with small random noise added to the image each time, then averages the results. This cancels out the noise and reveals more stable patterns. Visually smoother and more "reliable" looking than vanilla, but still texture-rich. The smoothgrad_samples parameter controls how many noisy runs are averaged — more = smoother but slower. +Integrated Gradients — Instead of just looking at gradients at the input image, this traces a path from a blank black image to your actual image and accumulates gradients along that path. This gives it a strong theoretical foundation (it satisfies mathematical axioms about attribution). Produces clean, smooth maps that highlight meaningful regions rather than just any active pixels. The ig_steps parameter controls how many steps along that path — more = more accurate but slower. This is usually the best starting method. +Blur IG — A variant of Integrated Gradients where instead of interpolating from black to the image, it interpolates from a heavily blurred version to the sharp image. This tends to produce maps focused on edges and structural details rather than color/texture. Produces a different aesthetic — more architectural-looking. +Guided IG — Another IG variant that uses gradient information to choose a smarter integration path, concentrating steps where the gradients are largest. Produces sharper, higher-contrast attribution maps with more defined edges. Often the most visually striking for portraits. +XRAI — Completely different approach. Instead of pixel-level gradients, it segments the image into regions and attributes importance to each region as a whole. Produces a patchwork/segmented look — almost like stained glass or a heat map made of irregular shapes. Very different aesthetic from the gradient methods. Uses algorithm="fast" by default; changing to "full" in the code gives higher quality but is much slower. ## What is it? diff --git a/documentation/model.md b/documentation/model.md new file mode 100644 index 0000000..63705f8 --- /dev/null +++ b/documentation/model.md @@ -0,0 +1,36 @@ +# Model Examples +Keeping everything the same, but changing the model to demonstrate visual differences. + +## Understanding Models +Models +The model is the "brain" being interrogated — it's a pretrained ImageNet classifier whose internal decision-making we're visualizing. Different models have different "personalities" in how they look at images. + +need to explain why these models, where they came from. explain reasoning behind choices + +VGG16 / VGG19 — Old, deep, sequential architecture from 2014. For portraits, these are the most interesting because they're blunt and spatially crude. The attention patterns feel almost aggressive — big, blocky regions. Good for the "surveillance" feeling you're going for. VGG16 is the default for a reason. +ResNet50 / ResNet101 — More modern, uses skip connections. Produces cleaner, more distributed attention. Less visually dramatic than VGG but more "honest" about what it's actually classifying. + +DenseNet121 — Dense connectivity means attention is highly distributed across the image. Produces diffuse, almost painterly saliency maps. +MobileNetV3 — Designed for mobile devices, so it's aggressively compressed. Produces the sparsest, most stripped-down attention — shows only the absolute minimum the model needs to make a decision. +InceptionV3 — Uses parallel convolutions at multiple scales. Produces multi-scale attention that can look almost fractal. Requires input_size 299 instead of 224. + +## What is it? + +Model_name: THIS IS WHAT CHANGES +method: guided IG +composition: pure_white +colormap: magma +intensity: 0.75 +contrast: 1.5 +threshold: 0.25 +class_index: -1 +input_size: 224 +uses_smooth_grad: true +snoothgrad_samples: 25 + + +## listed out + + +cool +![cool](method_demo/cool.png) diff --git a/saliency_nodes.py b/saliency_nodes.py index 8a571ef..31deb08 100644 --- a/saliency_nodes.py +++ b/saliency_nodes.py @@ -36,7 +36,8 @@ IMAGENET_STD = np.array([0.229, 0.224, 0.225], dtype=np.float32) SUPPORTED_MODELS = [ "vgg16", "vgg19", "resnet50", "resnet101", "inception_v3", "mobilenet_v3_large", - "efficientnet_b0", "densenet121", + # "efficientnet_b0", + "densenet121", ] _model_cache: dict = {} @@ -72,7 +73,7 @@ def _load_model(model_name: str): "vgg16": tv.VGG16_Weights.DEFAULT, "vgg19": tv.VGG19_Weights.DEFAULT, "mobilenet_v3_large": tv.MobileNet_V3_Large_Weights.DEFAULT, - "efficientnet_b0": tv.EfficientNet_B0_Weights.DEFAULT, + # "efficientnet_b0": tv.EfficientNet_B0_Weights.DEFAULT, "densenet121": tv.DenseNet121_Weights.DEFAULT, } model = constructor(weights=weight_map.get(model_name))