diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00000.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00000.jpg index 25b085b..49d9a3d 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00000.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00000.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00001.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00001.jpg index 8efe746..9bdfff7 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00001.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00001.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00002.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00002.jpg index c0f2422..0c388a2 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00002.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00002.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00003.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00003.jpg index 9e1f225..8eb6a49 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00003.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00003.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00004.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00004.jpg index c4b8035..7607dc1 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00004.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00004.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00005.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00005.jpg index 59af7d3..21cc55e 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00005.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00005.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00006.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00006.jpg index 16af058..8b5ad49 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00006.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00006.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00007.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00007.jpg index a4dfa89..2c4fa2d 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00007.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00007.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00008.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00008.jpg index 8837b37..d1bb6e7 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00008.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00008.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00009.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00009.jpg index ebfdd00..17d2718 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00009.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00009.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00010.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00010.jpg index fd05f13..2c7583e 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00010.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00010.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00011.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00011.jpg index 3d8cdb6..95268e9 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00011.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00011.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00012.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00012.jpg index 478a4cc..88cbb6d 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00012.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00012.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00013.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00013.jpg index b7b422b..79353fd 100644 Binary files a/legacy_videos/legacy_clips_output/v32_v32_01/00013.jpg and b/legacy_videos/legacy_clips_output/v32_v32_01/00013.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00073.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00073.jpg new file mode 100644 index 0000000..16c422a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00073.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00074.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00074.jpg new file mode 100644 index 0000000..740d1d1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00074.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00075.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00075.jpg new file mode 100644 index 0000000..c9dd6a4 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00075.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00076.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00076.jpg new file mode 100644 index 0000000..6c55424 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00076.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00077.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00077.jpg new file mode 100644 index 0000000..cdbb1c5 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00077.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00078.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00078.jpg new file mode 100644 index 0000000..5e32a51 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00078.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00079.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00079.jpg new file mode 100644 index 0000000..b0f38eb Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00079.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00080.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00080.jpg new file mode 100644 index 0000000..20e3c92 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00080.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00081.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00081.jpg new file mode 100644 index 0000000..2b38a00 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00081.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00082.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00082.jpg new file mode 100644 index 0000000..d2f8b27 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00082.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00083.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00083.jpg new file mode 100644 index 0000000..3a41dd2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00083.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00084.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00084.jpg new file mode 100644 index 0000000..d7ef39c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00084.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00085.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00085.jpg new file mode 100644 index 0000000..f51ef14 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00085.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00086.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00086.jpg new file mode 100644 index 0000000..151d215 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00086.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00087.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00087.jpg new file mode 100644 index 0000000..f928d10 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00087.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00088.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00088.jpg new file mode 100644 index 0000000..7885090 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00088.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00089.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00089.jpg new file mode 100644 index 0000000..0b7483c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00089.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00090.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00090.jpg new file mode 100644 index 0000000..eaf5d0e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00090.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00091.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00091.jpg new file mode 100644 index 0000000..5dbfa4e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00091.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00092.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00092.jpg new file mode 100644 index 0000000..8aa491e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00092.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00093.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00093.jpg new file mode 100644 index 0000000..e328d53 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00093.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00094.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00094.jpg new file mode 100644 index 0000000..c2daa9c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00094.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00095.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00095.jpg new file mode 100644 index 0000000..ab2cdf8 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00095.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00096.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00096.jpg new file mode 100644 index 0000000..4966dfe Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00096.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00097.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00097.jpg new file mode 100644 index 0000000..3f4a7fd Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00097.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00098.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00098.jpg new file mode 100644 index 0000000..3c556e8 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00098.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00099.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00099.jpg new file mode 100644 index 0000000..1056473 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00099.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00100.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00100.jpg new file mode 100644 index 0000000..8c67aac Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00100.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00101.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00101.jpg new file mode 100644 index 0000000..a5759ce Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00101.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00102.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00102.jpg new file mode 100644 index 0000000..4806fa1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00102.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00103.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00103.jpg new file mode 100644 index 0000000..f8f21b6 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00103.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00104.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00104.jpg new file mode 100644 index 0000000..6ec258c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00104.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00105.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00105.jpg new file mode 100644 index 0000000..3b466ef Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00105.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00106.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00106.jpg new file mode 100644 index 0000000..d0eb8d7 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00106.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00107.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00107.jpg new file mode 100644 index 0000000..92820ca Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00107.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00108.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00108.jpg new file mode 100644 index 0000000..ca78bce Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00108.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00109.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00109.jpg new file mode 100644 index 0000000..cfc0b95 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00109.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00110.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00110.jpg new file mode 100644 index 0000000..ff631a0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00110.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00111.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00111.jpg new file mode 100644 index 0000000..fd5c564 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00111.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00112.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00112.jpg new file mode 100644 index 0000000..e476dbb Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00112.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00113.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00113.jpg new file mode 100644 index 0000000..01edab9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00113.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00114.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00114.jpg new file mode 100644 index 0000000..d88bc95 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00114.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00115.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00115.jpg new file mode 100644 index 0000000..4a58a5a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00115.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00116.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00116.jpg new file mode 100644 index 0000000..59ee20b Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00116.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00117.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00117.jpg new file mode 100644 index 0000000..99ce78c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00117.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00118.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00118.jpg new file mode 100644 index 0000000..b6d37c9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00118.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00119.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00119.jpg new file mode 100644 index 0000000..c38bbed Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00119.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00120.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00120.jpg new file mode 100644 index 0000000..d6c45d3 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00120.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00121.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00121.jpg new file mode 100644 index 0000000..1e669a2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00121.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00122.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00122.jpg new file mode 100644 index 0000000..4da8643 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00122.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00123.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00123.jpg new file mode 100644 index 0000000..03c3788 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00123.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00124.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00124.jpg new file mode 100644 index 0000000..c1545e9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00124.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00125.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00125.jpg new file mode 100644 index 0000000..95e32fe Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00125.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00126.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00126.jpg new file mode 100644 index 0000000..19b1b57 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00126.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00127.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00127.jpg new file mode 100644 index 0000000..e43e6aa Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00127.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00128.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00128.jpg new file mode 100644 index 0000000..95c4a41 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00128.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00129.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00129.jpg new file mode 100644 index 0000000..12269b2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00129.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00130.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00130.jpg new file mode 100644 index 0000000..7bfea03 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00130.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00131.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00131.jpg new file mode 100644 index 0000000..701e362 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00131.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00132.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00132.jpg new file mode 100644 index 0000000..c2cf1fc Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00132.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00133.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00133.jpg new file mode 100644 index 0000000..37917ff Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00133.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00134.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00134.jpg new file mode 100644 index 0000000..240dc09 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00134.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00135.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00135.jpg new file mode 100644 index 0000000..cc2fb2f Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00135.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00136.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00136.jpg new file mode 100644 index 0000000..1d26f1c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00136.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00137.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00137.jpg new file mode 100644 index 0000000..6f1f133 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00137.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00138.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00138.jpg new file mode 100644 index 0000000..137a7ea Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00138.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00139.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00139.jpg new file mode 100644 index 0000000..159b379 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00139.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00140.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00140.jpg new file mode 100644 index 0000000..872a01b Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00140.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00141.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00141.jpg new file mode 100644 index 0000000..911f4d1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00141.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00142.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00142.jpg new file mode 100644 index 0000000..a3e9032 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00142.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00143.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00143.jpg new file mode 100644 index 0000000..b346cd4 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00143.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00144.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00144.jpg new file mode 100644 index 0000000..e2f523b Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00144.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00145.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00145.jpg new file mode 100644 index 0000000..e4d8a34 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00145.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00146.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00146.jpg new file mode 100644 index 0000000..9c3e736 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00146.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00147.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00147.jpg new file mode 100644 index 0000000..87d67ff Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00147.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00148.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00148.jpg new file mode 100644 index 0000000..35d7936 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00148.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00149.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00149.jpg new file mode 100644 index 0000000..128a027 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00149.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00150.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00150.jpg new file mode 100644 index 0000000..d1e5253 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00150.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00151.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00151.jpg new file mode 100644 index 0000000..d172048 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00151.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00152.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00152.jpg new file mode 100644 index 0000000..a935d31 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00152.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00153.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00153.jpg new file mode 100644 index 0000000..75d58f8 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00153.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00154.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00154.jpg new file mode 100644 index 0000000..eae76d1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00154.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00155.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00155.jpg new file mode 100644 index 0000000..26a944c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00155.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00156.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00156.jpg new file mode 100644 index 0000000..8a01a56 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00156.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00157.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00157.jpg new file mode 100644 index 0000000..e1429cf Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00157.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00158.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00158.jpg new file mode 100644 index 0000000..49af610 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00158.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00159.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00159.jpg new file mode 100644 index 0000000..3af7159 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00159.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00160.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00160.jpg new file mode 100644 index 0000000..c25486e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00160.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00161.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00161.jpg new file mode 100644 index 0000000..e6ed33a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00161.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00162.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00162.jpg new file mode 100644 index 0000000..e752d69 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00162.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00163.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00163.jpg new file mode 100644 index 0000000..d94ad60 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00163.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00164.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00164.jpg new file mode 100644 index 0000000..1c45792 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00164.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00165.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00165.jpg new file mode 100644 index 0000000..ae763f8 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00165.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00166.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00166.jpg new file mode 100644 index 0000000..b6aca73 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00166.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00167.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00167.jpg new file mode 100644 index 0000000..4102a47 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00167.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00168.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00168.jpg new file mode 100644 index 0000000..3359082 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00168.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00169.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00169.jpg new file mode 100644 index 0000000..8444948 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00169.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00170.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00170.jpg new file mode 100644 index 0000000..17d0071 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00170.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00171.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00171.jpg new file mode 100644 index 0000000..742d5a8 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00171.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00172.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00172.jpg new file mode 100644 index 0000000..673b9b3 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00172.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00173.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00173.jpg new file mode 100644 index 0000000..e814a91 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00173.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00174.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00174.jpg new file mode 100644 index 0000000..a41488a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00174.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00175.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00175.jpg new file mode 100644 index 0000000..fb884b0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00175.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00176.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00176.jpg new file mode 100644 index 0000000..6ac437e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00176.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00177.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00177.jpg new file mode 100644 index 0000000..0715a1c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00177.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00178.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00178.jpg new file mode 100644 index 0000000..69148e0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00178.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/00179.jpg b/legacy_videos/legacy_clips_output/v32_v32_01/00179.jpg new file mode 100644 index 0000000..32ac449 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/00179.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_01/video.avi b/legacy_videos/legacy_clips_output/v32_v32_01/video.avi new file mode 100644 index 0000000..e192ba0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_01/video.avi differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00000.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00000.jpg new file mode 100644 index 0000000..47ca6a9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00000.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00001.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00001.jpg new file mode 100644 index 0000000..d5ece42 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00001.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00002.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00002.jpg new file mode 100644 index 0000000..19a79e4 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00002.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00003.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00003.jpg new file mode 100644 index 0000000..0029337 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00003.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00004.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00004.jpg new file mode 100644 index 0000000..b23edf7 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00004.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00005.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00005.jpg new file mode 100644 index 0000000..3996628 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00005.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00006.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00006.jpg new file mode 100644 index 0000000..4b99e1a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00006.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00007.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00007.jpg new file mode 100644 index 0000000..109fc91 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00007.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00008.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00008.jpg new file mode 100644 index 0000000..b7154f9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00008.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00009.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00009.jpg new file mode 100644 index 0000000..d564dd3 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00009.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00010.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00010.jpg new file mode 100644 index 0000000..d422add Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00010.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00011.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00011.jpg new file mode 100644 index 0000000..e648bec Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00011.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00012.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00012.jpg new file mode 100644 index 0000000..ccc7a3c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00012.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00013.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00013.jpg new file mode 100644 index 0000000..3d419f1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00013.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00014.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00014.jpg new file mode 100644 index 0000000..5cd522b Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00014.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00015.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00015.jpg new file mode 100644 index 0000000..b1754f2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00015.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00016.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00016.jpg new file mode 100644 index 0000000..55189a1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00016.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00017.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00017.jpg new file mode 100644 index 0000000..07163c7 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00017.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00018.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00018.jpg new file mode 100644 index 0000000..d95578e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00018.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00019.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00019.jpg new file mode 100644 index 0000000..0a3323c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00019.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00020.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00020.jpg new file mode 100644 index 0000000..db69c04 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00020.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00021.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00021.jpg new file mode 100644 index 0000000..5220c81 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00021.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00022.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00022.jpg new file mode 100644 index 0000000..115bfcf Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00022.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00023.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00023.jpg new file mode 100644 index 0000000..73056b4 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00023.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00024.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00024.jpg new file mode 100644 index 0000000..40274e9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00024.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00025.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00025.jpg new file mode 100644 index 0000000..13ad82a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00025.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00026.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00026.jpg new file mode 100644 index 0000000..4d0afca Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00026.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00027.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00027.jpg new file mode 100644 index 0000000..1836689 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00027.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00028.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00028.jpg new file mode 100644 index 0000000..b89d9bc Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00028.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00029.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00029.jpg new file mode 100644 index 0000000..cd748b0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00029.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00030.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00030.jpg new file mode 100644 index 0000000..0f58f42 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00030.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00031.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00031.jpg new file mode 100644 index 0000000..259895c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00031.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00032.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00032.jpg new file mode 100644 index 0000000..7509e62 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00032.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00033.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00033.jpg new file mode 100644 index 0000000..6d4cac2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00033.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00034.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00034.jpg new file mode 100644 index 0000000..2284091 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00034.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00035.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00035.jpg new file mode 100644 index 0000000..936d5b3 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00035.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00036.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00036.jpg new file mode 100644 index 0000000..2f4bb29 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00036.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00037.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00037.jpg new file mode 100644 index 0000000..791ae75 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00037.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00038.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00038.jpg new file mode 100644 index 0000000..5079bb9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00038.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00039.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00039.jpg new file mode 100644 index 0000000..9a4c81a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00039.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00040.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00040.jpg new file mode 100644 index 0000000..90ece24 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00040.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00041.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00041.jpg new file mode 100644 index 0000000..4cb9fac Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00041.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00042.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00042.jpg new file mode 100644 index 0000000..97af238 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00042.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00043.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00043.jpg new file mode 100644 index 0000000..a0026c0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00043.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00044.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00044.jpg new file mode 100644 index 0000000..a1b179e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00044.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00045.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00045.jpg new file mode 100644 index 0000000..9f5047a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00045.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00046.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00046.jpg new file mode 100644 index 0000000..e2b1d3e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00046.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00047.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00047.jpg new file mode 100644 index 0000000..4c8c6b0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00047.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00048.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00048.jpg new file mode 100644 index 0000000..3921ab9 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00048.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00049.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00049.jpg new file mode 100644 index 0000000..c31ce69 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00049.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00050.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00050.jpg new file mode 100644 index 0000000..bff100e Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00050.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00051.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00051.jpg new file mode 100644 index 0000000..2ffa6e7 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00051.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00052.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00052.jpg new file mode 100644 index 0000000..9490c1a Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00052.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00053.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00053.jpg new file mode 100644 index 0000000..420f0ec Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00053.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00054.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00054.jpg new file mode 100644 index 0000000..a53dba0 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00054.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00055.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00055.jpg new file mode 100644 index 0000000..201d885 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00055.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00056.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00056.jpg new file mode 100644 index 0000000..35c5836 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00056.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00057.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00057.jpg new file mode 100644 index 0000000..cce625d Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00057.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00058.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00058.jpg new file mode 100644 index 0000000..78ba2ab Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00058.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00059.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00059.jpg new file mode 100644 index 0000000..44c7ed2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00059.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00060.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00060.jpg new file mode 100644 index 0000000..e989a71 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00060.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00061.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00061.jpg new file mode 100644 index 0000000..e8b35b6 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00061.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00062.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00062.jpg new file mode 100644 index 0000000..8848f59 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00062.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00063.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00063.jpg new file mode 100644 index 0000000..06315b1 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00063.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00064.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00064.jpg new file mode 100644 index 0000000..05dfa5f Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00064.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00065.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00065.jpg new file mode 100644 index 0000000..ed4c314 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00065.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00066.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00066.jpg new file mode 100644 index 0000000..427f9c2 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00066.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00067.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00067.jpg new file mode 100644 index 0000000..5a8252c Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00067.jpg differ diff --git a/legacy_videos/legacy_clips_output/v32_v32_02/00068.jpg b/legacy_videos/legacy_clips_output/v32_v32_02/00068.jpg new file mode 100644 index 0000000..8b7c4b5 Binary files /dev/null and b/legacy_videos/legacy_clips_output/v32_v32_02/00068.jpg differ diff --git a/lib/FeatVGG.py b/lib/FeatVGG.py index bb06ae3..32e33fe 100755 --- a/lib/FeatVGG.py +++ b/lib/FeatVGG.py @@ -17,7 +17,6 @@ class FeatVGG(nn.Module): def __init__(self, content_layers=["relu3_1"]): super(FeatVGG, self).__init__() self.content_layers = conv_to_relu(content_layers) - # set up vgg19 model self.vgg19 = vgg_util.get_renamed_vgg() self.last_c_layer = self.content_layers[-1] is_last_content = False @@ -52,11 +51,12 @@ class VGGNet_multilayer(nn.Module): self.vgg = models.vgg19(pretrained=True).features def forward(self, x): - # x: rgb image - # ranges in [0,1] - # should be normalzied with mean = [0.485, 0.456, 0.406] - # and variance = [0.229, 0.224, 0.225] - """Extract multiple convolutional feature maps.""" + """Extract multiple convolutional feature maps. + x: rgb image + ranges in [0,1] + should be normalzied with mean = [0.485, 0.456, 0.406] + and variance = [0.229, 0.224, 0.225] + """ features = [] for name, layer in self.vgg._modules.items(): x = layer(x) diff --git a/lib/HardSigmoid.py b/lib/HardSigmoid.py deleted file mode 100755 index 267382b..0000000 --- a/lib/HardSigmoid.py +++ /dev/null @@ -1,20 +0,0 @@ -from torch.autograd import Function, Variable - - -class HardSigmoid(Function): - @staticmethod - def forward(ctx, input): - output = (0.2 * input) + 0.5 - output[output > 1] = 1 - output[output < 0] = 0 - ctx.save_for_backward(input) - return output - - @staticmethod - def backward(ctx, grad_output): - input = ctx.saved_variables - grad_input = input.new(input.size()) - grad_input.data.fill_(0.2) - grad_input[input <= -2.5] = 0 - grad_input[input >= 2.5] = 0 - return grad_input diff --git a/lib/RefPairTransformsWithGlobalOrigRGB.py b/lib/RefPairTransformsWithGlobalOrigRGB.py deleted file mode 100644 index 9c9dfc4..0000000 --- a/lib/RefPairTransformsWithGlobalOrigRGB.py +++ /dev/null @@ -1,359 +0,0 @@ -from __future__ import division - -import math -import random - -import torch -from PIL import Image, ImageEnhance, ImageOps - -try: - import accimage -except ImportError: - accimage = None -import collections -import numbers -import types -import warnings - -import numpy as np -from skimage import color - -import lib.functional as F - -__all__ = [ - "Compose", - "Concatenate", - "ToTensor", - "Normalize", - "Resize", - "Scale", - "CenterCrop", - "Pad", - "RandomCrop", - "RandomHorizontalFlip", - "RandomVerticalFlip", - "RandomResizedCrop", - "RandomSizedCrop", - "FiveCrop", - "TenCrop", - "RGB2Lab", -] - - -def CustomFunc(inputs, func, *args, **kwargs): - im_l = func(inputs[0], *args, **kwargs) - im_ab = func(inputs[1], *args, **kwargs) - warp_ba = func(inputs[2], *args, **kwargs) - warp_aba = func(inputs[3], *args, **kwargs) - im_gbl_ab = func(inputs[4], *args, **kwargs) - bgr_mc_im = func(inputs[5], *args, **kwargs) - - layer_data = [im_l, im_ab, warp_ba, warp_aba, im_gbl_ab, bgr_mc_im] - - for l in range(5): - layer = inputs[6 + l] - err_ba = func(layer[0], *args, **kwargs) - err_ab = func(layer[1], *args, **kwargs) - - layer_data.append([err_ba, err_ab]) - - return layer_data - - -class Compose(object): - """Composes several transforms together. - - Args: - transforms (list of ``Transform`` objects): list of transforms to compose. - - Example: - >>> transforms.Compose([ - >>> transforms.CenterCrop(10), - >>> transforms.ToTensor(), - >>> ]) - """ - - def __init__(self, transforms): - self.transforms = transforms - - def __call__(self, inputs): - for t in self.transforms: - inputs = t(inputs) - return inputs - - -class Concatenate(object): - """ - Input: [im_l, im_ab, inputs] - inputs = [warp_ba_l, warp_ba_ab, warp_aba, err_pm, err_aba] - - Output:[im_l, err_pm, warp_ba, warp_aba, im_ab, err_aba] - """ - - def __call__(self, inputs): - im_l = inputs[0] - im_ab = inputs[1] - warp_ba = inputs[2] - warp_aba = inputs[3] - im_glb_ab = inputs[4] - bgr_mc_im = inputs[5] - bgr_mc_im = bgr_mc_im[[2, 1, 0], ...] - - err_ba = [] - err_ab = [] - - for l in range(5): - layer = inputs[6 + l] - err_ba.append(layer[0]) - err_ab.append(layer[1]) - - cerr_ba = torch.cat(err_ba, 0) - cerr_ab = torch.cat(err_ab, 0) - - return (im_l, cerr_ba, warp_ba, warp_aba, im_glb_ab, bgr_mc_im, im_ab, cerr_ab) - - -class ToTensor(object): - """Convert a ``PIL Image`` or ``numpy.ndarray`` to tensor. - - Converts a PIL Image or numpy.ndarray (H x W x C) in the range - [0, 255] to a torch.FloatTensor of shape (C x H x W) in the range [0.0, 1.0]. - """ - - def __call__(self, inputs): - """ - Args: - pic (PIL Image or numpy.ndarray): Image to be converted to tensor. - - Returns: - Tensor: Converted image. - """ - inputs = CustomFunc(inputs, F.to_mytensor) - return inputs - - -class Normalize(object): - """Normalize an tensor image with mean and standard deviation. - Given mean: ``(M1,...,Mn)`` and std: ``(S1,..,Sn)`` for ``n`` channels, this transform - will normalize each channel of the input ``torch.*Tensor`` i.e. - ``input[channel] = (input[channel] - mean[channel]) / std[channel]`` - - Args: - mean (sequence): Sequence of means for each channel. - std (sequence): Sequence of standard deviations for each channel. - """ - - def __call__(self, inputs): - """ - Args: - tensor (Tensor): Tensor image of size (C, H, W) to be normalized. - - Returns: - Tensor: Normalized Tensor image. - """ - - im_l = F.normalize(inputs[0], 50, 1) # [0, 100] - im_ab = F.normalize(inputs[1], (0, 0), (1, 1)) # [-100, 100] - - inputs[2][0:1, :, :] = F.normalize(inputs[2][0:1, :, :], 50, 1) - inputs[2][1:3, :, :] = F.normalize(inputs[2][1:3, :, :], (0, 0), (1, 1)) - warp_ba = inputs[2] - - inputs[3][0:1, :, :] = F.normalize(inputs[3][0:1, :, :], 50, 1) - inputs[3][1:3, :, :] = F.normalize(inputs[3][1:3, :, :], (0, 0), (1, 1)) - warp_aba = inputs[3] - - im_gbl_ab = F.normalize(inputs[4], (0, 0), (1, 1)) # [-100, 100] - - bgr_mc_im = F.normalize(inputs[5], (123.68, 116.78, 103.938), (1, 1, 1)) - - layer_data = [im_l, im_ab, warp_ba, warp_aba, im_gbl_ab, bgr_mc_im] - - for l in range(5): - layer = inputs[6 + l] - err_ba = F.normalize(layer[0], 127, 2) # [0, 255] - err_ab = F.normalize(layer[1], 127, 2) # [0, 255] - layer_data.append([err_ba, err_ab]) - - return layer_data - - -class Resize(object): - """Resize the input PIL Image to the given size. - - Args: - size (sequence or int): Desired output size. If size is a sequence like - (h, w), output size will be matched to this. If size is an int, - smaller edge of the image will be matched to this number. - i.e, if height > width, then image will be rescaled to - (size * height / width, size) - interpolation (int, optional): Desired interpolation. Default is - ``PIL.Image.BILINEAR`` - """ - - def __init__(self, size, interpolation=Image.BILINEAR): - assert isinstance(size, int) or (isinstance(size, collections.Iterable) and len(size) == 2) - self.size = size - self.interpolation = interpolation - - def __call__(self, inputs): - """ - Args: - img (PIL Image): Image to be scaled. - - Returns: - PIL Image: Rescaled image. - """ - return CustomFunc(inputs, F.resize, self.size, self.interpolation) - - -class RandomCrop(object): - """Crop the given PIL Image at a random location. - - Args: - size (sequence or int): Desired output size of the crop. If size is an - int instead of sequence like (h, w), a square crop (size, size) is - made. - padding (int or sequence, optional): Optional padding on each border - of the image. Default is 0, i.e no padding. If a sequence of length - 4 is provided, it is used to pad left, top, right, bottom borders - respectively. - """ - - def __init__(self, size, padding=0): - if isinstance(size, numbers.Number): - self.size = (int(size), int(size)) - else: - self.size = size - self.padding = padding - - @staticmethod - def get_params(img, output_size): - """Get parameters for ``crop`` for a random crop. - - Args: - img (PIL Image): Image to be cropped. - output_size (tuple): Expected output size of the crop. - - Returns: - tuple: params (i, j, h, w) to be passed to ``crop`` for random crop. - """ - w, h = img.size - th, tw = output_size - if w == tw and h == th: - return 0, 0, h, w - - i = random.randint(0, h - th) - j = random.randint(0, w - tw) - return i, j, th, tw - - def __call__(self, inputs): - """ - Args: - img (PIL Image): Image to be cropped. - - Returns: - PIL Image: Cropped image. - """ - if self.padding > 0: - inputs = CustomFunc(inputs, F.pad, self.padding) - - i, j, h, w = self.get_params(inputs[0], self.size) - return CustomFunc(inputs, F.crop, i, j, h, w) - - -class CenterCrop(object): - """Crop the given PIL Image at a random location. - - Args: - size (sequence or int): Desired output size of the crop. If size is an - int instead of sequence like (h, w), a square crop (size, size) is - made. - padding (int or sequence, optional): Optional padding on each border - of the image. Default is 0, i.e no padding. If a sequence of length - 4 is provided, it is used to pad left, top, right, bottom borders - respectively. - """ - - def __init__(self, size, padding=0): - if isinstance(size, numbers.Number): - self.size = (int(size), int(size)) - else: - self.size = size - self.padding = padding - - @staticmethod - def get_params(img, output_size): - """Get parameters for ``crop`` for a random crop. - - Args: - img (PIL Image): Image to be cropped. - output_size (tuple): Expected output size of the crop. - - Returns: - tuple: params (i, j, h, w) to be passed to ``crop`` for random crop. - """ - w, h = img.size - th, tw = output_size - if w == tw and h == th: - return 0, 0, h, w - - i = (h - th) // 2 - j = (w - tw) // 2 - return i, j, th, tw - - def __call__(self, inputs): - """ - Args: - img (PIL Image): Image to be cropped. - - Returns: - PIL Image: Cropped image. - """ - if self.padding > 0: - inputs = CustomFunc(inputs, F.pad, self.padding) - - i, j, h, w = self.get_params(inputs[0], self.size) - return CustomFunc(inputs, F.crop, i, j, h, w) - - -class RandomHorizontalFlip(object): - """Horizontally flip the given PIL Image randomly with a probability of 0.5.""" - - def __call__(self, inputs): - """ - Args: - img (PIL Image): Image to be flipped. - - Returns: - PIL Image: Randomly flipped image. - """ - - if random.random() < 0.5: - return CustomFunc(inputs, F.hflip) - return inputs - - -class RGB2Lab(object): - def __call__(self, inputs): - """ - Args: - img (PIL Image): Image to be flipped. - - Returns: - PIL Image: Randomly flipped image. - """ - - def __call__(self, inputs): - image_lab = color.rgb2lab(inputs[0]) - warp_ba_lab = color.rgb2lab(inputs[2]) - warp_aba_lab = color.rgb2lab(inputs[3]) - im_gbl_lab = color.rgb2lab(inputs[4]) - - inputs[0] = image_lab[:, :, :1] # l channel - inputs[1] = image_lab[:, :, 1:] # ab channel - inputs[2] = warp_ba_lab # lab channel - inputs[3] = warp_aba_lab # lab channel - inputs[4] = im_gbl_lab[:, :, 1:] # ab channel - - return inputs diff --git a/lib/flowlib.py b/lib/flowlib.py deleted file mode 100644 index f982829..0000000 --- a/lib/flowlib.py +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/python -""" -# ============================== -# flowlib.py -# library for optical flow processing -# Author: Ruoteng Li -# Date: 6th Aug 2016 -# ============================== -""" -import matplotlib.colors as cl -import matplotlib.pyplot as plt -import numpy as np -import png -from PIL import Image - -UNKNOWN_FLOW_THRESH = 1e7 -SMALLFLOW = 0.0 -LARGEFLOW = 1e8 - -""" -============= -Flow Section -============= -""" - - -def show_flow(filename): - """ - visualize optical flow map using matplotlib - :param filename: optical flow file - :return: None - """ - flow = read_flow(filename) - img = flow_to_image(flow) - plt.imshow(img) - plt.show() - - -def visualize_flow(flow, mode="Y"): - """ - this function visualize the input flow - :param flow: input flow in array - :param mode: choose which color mode to visualize the flow (Y: Ccbcr, RGB: RGB color) - :return: None - """ - if mode == "Y": - # Ccbcr color wheel - img = flow_to_image(flow) - plt.imshow(img) - plt.show() - elif mode == "RGB": - (h, w) = flow.shape[0:2] - du = flow[:, :, 0] - dv = flow[:, :, 1] - valid = flow[:, :, 2] - max_flow = max(np.max(du), np.max(dv)) - img = np.zeros((h, w, 3), dtype=np.float64) - # angle layer - img[:, :, 0] = np.arctan2(dv, du) / (2 * np.pi) - # magnitude layer, normalized to 1 - img[:, :, 1] = np.sqrt(du * du + dv * dv) * 8 / max_flow - # phase layer - img[:, :, 2] = 8 - img[:, :, 1] - # clip to [0,1] - small_idx = img[:, :, 0:3] < 0 - large_idx = img[:, :, 0:3] > 1 - img[small_idx] = 0 - img[large_idx] = 1 - # convert to rgb - img = cl.hsv_to_rgb(img) - # remove invalid point - img[:, :, 0] = img[:, :, 0] * valid - img[:, :, 1] = img[:, :, 1] * valid - img[:, :, 2] = img[:, :, 2] * valid - # show - plt.imshow(img) - plt.show() - - return None - - -def read_flow(filename): - """ - read optical flow from Middlebury .flo file - :param filename: name of the flow file - :return: optical flow data in matrix - """ - f = open(filename, "rb") - try: - magic = np.fromfile(f, np.float32, count=1)[0] # For Python3.x - except: - magic = np.fromfile(f, np.float32, count=1) # For Python2.x - data2d = None - - if 202021.25 != magic: - print("Magic number incorrect. Invalid .flo file") - else: - w = np.fromfile(f, np.int32, count=1)[0] - h = np.fromfile(f, np.int32, count=1)[0] - # print("Reading %d x %d flo file" % (h, w)) - data2d = np.fromfile(f, np.float32, count=2 * w * h) - # reshape data into 3D array (columns, rows, channels) - data2d = np.resize(data2d, (h, w, 2)) - f.close() - return data2d - - -def read_flow_png(flow_file): - """ - Read optical flow from KITTI .png file - :param flow_file: name of the flow file - :return: optical flow data in matrix - """ - flow_object = png.Reader(filename=flow_file) - flow_direct = flow_object.asDirect() - flow_data = list(flow_direct[2]) - (w, h) = flow_direct[3]["size"] - flow = np.zeros((h, w, 3), dtype=np.float64) - for i in range(len(flow_data)): - flow[i, :, 0] = flow_data[i][0::3] - flow[i, :, 1] = flow_data[i][1::3] - flow[i, :, 2] = flow_data[i][2::3] - - invalid_idx = flow[:, :, 2] == 0 - flow[:, :, 0:2] = (flow[:, :, 0:2] - 2 ** 15) / 64.0 - flow[invalid_idx, 0] = 0 - flow[invalid_idx, 1] = 0 - return flow - - -def write_flow(flow, filename): - """ - write optical flow in Middlebury .flo format - :param flow: optical flow map - :param filename: optical flow file path to be saved - :return: None - """ - f = open(filename, "wb") - magic = np.array([202021.25], dtype=np.float32) - (height, width) = flow.shape[0:2] - w = np.array([width], dtype=np.int32) - h = np.array([height], dtype=np.int32) - magic.tofile(f) - w.tofile(f) - h.tofile(f) - flow.tofile(f) - f.close() - - -def segment_flow(flow): - h = flow.shape[0] - w = flow.shape[1] - u = flow[:, :, 0] - v = flow[:, :, 1] - - idx = (abs(u) > LARGEFLOW) | (abs(v) > LARGEFLOW) - idx2 = abs(u) == SMALLFLOW - class0 = (v == 0) & (u == 0) - u[idx2] = 0.00001 - tan_value = v / u - - class1 = (tan_value < 1) & (tan_value >= 0) & (u > 0) & (v >= 0) - class2 = (tan_value >= 1) & (u >= 0) & (v >= 0) - class3 = (tan_value < -1) & (u <= 0) & (v >= 0) - class4 = (tan_value < 0) & (tan_value >= -1) & (u < 0) & (v >= 0) - class8 = (tan_value >= -1) & (tan_value < 0) & (u > 0) & (v <= 0) - class7 = (tan_value < -1) & (u >= 0) & (v <= 0) - class6 = (tan_value >= 1) & (u <= 0) & (v <= 0) - class5 = (tan_value >= 0) & (tan_value < 1) & (u < 0) & (v <= 0) - - seg = np.zeros((h, w)) - - seg[class1] = 1 - seg[class2] = 2 - seg[class3] = 3 - seg[class4] = 4 - seg[class5] = 5 - seg[class6] = 6 - seg[class7] = 7 - seg[class8] = 8 - seg[class0] = 0 - seg[idx] = 0 - - return seg - - -def flow_error(tu, tv, u, v): - """ - Calculate average end point error - :param tu: ground-truth horizontal flow map - :param tv: ground-truth vertical flow map - :param u: estimated horizontal flow map - :param v: estimated vertical flow map - :return: End point error of the estimated flow - """ - smallflow = 0.0 - """ - stu = tu[bord+1:end-bord,bord+1:end-bord] - stv = tv[bord+1:end-bord,bord+1:end-bord] - su = u[bord+1:end-bord,bord+1:end-bord] - sv = v[bord+1:end-bord,bord+1:end-bord] - """ - stu = tu[:] - stv = tv[:] - su = u[:] - sv = v[:] - - idxUnknow = (abs(stu) > UNKNOWN_FLOW_THRESH) | (abs(stv) > UNKNOWN_FLOW_THRESH) - stu[idxUnknow] = 0 - stv[idxUnknow] = 0 - su[idxUnknow] = 0 - sv[idxUnknow] = 0 - - ind2 = [(np.absolute(stu) > smallflow) | (np.absolute(stv) > smallflow)] - index_su = su[ind2] - index_sv = sv[ind2] - an = 1.0 / np.sqrt(index_su ** 2 + index_sv ** 2 + 1) - un = index_su * an - vn = index_sv * an - - index_stu = stu[ind2] - index_stv = stv[ind2] - tn = 1.0 / np.sqrt(index_stu ** 2 + index_stv ** 2 + 1) - tun = index_stu * tn - tvn = index_stv * tn - - """ - angle = un * tun + vn * tvn + (an * tn) - index = [angle == 1.0] - angle[index] = 0.999 - ang = np.arccos(angle) - mang = np.mean(ang) - mang = mang * 180 / np.pi - """ - - epe = np.sqrt((stu - su) ** 2 + (stv - sv) ** 2) - epe = epe[ind2] - mepe = np.mean(epe) - return mepe - - -def flow_to_image(flow, display=False): - """ - Convert flow into middlebury color code image - :param flow: optical flow map - :return: optical flow image in middlebury color - """ - u = flow[:, :, 0] - v = flow[:, :, 1] - - maxu = -999.0 - maxv = -999.0 - minu = 999.0 - minv = 999.0 - - idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH) - u[idxUnknow] = 0 - v[idxUnknow] = 0 - - maxu = max(maxu, np.max(u)) - minu = min(minu, np.min(u)) - - maxv = max(maxv, np.max(v)) - minv = min(minv, np.min(v)) - - rad = np.sqrt(u ** 2 + v ** 2) - maxrad = max(-1, np.max(rad)) - - if display: - print("max flow: %.4f\nflow range:\nu = %.3f .. %.3f\nv = %.3f .. %.3f" % (maxrad, minu, maxu, minv, maxv)) - - u = u / (maxrad + np.finfo(float).eps) - v = v / (maxrad + np.finfo(float).eps) - - img = compute_color(u, v) - - idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2) - img[idx] = 0 - - return np.uint8(img) - - -def evaluate_flow_file(gt, pred): - """ - evaluate the estimated optical flow end point error according to ground truth provided - :param gt: ground truth file path - :param pred: estimated optical flow file path - :return: end point error, float32 - """ - # Read flow files and calculate the errors - gt_flow = read_flow(gt) # ground truth flow - eva_flow = read_flow(pred) # predicted flow - # Calculate errors - average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], eva_flow[:, :, 0], eva_flow[:, :, 1]) - return average_pe - - -def evaluate_flow(gt_flow, pred_flow): - """ - gt: ground-truth flow - pred: estimated flow - """ - average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], pred_flow[:, :, 0], pred_flow[:, :, 1]) - return average_pe - - -""" -============== -Disparity Section -============== -""" - - -def read_disp_png(file_name): - """ - Read optical flow from KITTI .png file - :param file_name: name of the flow file - :return: optical flow data in matrix - """ - image_object = png.Reader(filename=file_name) - image_direct = image_object.asDirect() - image_data = list(image_direct[2]) - (w, h) = image_direct[3]["size"] - channel = len(image_data[0]) / w - flow = np.zeros((h, w, channel), dtype=np.uint16) - for i in range(len(image_data)): - for j in range(channel): - flow[i, :, j] = image_data[i][j::channel] - return flow[:, :, 0] / 256 - - -def disp_to_flowfile(disp, filename): - """ - Read KITTI disparity file in png format - :param disp: disparity matrix - :param filename: the flow file name to save - :return: None - """ - f = open(filename, "wb") - magic = np.array([202021.25], dtype=np.float32) - (height, width) = disp.shape[0:2] - w = np.array([width], dtype=np.int32) - h = np.array([height], dtype=np.int32) - empty_map = np.zeros((height, width), dtype=np.float32) - data = np.dstack((disp, empty_map)) - magic.tofile(f) - w.tofile(f) - h.tofile(f) - data.tofile(f) - f.close() - - -""" -============== -Image Section -============== -""" - - -def read_image(filename): - """ - Read normal image of any format - :param filename: name of the image file - :return: image data in matrix uint8 type - """ - img = Image.open(filename) - im = np.array(img) - return im - - -def warp_image(im, flow): - """ - Use optical flow to warp image to the next - :param im: image to warp - :param flow: optical flow - :return: warped image - """ - from scipy import interpolate - - image_height = im.shape[0] - image_width = im.shape[1] - flow_height = flow.shape[0] - flow_width = flow.shape[1] - n = image_height * image_width - (iy, ix) = np.float32(np.mgrid[0:image_height, 0:image_width]) - (fy, fx) = np.float32(np.mgrid[0:flow_height, 0:flow_width]) - fx += flow[:, :, 0].astype(np.float) - fy += flow[:, :, 1] - mask = np.logical_or(fx < 0, fx > flow_width) - mask = np.logical_or(mask, fy < 0) - mask = np.logical_or(mask, fy > flow_height) - fx = np.minimum(np.maximum(fx, 0), flow_width) - fy = np.minimum(np.maximum(fy, 0), flow_height) - points = np.concatenate((ix.reshape(n, 1), iy.reshape(n, 1)), axis=1) - xi = np.concatenate((fx.reshape(n, 1), fy.reshape(n, 1)), axis=1) - warp = np.zeros((image_height, image_width, im.shape[2])) - for i in range(im.shape[2]): - channel = im[:, :, i] - # plt.imshow(channel, cmap='gray') - values = channel.reshape(n, 1) - new_channel = interpolate.griddata(points, values, xi, method="cubic") - new_channel = np.reshape(new_channel, [flow_height, flow_width]) - new_channel[mask] = 1 - warp[:, :, i] = new_channel.astype(np.uint8) - - return warp.astype(np.uint8) - - -""" -============== -Others -============== -""" - - -def scale_image(image, new_range): - """ - Linearly scale the image into desired range - :param image: input image - :param new_range: the new range to be aligned - :return: image normalized in new range - """ - min_val = np.min(image).astype(np.float32) - max_val = np.max(image).astype(np.float32) - min_val_new = np.array(min(new_range), dtype=np.float32) - max_val_new = np.array(max(new_range), dtype=np.float32) - scaled_image = (image - min_val) / (max_val - min_val) * (max_val_new - min_val_new) + min_val_new - return scaled_image.astype(np.uint8) - - -def compute_color(u, v): - """ - compute optical flow color map - :param u: optical flow horizontal map - :param v: optical flow vertical map - :return: optical flow in color code - """ - [h, w] = u.shape - img = np.zeros([h, w, 3]) - nanIdx = np.isnan(u) | np.isnan(v) - u[nanIdx] = 0 - v[nanIdx] = 0 - - colorwheel = make_color_wheel() - ncols = np.size(colorwheel, 0) - - rad = np.sqrt(u ** 2 + v ** 2) - - a = np.arctan2(-v, -u) / np.pi - - fk = (a + 1) / 2 * (ncols - 1) + 1 - - k0 = np.floor(fk).astype(int) - - k1 = k0 + 1 - k1[k1 == ncols + 1] = 1 - f = fk - k0 - - for i in range(0, np.size(colorwheel, 1)): - tmp = colorwheel[:, i] - col0 = tmp[k0 - 1] / 255 - col1 = tmp[k1 - 1] / 255 - col = (1 - f) * col0 + f * col1 - - idx = rad <= 1 - col[idx] = 1 - rad[idx] * (1 - col[idx]) - notidx = np.logical_not(idx) - - col[notidx] *= 0.75 - img[:, :, i] = np.uint8(np.floor(255 * col * (1 - nanIdx))) - - return img - - -def make_color_wheel(): - """ - Generate color wheel according Middlebury color code - :return: Color wheel - """ - RY = 15 - YG = 6 - GC = 4 - CB = 11 - BM = 13 - MR = 6 - - ncols = RY + YG + GC + CB + BM + MR - - colorwheel = np.zeros([ncols, 3]) - - col = 0 - - # RY - colorwheel[0:RY, 0] = 255 - colorwheel[0:RY, 1] = np.transpose(np.floor(255 * np.arange(0, RY) / RY)) - col += RY - - # YG - colorwheel[col : col + YG, 0] = 255 - np.transpose(np.floor(255 * np.arange(0, YG) / YG)) - colorwheel[col : col + YG, 1] = 255 - col += YG - - # GC - colorwheel[col : col + GC, 1] = 255 - colorwheel[col : col + GC, 2] = np.transpose(np.floor(255 * np.arange(0, GC) / GC)) - col += GC - - # CB - colorwheel[col : col + CB, 1] = 255 - np.transpose(np.floor(255 * np.arange(0, CB) / CB)) - colorwheel[col : col + CB, 2] = 255 - col += CB - - # BM - colorwheel[col : col + BM, 2] = 255 - colorwheel[col : col + BM, 0] = np.transpose(np.floor(255 * np.arange(0, BM) / BM)) - col += +BM - - # MR - colorwheel[col : col + MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR)) - colorwheel[col : col + MR, 0] = 255 - - return colorwheel diff --git a/lib/functional.py b/lib/functional.py index f36181d..0b276ee 100755 --- a/lib/functional.py +++ b/lib/functional.py @@ -208,24 +208,23 @@ def resize(img, size, interpolation=Image.BILINEAR): """ if not _is_pil_image(img): raise TypeError("img should be PIL Image. Got {}".format(type(img))) - if not (isinstance(size, int) or (isinstance(size, collections.Iterable) and len(size) == 2)): + if not isinstance(size, int) and (not isinstance(size, collections.Iterable) or len(size) != 2): raise TypeError("Got inappropriate size arg: {}".format(size)) - if isinstance(size, int): - w, h = img.size - if (w <= h and w == size) or (h <= w and h == size): - return img - if w < h: - ow = size - oh = int(round(size * h / w)) - return img.resize((ow, oh), interpolation) - else: - oh = size - ow = int(round(size * w / h)) - return img.resize((ow, oh), interpolation) - else: + if not isinstance(size, int): return img.resize(size[::-1], interpolation) + w, h = img.size + if (w <= h and w == size) or (h <= w and h == size): + return img + if w < h: + ow = size + oh = int(round(size * h / w)) + else: + oh = size + ow = int(round(size * w / h)) + return img.resize((ow, oh), interpolation) + def scale(*args, **kwargs): warnings.warn("The use of the transforms.Scale transform is deprecated, " + "please use transforms.Resize instead.") diff --git a/lib/imageloader_imagenet.py b/lib/imageloader_imagenet.py deleted file mode 100644 index c4bef26..0000000 --- a/lib/imageloader_imagenet.py +++ /dev/null @@ -1,179 +0,0 @@ -import os.path as osp - -import cv2 -import matplotlib.pyplot as plt -import numpy as np -import torch -import torch.utils.data as data -import torchvision.transforms as transforms -from PIL import Image -from scipy.ndimage.filters import gaussian_filter -from utils.util_distortion import CenterPadCrop_numpy, Distortion_with_flow, RGB2Lab, ToTensor, random_mask - -import lib.functional as F -from lib.RefPairDatasetWithGlobalOrigRGB_optimize import ToTensor, combo5_loader, image_loader, parse_images - -cv2.setNumThreads(0) - - -class ImageNetDataset(data.Dataset): - def __init__( - self, - data_root, - epoch, - image_size, - with_bad=False, - with_mid=False, - transforms_imagenet=None, - distortion_level=3, - brightnessjitter=5, - extra_reference_transform=None, - real_reference_probability=1, - mask_reference=False, - no_reference_probability=0, - ): - image_pairs = [] - curr_image_pairs = parse_images(data_root, with_bad, with_mid) - image_pairs += curr_image_pairs - print("##### parsing image_a pairs in %s: %d pairs #####" % (data_root, len(curr_image_pairs))) - if len(image_pairs) == 0: - raise RuntimeError("Found 0 image_a pairs in all the data_roots") - - self.image_pairs = image_pairs - self.epoch = epoch - self.image_size = image_size - self.real_len = len(self.image_pairs) - self.image_pairs *= epoch - - self.real_reference_probability = real_reference_probability - self.no_reference_probability = no_reference_probability - self.mask_reference = mask_reference - - self.transforms_imagenet_raw = transforms_imagenet - self.transforms_imagenet = transforms.Compose(transforms_imagenet) - self.extra_reference_transform = transforms.Compose(extra_reference_transform) - self.flow_transform = transforms.Compose([CenterPadCrop_numpy(self.image_size), ToTensor()]) - - self.distortion_level = distortion_level - self.distortion_transform = Distortion_with_flow() - self.brightnessjitter = brightnessjitter - - def __getitem__(self, index): - try: - # print('imagenet loader') - - image_id = 0 - pair_id = index - - combo_path = None - image_a_path = None - image_b_path = None - - image_names = ["", ""] - dir_root, cls_dir, image_names[0], image_names[1], is_good = self.image_pairs[pair_id] - sub_dir = osp.join(dir_root, cls_dir) - if is_good >= 1: - image_a_path = osp.join(sub_dir, "input", "%s.JPEG" % image_names[image_id]) - image_b_path = osp.join(sub_dir, "input", "%s.JPEG" % image_names[1 - image_id]) - elif is_good == 0: - image_a_path = osp.join(sub_dir, "input_mid", "%s.JPEG" % image_names[image_id]) - image_b_path = osp.join(sub_dir, "input_mid", "%s.JPEG" % image_names[1 - image_id]) - else: - image_a_path = osp.join(sub_dir, "input_bad", "%s.JPEG" % image_names[image_id]) - image_b_path = osp.join(sub_dir, "input_bad", "%s.JPEG" % image_names[1 - image_id]) - - if np.random.random() > 0.5: - image_a_path, image_b_path = image_b_path, image_a_path - - I1 = image_loader(image_a_path) - if np.random.random() < self.no_reference_probability: - no_reference_flag = 1 - I_reference_fake = Image.fromarray(np.array(I1) * 0) - I_reference_real = I_reference_fake - else: - no_reference_flag = 0 - I_reference_fake = I1 - I_reference_real = image_loader(image_b_path) - - ## generate the flow - height, width = np.array(I1).shape[0], np.array(I1).shape[1] - alpha = np.random.rand() * self.distortion_level - distortion_range = 50 - random_state = np.random.RandomState(None) - shape = self.image_size[0], self.image_size[1] - # dx: flow on the vertical direction; dy: flow on the horizontal direction - forward_dx = ( - gaussian_filter((random_state.rand(*shape) * 2 - 1), distortion_range, mode="constant", cval=0) - * alpha - * 1000 - ) - forward_dy = ( - gaussian_filter((random_state.rand(*shape) * 2 - 1), distortion_range, mode="constant", cval=0) - * alpha - * 1000 - ) - - # transform the input image for colorization - I1 = self.transforms_imagenet(I1) - - # DEBUG - # from utils.util import batch_lab2rgb_transpose_mc - # plt.imshow(image_loader(image_a_path)) - # plt.figure() - # I1 = I1.unsqueeze(0) - # plt.imshow(batch_lab2rgb_transpose_mc(I1[0:32, 0:1, :, :], I1[0:32, 1:3, :, :])) - - # transform the fake reference - I_reference_fake = self.extra_reference_transform(I_reference_fake) - for transform in self.transforms_imagenet_raw: - if type(transform) is RGB2Lab and self.extra_reference_transform is not None: - I_reference_fake = self.distortion_transform(I_reference_fake, forward_dx, forward_dy) - I_reference_fake = transform(I_reference_fake) - I_reference_fake[0:1, :, :] = I_reference_fake[0:1, :, :] + torch.randn(1) * self.brightnessjitter - - # transform the real reference - I_reference_real = self.transforms_imagenet(I_reference_real) - - # generate the masked image - if self.mask_reference: - mask = torch.Tensor(random_mask(self.image_size[0], self.image_size[1])) - I_reference_fake[1:2, :, :] = ( - I_reference_fake[1:2, :, :] * (1 - mask) + I_reference_real[1:2, :, :] * mask - ) - I_reference_fake[2:3, :, :] = ( - I_reference_fake[2:3, :, :] * (1 - mask) + I_reference_real[2:3, :, :] * mask - ) - - # choose the reference - if no_reference_flag: - I_reference_output, fake_refernce_flag, no_reference_flag = ( - I_reference_real, - torch.Tensor([0]), - torch.Tensor([1]).unsqueeze(-1).unsqueeze(-1), - ) - else: - no_reference_flag = torch.Tensor([0]).unsqueeze(-1).unsqueeze(-1) - I_reference_output, fake_refernce_flag = ( - (I_reference_real, torch.Tensor([0])) - if np.random.random() < self.real_reference_probability - else (I_reference_fake, torch.Tensor([1])) - ) - - outputs = [] - outputs.append(I1) - outputs.append(I_reference_output) - outputs.append(fake_refernce_flag) - outputs.append(no_reference_flag) - - except Exception as e: - if combo_path is not None: - print("problem in ", combo_path) - print("problem in, ", image_a_path) - print(e) - return self.__getitem__(np.random.randint(0, len(self.image_pairs))) - - return outputs - - def __len__(self): - return len(self.image_pairs) - # return len(self.image_pairs) * 2 diff --git a/lib/imports.py b/lib/imports.py deleted file mode 100755 index 2c49d6d..0000000 --- a/lib/imports.py +++ /dev/null @@ -1,82 +0,0 @@ -import collections -import datetime -import itertools -import json -import math -import os -import pickle -import random -import re -import string -import sys -import threading -import time -import warnings -from abc import abstractmethod -from collections import Counter, Iterable, OrderedDict -from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor -from functools import partial -from glob import glob, iglob -from itertools import chain -from operator import attrgetter, itemgetter - -import bcolz -import cv2 -import graphviz -import IPython -import matplotlib -import numpy as np -import pandas as pd -import PIL -import scipy -import seaborn as sns -import sklearn -import sklearn_pandas -from IPython.lib.deepreload import reload as dreload -from IPython.lib.display import FileLink -from ipywidgets import fixed, interact, interactive, widgets -from isoweek import Week -from matplotlib import animation -from matplotlib import pyplot as plt -from matplotlib import rcParams -from pandas_summary import DataFrameSummary -from PIL import Image, ImageEnhance, ImageOps -from sklearn import ensemble, metrics, preprocessing - -matplotlib.rc("animation", html="html5") -np.set_printoptions(precision=5, linewidth=110, suppress=True) - - -def in_notebook(): - return "ipykernel" in sys.modules - - -import tqdm as tq -from tqdm import tnrange, tqdm_notebook - - -def clear_tqdm(): - inst = getattr(tq.tqdm, "_instances", None) - if not inst: - return - for i in range(len(inst)): - inst.pop().close() - - -if in_notebook(): - - def tqdm(*args, **kwargs): - clear_tqdm() - return tq.tqdm(*args, file=sys.stdout, **kwargs) - - def trange(*args, **kwargs): - clear_tqdm() - return tq.trange(*args, file=sys.stdout, **kwargs) - - -else: - from tqdm import tqdm, trange - - tnrange = trange - tqdm_notebook = tqdm - diff --git a/lib/videoloader.py b/lib/videoloader.py deleted file mode 100644 index 0d281d0..0000000 --- a/lib/videoloader.py +++ /dev/null @@ -1,390 +0,0 @@ -import sys - -sys.path.insert(0, "..") -import os -import struct - -import cv2 -import numpy as np -import torch.utils.data as data -from PIL import Image - -cv2.setNumThreads(0) -import pdb -import random - -import matplotlib.pyplot as plt -import torch -import torchvision -import torchvision.transforms as transforms -from PIL import Image -from skimage import color -from utils.flowlib import read_flow -from utils.util_distortion import CenterPad, CenterPadCrop_numpy -from utils.warping import WarpingLayer - -import lib.functional as F - - -class RGB2Lab(object): - def __init__(self): - pass - - def __call__(self, inputs): - image_lab = color.rgb2lab(inputs) - return image_lab - - -class Normalize(object): - def __init__(self): - pass - - def __call__(self, inputs): - inputs[0:1, :, :] = F.normalize(inputs[0:1, :, :], 50, 1) - inputs[1:3, :, :] = F.normalize(inputs[1:3, :, :], (0, 0), (1, 1)) - return inputs - - -class ToTensor(object): - def __init__(self): - pass - - def __call__(self, inputs): - outputs = F.to_mytensor(inputs) # permute channel and transform to tensor - return outputs - - -class RandomErasing(object): - def __init__(self, probability=0.6, sl=0.05, sh=0.6): - self.probability = probability - self.sl = sl - self.sh = sh - - def __call__(self, img): - img = np.array(img) - if random.uniform(0, 1) > self.probability: - return Image.fromarray(img) - - area = img.shape[0] * img.shape[1] - h0 = img.shape[0] - w0 = img.shape[1] - channel = img.shape[2] - - h = int(round(random.uniform(self.sl, self.sh) * h0)) - w = int(round(random.uniform(self.sl, self.sh) * w0)) - - if w < img.shape[1] and h < img.shape[0]: - x1 = random.randint(0, img.shape[0] - h) - y1 = random.randint(0, img.shape[1] - w) - img[x1 : x1 + h, y1 : y1 + w, :] = np.random.rand(h, w, channel) * 255 - return Image.fromarray(img) - - return Image.fromarray(img) - - -class CenterCrop(object): - """ - center crop the numpy array - """ - - def __init__(self, image_size): - # target size - self.h0, self.w0 = image_size - - def __call__(self, input_numpy): - if input_numpy.ndim == 3: - h, w, channel = input_numpy.shape - output_numpy = np.zeros((self.h0, self.w0, channel)) - output_numpy = input_numpy[ - (h - self.h0) // 2 : (h - self.h0) // 2 + self.h0, (w - self.w0) // 2 : (w - self.w0) // 2 + self.w0, : - ] - else: - h, w = input_numpy.shape - output_numpy = np.zeros((self.h0, self.w0)) - output_numpy = input_numpy[ - (h - self.h0) // 2 : (h - self.h0) // 2 + self.h0, (w - self.w0) // 2 : (w - self.w0) // 2 + self.w0 - ] - return output_numpy - - -def parse_images(data_root): - image_pairs = [] - subdirs = sorted(os.listdir(data_root)) - for subdir in subdirs: - path = os.path.join(data_root, subdir) - if not os.path.isdir(path): - continue - - # parse_file = os.path.join(path, 'pairs_output.txt') - parse_file = os.path.join(path, "pairs_output_new.txt") - if os.path.exists(parse_file): - f = open(parse_file, "r") - lines = f.readlines() - for line in lines: - line = line.replace("\n", "") - ( - image1_name, - image2_name, - reference_video_name, - reference_video_name1, - reference_name1, - reference_name2, - reference_name3, - reference_name4, - reference_name5, - reference_gt1, - reference_gt2, - reference_gt3, - ) = line.split() - image1_name = image1_name.split(".")[0] - image2_name = image2_name.split(".")[0] - reference_video_name = reference_video_name.split(".")[0] - reference_video_name1 = reference_video_name1.split(".")[0] - reference_name1 = reference_name1.split(".")[0] - reference_name2 = reference_name2.split(".")[0] - reference_name3 = reference_name3.split(".")[0] - reference_name4 = reference_name4.split(".")[0] - reference_name5 = reference_name5.split(".")[0] - - reference_gt1 = reference_gt1.split(".")[0] - reference_gt2 = reference_gt2.split(".")[0] - reference_gt3 = reference_gt3.split(".")[0] - - flow_forward_name = image1_name + "_forward" - flow_backward_name = image1_name + "_backward" - mask_name = image1_name + "_mask" - - item = ( - image1_name + ".jpg", - image2_name + ".jpg", - reference_video_name + ".jpg", - reference_name1 + ".JPEG", - reference_name2 + ".JPEG", - reference_name3 + ".JPEG", - reference_name4 + ".JPEG", - reference_name5 + ".JPEG", - flow_forward_name + ".flo", - flow_backward_name + ".flo", - mask_name + ".pgm", - reference_gt1 + ".jpg", - reference_gt2 + ".jpg", - reference_gt3 + ".jpg", - path, - ) - image_pairs.append(item) - - f.close() - - else: - raise (RuntimeError("Error when parsing pair_output_count.txt in subfolders of: " + path + "\n")) - - return image_pairs - - -class VideosDataset(torch.utils.data.Dataset): - def __init__( - self, - data_root, - epoch, - image_size, - image_transform=None, - use_google_reference=False, - real_reference_probability=1, - nonzero_placeholder_probability=0.5, - ): - self.data_root = data_root - self.image_transform = image_transform - self.CenterPad = CenterPad(image_size) - self.ToTensor = ToTensor() - self.CenterCrop = CenterCrop(image_size) - - assert len(self.data_root) > 0, "find no dataroot" - self.epoch = epoch - self.image_pairs = parse_images(self.data_root) - self.real_len = len(self.image_pairs) - print("##### parsing image pairs in %s: %d pairs #####" % (data_root, self.real_len)) - self.image_pairs *= epoch - self.use_google_reference = use_google_reference - self.real_reference_probability = real_reference_probability - self.nonzero_placeholder_probability = nonzero_placeholder_probability - - def __getitem__(self, index): - ( - image1_name, - image2_name, - reference_video_name, - reference_name1, - reference_name2, - reference_name3, - reference_name4, - reference_name5, - flow_forward_name, - flow_backward_name, - mask_name, - reference_gt1, - reference_gt2, - reference_gt3, - path, - ) = self.image_pairs[index] - try: - # print('video loader') - I1 = Image.open(os.path.join(path, "input_pad", image1_name)) - I2 = Image.open(os.path.join(path, "input_pad", image2_name)) - - # randomly choose a reference frame from the video - I_reference_video = Image.open( - os.path.join(path, "reference_gt", random.choice([reference_gt1, reference_gt2, reference_gt3])) - ) - I_reference_video_real = Image.open( - os.path.join( - path, - "reference", - random.choice( - [reference_name1, reference_name2, reference_name3, reference_name4, reference_name5] - ), - ) - ) - - flow_forward = read_flow(os.path.join(path, "flow", flow_forward_name)) # numpy - flow_backward = read_flow(os.path.join(path, "flow", flow_backward_name)) # numpy - mask = Image.open(os.path.join(path, "mask", mask_name)) - - # binary mask - mask = np.array(mask) - mask[mask < 240] = 0 - mask[mask >= 240] = 1 - - # transform - I1 = self.image_transform(I1) - I2 = self.image_transform(I2) - I_reference_video = self.image_transform(self.CenterPad(I_reference_video)) - I_reference_video_real = self.image_transform(self.CenterPad(I_reference_video_real)) - flow_forward = self.ToTensor(self.CenterCrop(flow_forward)) - flow_backward = self.ToTensor(self.CenterCrop(flow_backward)) - mask = self.ToTensor(self.CenterCrop(mask)) - - if np.random.random() < self.real_reference_probability: - I_reference_output = I_reference_video_real - placeholder = torch.zeros_like(I1) - self_ref_flag = torch.zeros_like(I1) - else: - I_reference_output = I_reference_video - placeholder = I2 if np.random.random() < self.nonzero_placeholder_probability else torch.zeros_like(I1) - self_ref_flag = torch.ones_like(I1) - - outputs = [] - outputs.append(I1) - outputs.append(I2) - outputs.append(I_reference_output) - # outputs.append(I_reference_video_real) - outputs.append(flow_forward) - outputs.append(flow_backward) - outputs.append(mask) - outputs.append(placeholder) - outputs.append(self_ref_flag) - - except Exception as e: - print("problem in, ", path) - print(e) - return self.__getitem__(np.random.randint(0, len(self.image_pairs))) - - return outputs - - def __len__(self): - return len(self.image_pairs) - - -import torchvision.utils as vutils -from torch.autograd import Variable - - -def batch_lab2rgb_transpose_mc(img_l_mc, img_ab_mc, nrow=8): - if isinstance(img_l_mc, Variable): - img_l_mc = img_l_mc.data.cpu() - if isinstance(img_ab_mc, Variable): - img_ab_mc = img_ab_mc.data.cpu() - - if img_l_mc.is_cuda: - img_l_mc = img_l_mc.cpu() - if img_ab_mc.is_cuda: - img_ab_mc = img_ab_mc.cpu() - - assert img_l_mc.dim() == 4 and img_ab_mc.dim() == 4, "only for batch input" - - l_norm, ab_norm = 1.0, 1.0 - l_mean, ab_mean = 50.0, 0 - img_l = img_l_mc * l_norm + l_mean - img_ab = img_ab_mc * ab_norm + ab_mean - pred_lab = torch.cat((img_l, img_ab), dim=1) - grid_lab = vutils.make_grid(pred_lab, nrow=nrow).numpy().astype("float64") - grid_rgb = (np.clip(color.lab2rgb(grid_lab.transpose((1, 2, 0))), 0, 1) * 255).astype("uint8") - return grid_rgb - - -# if __name__ == "__main__": -def test_videoloader(): - # dataset initI1lization - # image size = 256 - train_dataset = VideosDataset(data_root="../../Dataset/video/video_pair", epoch=10) - - data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=1, shuffle=True, num_workers=0) - - iter_instance = iter(data_loader) - sample = iter_instance.next() - I1_lab, I2_lab, I_reference_lab, flow_forward, flow_backward, mask = sample - - # l: [-50,50], ab: [-128, 128] - l_norm, ab_norm = 1.0, 1.0 - l_mean, ab_mean = 50.0, 0 - I1_l = I1_lab[:, 0:1, :, :] - I1_ab = I1_lab[:, 1:3, :, :] - I2_l = I2_lab[:, 0:1, :, :] - I2_ab = I2_lab[:, 1:3, :, :] - I_reference_l = I_reference_lab[:, 0:1, :, :] - I_reference_ab = I_reference_lab[:, 1:3, :, :] - - # test image samples - image_A_grid = batch_lab2rgb_transpose_mc(I1_l, I1_ab, nrow=4).astype(np.uint8) - image_B_grid = batch_lab2rgb_transpose_mc(I2_l, I2_ab, nrow=4).astype(np.uint8) - image_ref_grid = batch_lab2rgb_transpose_mc(I_reference_l, I_reference_ab, nrow=4).astype(np.uint8) - plt.imsave("A.png", image_A_grid) - plt.imsave("B.png", image_B_grid) - plt.figure() - plt.imshow(image_A_grid) - plt.title("image A") - plt.figure() - plt.imshow(image_B_grid) - plt.title("image B") - plt.figure() - plt.imshow(image_ref_grid) - plt.title("image ref") - # plt.show() - - # test warping - print(I2_lab.shape) - print(flow_forward.shape) - warping_layer = WarpingLayer("gpu") - I21_lab = warping_layer(I2_lab.cuda(), flow_forward.cuda()) - I21_lab = I21_lab * mask.cuda() - - I21_l = I21_lab[:, 0:1, :, :] - I21_ab = I21_lab[:, 1:3, :, :] - image_warp_grid = batch_lab2rgb_transpose_mc(I21_l, I21_ab, nrow=4).astype(np.uint8) - plt.imsave("BA.png", image_warp_grid) - plt.figure() - plt.imshow(image_warp_grid) - plt.title("image warp") - # plt.show() - - # image difference - plt.figure() - plt.imshow(np.abs(image_warp_grid - image_A_grid).astype(np.uint8)) - plt.title("warp-A") - plt.figure() - plt.imshow(np.abs(image_B_grid - image_A_grid).astype(np.uint8)) - plt.title("B-A") - # plt.show() - plt.imsave("difference_warp.png", np.abs(image_warp_grid - image_A_grid).astype(np.uint8)) - plt.imsave("difference_BA.png", np.abs(image_B_grid - image_A_grid).astype(np.uint8)) - - return None diff --git a/lib/videoloader_imagenet.py b/lib/videoloader_imagenet.py deleted file mode 100644 index 12af761..0000000 --- a/lib/videoloader_imagenet.py +++ /dev/null @@ -1,203 +0,0 @@ -import os -import os.path as osp -import pdb -import struct -import time - -import cv2 -import matplotlib.pyplot as plt -import numpy as np -import torch -import torch.utils.data as data -import torchvision.transforms as transforms -from PIL import Image -from scipy.misc import imread, imresize, imsave -from scipy.ndimage import geometric_transform, map_coordinates -from scipy.ndimage.filters import gaussian_filter -from skimage import color -from utils.util_distortion import ( - CenterPad, - CenterPadCrop_numpy, - Distortion, - Distortion_with_flow, - Normalize, - RGB2Lab, - ToTensor, -) - -import lib.functional as F -from lib.RefPairDatasetWithGlobalOrigRGB_optimize import ToTensor, combo5_loader, image_loader, parse_images - -cv2.setNumThreads(0) - - -class VideosDataset_ImageNet(data.Dataset): - def __init__( - self, - data_root, - epoch, - image_size, - with_bad=False, - with_mid=False, - transforms_imagenet=None, - distortion_level=3, - brightnessjitter=0, - nonzero_placeholder_probability=0.5, - extra_reference_transform=None, - real_reference_probability=1, - ): - image_pairs = [] - curr_image_pairs = parse_images(data_root, with_bad, with_mid) - image_pairs += curr_image_pairs - print("##### parsing image_a pairs in %s: %d pairs #####" % (data_root, len(curr_image_pairs))) - if len(image_pairs) == 0: - raise RuntimeError("Found 0 image_a pairs in all the data_roots") - - self.image_pairs = image_pairs - self.transforms_imagenet_raw = transforms_imagenet - self.extra_reference_transform = transforms.Compose(extra_reference_transform) - self.real_reference_probability = real_reference_probability - self.transforms_imagenet = transforms.Compose(transforms_imagenet) - self.epoch = epoch - self.image_size = image_size - self.real_len = len(self.image_pairs) - self.image_pairs *= epoch - self.distortion_level = distortion_level - self.distortion_transform = Distortion_with_flow() - self.brightnessjitter = brightnessjitter - self.flow_transform = transforms.Compose([CenterPadCrop_numpy(self.image_size), ToTensor()]) - self.nonzero_placeholder_probability = nonzero_placeholder_probability - self.ToTensor = ToTensor() - self.Normalize = Normalize() - - def __getitem__(self, index): - try: - # print('imagenet loader') - - image_id = 0 - pair_id = index - - combo_path = None - image_a_path = None - image_b_path = None - - image_names = ["", ""] - dir_root, cls_dir, image_names[0], image_names[1], is_good = self.image_pairs[pair_id] - sub_dir = osp.join(dir_root, cls_dir) - if is_good >= 1: - image_a_path = osp.join(sub_dir, "input", "%s.JPEG" % image_names[image_id]) - image_b_path = osp.join(sub_dir, "input", "%s.JPEG" % image_names[1 - image_id]) - elif is_good == 0: - image_a_path = osp.join(sub_dir, "input_mid", "%s.JPEG" % image_names[image_id]) - image_b_path = osp.join(sub_dir, "input_mid", "%s.JPEG" % image_names[1 - image_id]) - else: - image_a_path = osp.join(sub_dir, "input_bad", "%s.JPEG" % image_names[image_id]) - image_b_path = osp.join(sub_dir, "input_bad", "%s.JPEG" % image_names[1 - image_id]) - - if np.random.random() > 0.5: - image_a_path, image_b_path = image_b_path, image_a_path - - I1 = image_loader(image_a_path) - I2 = I1 - I_reference_video = I1 - I_reference_video_real = image_loader(image_b_path) - - ## generate the flow - height, width = np.array(I2).shape[0], np.array(I2).shape[1] - alpha = np.random.rand() * self.distortion_level - distortion_range = 50 - random_state = np.random.RandomState(None) - shape = self.image_size[0], self.image_size[1] - # dx: flow on the vertical direction; dy: flow on the horizontal direction - forward_dx = ( - gaussian_filter((random_state.rand(*shape) * 2 - 1), distortion_range, mode="constant", cval=0) - * alpha - * 1000 - ) - forward_dy = ( - gaussian_filter((random_state.rand(*shape) * 2 - 1), distortion_range, mode="constant", cval=0) - * alpha - * 1000 - ) - - for transform in self.transforms_imagenet_raw: - if type(transform) is RGB2Lab: - I1_raw = I1 - I1 = transform(I1) - else: - I1 = transform(I1) - - for transform in self.transforms_imagenet_raw: - if type(transform) is RGB2Lab: - I2 = self.distortion_transform(I2, forward_dx, forward_dy) - I2_raw = I2 - I2 = transform(I2) - I2[0:1, :, :] = I2[0:1, :, :] + torch.randn(1) * self.brightnessjitter - - I_reference_video = self.extra_reference_transform(I_reference_video) - for transform in self.transforms_imagenet_raw: - # if type(transform) is RGB2Lab and self.extra_reference_transform is not None: - # I_reference_video = self.extra_reference_transform(I_reference_video) - I_reference_video = transform(I_reference_video) - - I_reference_video_real = self.transforms_imagenet(I_reference_video_real) - - flow_forward_raw = np.stack((forward_dy, forward_dx), axis=-1) - flow_backward_raw = np.zeros_like(flow_forward_raw) - flow_forward = self.flow_transform(flow_forward_raw) - flow_backward = self.flow_transform(flow_backward_raw) - - # update the mask for the pixels on the border - grid_x, grid_y = np.meshgrid(np.arange(self.image_size[0]), np.arange(self.image_size[1]), indexing="ij") - grid = np.stack((grid_y, grid_x), axis=-1) - grid_warp = grid + flow_forward_raw - location_y = grid_warp[:, :, 0].flatten() - location_x = grid_warp[:, :, 1].flatten() - I2_raw = np.array(I2_raw).astype(float) - I21_r = map_coordinates(I2_raw[:, :, 0], np.stack((location_x, location_y)), cval=-1).reshape( - (self.image_size[0], self.image_size[1]) - ) - I21_g = map_coordinates(I2_raw[:, :, 1], np.stack((location_x, location_y)), cval=-1).reshape( - (self.image_size[0], self.image_size[1]) - ) - I21_b = map_coordinates(I2_raw[:, :, 2], np.stack((location_x, location_y)), cval=-1).reshape( - (self.image_size[0], self.image_size[1]) - ) - I21_raw = np.stack((I21_r, I21_g, I21_b), axis=2) - mask = np.ones((self.image_size[0], self.image_size[1])) - mask[(I21_raw[:, :, 0] == -1) & (I21_raw[:, :, 1] == -1) & (I21_raw[:, :, 2] == -1)] = 0 - mask[abs(I21_raw - I1_raw).sum(axis=-1) > 50] = 0 - mask = self.ToTensor(mask) - - if np.random.random() < self.real_reference_probability: - I_reference_output = I_reference_video_real - placeholder = torch.zeros_like(I1) - self_ref_flag = torch.zeros_like(I1) - else: - I_reference_output = I_reference_video - placeholder = I2 if np.random.random() < self.nonzero_placeholder_probability else torch.zeros_like(I1) - self_ref_flag = torch.ones_like(I1) - - outputs = [] - outputs.append(I1) - outputs.append(I2) - outputs.append(I_reference_output) - # outputs.append(I_reference_video_real) - outputs.append(flow_forward) - outputs.append(flow_backward) - outputs.append(mask) - outputs.append(placeholder) - outputs.append(self_ref_flag) - - except Exception as e: - if combo_path is not None: - print("problem in ", combo_path) - print("problem in, ", image_a_path) - print(e) - return self.__getitem__(np.random.randint(0, len(self.image_pairs))) - - return outputs - - def __len__(self): - return len(self.image_pairs) - # return len(self.image_pairs) * 2 diff --git a/models/ColorVidNet.py b/models/ColorVidNet.py index b27792e..70989cd 100644 --- a/models/ColorVidNet.py +++ b/models/ColorVidNet.py @@ -1,127 +1,11 @@ -from os import path as osp - import torch import torch.nn as nn -import torch.nn.functional as F import torch.nn.parallel -def type_name(m): - name = type(m).__name__ - if name == "TorchObject": - name = m._typename.replace("cudnn.", "") - return name - - -def model_tm2pm(model): - layer_map = {} - for m in model.modules: - layer_map[m.name] = tm2pm(m) - return layer_map - - -"""' -removed layers: [[ 'ab_conv1_1', 'bw_conv1_1' ]] -""" - -ref_color_layernames = [ - "relu1_1", - "conv1_2", - "relu1_2", - "conv1_2norm", - "conv1_2norm_ss", - "conv2_1", - "relu2_1", - "conv2_2", - "relu2_2", - "conv2_2norm", - "conv2_2norm_ss", - "conv3_1", - "relu3_1", - "conv3_2", - "relu3_2", - "conv3_3", - "relu3_3", - "conv3_3norm", - "conv3_3norm_ss", - "conv4_1", - "relu4_1", - "conv4_2", - "relu4_2", - "conv4_3", - "relu4_3", - "conv4_3norm", - "conv5_1", - "relu5_1", - "conv5_2", - "relu5_2", - "conv5_3", - "relu5_3", - "conv5_3norm", - "conv6_1", - "relu6_1", - "conv6_2", - "relu6_2", - "conv6_3", - "relu6_3", - "conv6_3norm", - "conv7_1", - "relu7_1", - "conv7_2", - "relu7_2", - "conv7_3", - "relu7_3", - "conv7_3norm", - "conv8_1", - "conv3_3_short", - "relu8_1_comb", - "conv8_2", - "relu8_2", - "conv8_3", - "relu8_3", - "conv8_3norm", - "conv9_1", - "conv2_2_short", - "relu9_1_comb", - "conv9_2", - "relu9_2", - "conv9_2norm", - "conv10_1", - "conv1_2_short", - "relu10_1_comb", - "conv10_2", - "relu10_2", - "conv10_ab", -] - - class ColorVidNet(nn.Module): - def __init__( - self, ic, ref_color_torch_model="convert_caffe_model/torch_model/reference_model/color_nodist_model.t7" - ): + def __init__(self, ic): super(ColorVidNet, self).__init__() - """torch_model = load_lua(ref_color_torch_model, unknown_classes=True) - if type(torch_model).__name__ == 'hashable_uniq_dict': - torch_model = torch_model.model - - # fix the ugly typo error - for m in torch_model.modules: - if m.name == 'conv8_1_comb' and type_name(m) == 'ReLU': - m.name = 'relu8_1_comb' - - pytorch_model = model_tm2pm(torch_model) - - self.add_module('conv1_1', nn.Sequential(nn.Conv2d(ic, 32, 3, 1, 1), nn.ReLU(), nn.Conv2d(32, 64, 3, 1, 1), nn.ReLU())) - - for l_name in ref_color_layernames: - self.add_module(l_name, pytorch_model[l_name]) - - ### replace all deconvolution with neareast upsample + convolution - assert type(self.conv8_1).__name__=='ConvTranspose2d' and type(self.conv9_1).__name__=='ConvTranspose2d' and \ - type(self.conv10_1).__name__=='ConvTranspose2d', 'conv8_1, conv9_1, conv10_1 should be all ConvTranspose2d' - """ - - ##### Written by Ak. ##### self.conv1_1 = nn.Sequential(nn.Conv2d(ic, 32, 3, 1, 1), nn.ReLU(), nn.Conv2d(32, 64, 3, 1, 1)) self.conv1_2 = nn.Conv2d(64, 64, 3, 1, 1) self.conv1_2norm = nn.BatchNorm2d(64, affine=False) @@ -191,11 +75,8 @@ class ColorVidNet(nn.Module): self.relu9_1_comb = nn.ReLU() self.relu9_2 = nn.ReLU() self.relu10_1_comb = nn.ReLU() - # self.relu10_2 = nn.ReLU() self.relu10_2 = nn.LeakyReLU(0.2, True) - ##### Above ##### - print("replace all deconv with [nearest + conv]") self.conv8_1 = nn.Sequential(nn.Upsample(scale_factor=2, mode="nearest"), nn.Conv2d(512, 256, 3, 1, 1)) self.conv9_1 = nn.Sequential(nn.Upsample(scale_factor=2, mode="nearest"), nn.Conv2d(256, 128, 3, 1, 1)) @@ -211,7 +92,6 @@ class ColorVidNet(nn.Module): self.conv7_3norm = nn.InstanceNorm2d(512) self.conv8_3norm = nn.InstanceNorm2d(256) self.conv9_2norm = nn.InstanceNorm2d(128) - # print(self) def forward(self, x): """ x: gray image (1 channel), ab(2 channel), ab_err, ba_err""" @@ -260,6 +140,5 @@ class ColorVidNet(nn.Module): conv10_1_comb = self.relu10_1_comb(conv10_1 + conv1_2_short) conv10_2 = self.relu10_2(self.conv10_2(conv10_1_comb)) conv10_ab = self.conv10_ab(conv10_2) - pred_ab = torch.tanh(conv10_ab) * 128 - return pred_ab + return torch.tanh(conv10_ab) * 128 diff --git a/models/FrameColor.py b/models/FrameColor.py index ba2acbf..d563a8a 100644 --- a/models/FrameColor.py +++ b/models/FrameColor.py @@ -1,21 +1,9 @@ -# from imports import * import torch -from utils.tensor_lab2rgb import tensor_lab2rgb from utils.util import * -if torch.cuda.is_available(): - device = torch.device("cuda") -else: - device = troch.device("cpu") - def warp_color(IA_l, IB_lab, features_B, vggnet, nonlocal_net, colornet, feature_noise=0, temperature=0.01): - """ - joint_train=True, enable grad; otherwise disable grad - """ - # change to rgb for feature extraction IA_rgb_from_gray = gray2rgb_batch(IA_l) - with torch.no_grad(): A_relu1_1, A_relu2_1, A_relu3_1, A_relu4_1, A_relu5_1 = vggnet( IA_rgb_from_gray, ["r12", "r22", "r32", "r42", "r52"], preprocess=True @@ -34,18 +22,17 @@ def warp_color(IA_l, IB_lab, features_B, vggnet, nonlocal_net, colornet, feature B_relu4_1 = feature_normalize(B_relu4_1) B_relu5_1 = feature_normalize(B_relu5_1) - # if feature_noise > 0: - # A_relu2_1 = A_relu2_1 + torch.randn_like(A_relu2_1, requires_grad=False) * feature_noise - # B_relu2_1 = B_relu2_1 + torch.randn_like(B_relu2_1, requires_grad=False) * feature_noise - # A_relu3_1 = A_relu3_1 + torch.randn_like(A_relu3_1, requires_grad=False) * feature_noise - # B_relu3_1 = B_relu3_1 + torch.randn_like(B_relu3_1, requires_grad=False) * feature_noise - # A_relu4_1 = A_relu4_1 + torch.randn_like(A_relu4_1, requires_grad=False) * feature_noise - # B_relu4_1 = B_relu4_1 + torch.randn_like(B_relu4_1, requires_grad=False) * feature_noise - # A_relu5_1 = A_relu5_1 + torch.randn_like(A_relu5_1, requires_grad=False) * feature_noise - # B_relu5_1 = B_relu5_1 + torch.randn_like(B_relu5_1, requires_grad=False) * feature_noise - nonlocal_BA_lab, similarity_map = nonlocal_net( - IB_lab, A_relu2_1, A_relu3_1, A_relu4_1, A_relu5_1, B_relu2_1, B_relu3_1, B_relu4_1, B_relu5_1, temperature=0.01 + IB_lab, + A_relu2_1, + A_relu3_1, + A_relu4_1, + A_relu5_1, + B_relu2_1, + B_relu3_1, + B_relu4_1, + B_relu5_1, + temperature=temperature, ) return nonlocal_BA_lab, similarity_map, features_A @@ -64,7 +51,7 @@ def frame_colorization( luminance_noise=0, temperature=0.01, ): - # change to rgb for feature extraction + IA_l = IA_lab[:, 0:1, :, :] if luminance_noise: IA_l = IA_l + torch.randn_like(IA_l, requires_grad=False) * luminance_noise diff --git a/models/GAN_models.py b/models/GAN_models.py index 024b3a3..a3b2f3d 100644 --- a/models/GAN_models.py +++ b/models/GAN_models.py @@ -81,11 +81,11 @@ class Self_Attention(nn.Module): def forward(self, x): """ - inputs : - x : input feature maps( B X C X W X H) - returns : - out : self attention value + input feature - attention: B X N X N (N is Width*Height) + inputs : + x : input feature maps( B X C X W X H) + returns : + out : self attention value + input feature + attention: B X N X N (N is Width*Height) """ m_batchsize, C, width, height = x.size() proj_query = self.query_conv(x).view(m_batchsize, -1, width * height).permute(0, 2, 1) # B X CX(N) @@ -98,13 +98,12 @@ class Self_Attention(nn.Module): out = out.view(m_batchsize, C, width, height) out = self.gamma * out + x - # return out, attention return out class Discriminator_WITHOUT_FC_x64_video(nn.Module): """ - Discriminative Network + Discriminative Network """ def __init__(self, in_size=6, ndf=64): @@ -137,18 +136,7 @@ class Discriminator_WITHOUT_FC_x64_video(nn.Module): nn.LeakyReLU(0.2, inplace=True), ) - # sate size: (ndf * 8) x 4 x 4 - # Self_Attention(self.ndf * 8, 'relu') - self.last = SpectralNorm(nn.Conv2d(self.ndf * 8, 1, [3, 6], 1, 0)) - # nn.Sigmoid() - # state size: 1 x 1 x 1 - - # for m in self.modules(): - # if isinstance(m, nn.Conv2d): - # m.weight.data.normal_(0.0, 0.02) - # if m.bias is not None: - # m.bias.data.zero_() def forward(self, input): feature1 = self.layer1(input) @@ -163,7 +151,7 @@ class Discriminator_WITHOUT_FC_x64_video(nn.Module): class Discriminator_WITHOUT_FC_x64_video_BIG(nn.Module): """ - Discriminative Network + Discriminative Network """ def __init__(self, in_size=6, ndf=64): @@ -219,7 +207,7 @@ class Discriminator_WITHOUT_FC_x64_video_BIG(nn.Module): class Discriminator_WITHOUT_FC_x64(nn.Module): """ - Discriminative Network + Discriminative Network """ def __init__(self, in_size=3, ndf=64): @@ -248,14 +236,6 @@ class Discriminator_WITHOUT_FC_x64(nn.Module): ) self.last = SpectralNorm(nn.Conv2d(self.ndf * 8, 1, 4, 1, 0)) - # nn.Sigmoid() - # state size: 1 x 1 x 1 - - # for m in self.modules(): - # if isinstance(m, nn.Conv2d): - # m.weight.data.normal_(0.0, 0.02) - # if m.bias is not None: - # m.bias.data.zero_() def forward(self, input): @@ -268,7 +248,7 @@ class Discriminator_WITHOUT_FC_x64(nn.Module): class Discriminator_WITHOUT_FC_x64_BIG(nn.Module): """ - Discriminative Network + Discriminative Network """ def __init__(self, in_size=3, ndf=64): @@ -316,7 +296,7 @@ class Discriminator_WITHOUT_FC_x64_BIG(nn.Module): class Discriminator_WITHOUT_FC_x128(nn.Module): """ - Discriminative Network + Discriminative Network """ def __init__(self, in_size=3, ndf=64): @@ -359,7 +339,7 @@ class Discriminator_WITHOUT_FC_x128(nn.Module): class Discriminator_WITHOUT_FC_x256(nn.Module): """ - Discriminative Network + Discriminative Network """ def __init__(self, in_size=3, ndf=64): diff --git a/models/NonlocalNet.py b/models/NonlocalNet.py index d15cdb1..1713db0 100644 --- a/models/NonlocalNet.py +++ b/models/NonlocalNet.py @@ -14,12 +14,14 @@ def find_local_patch(x, patch_size): x_unfold = F.unfold( x, kernel_size=(patch_size, patch_size), padding=(patch_size // 2, patch_size // 2), stride=(1, 1) ) - out = x_unfold.view(N, x_unfold.shape[1], H, W) - return out + + return x_unfold.view(N, x_unfold.shape[1], H, W) class WeightedAverage(nn.Module): - def __init__(self,): + def __init__( + self, + ): super(WeightedAverage, self).__init__() def forward(self, x_lab, patch_size=3, alpha=1, scale_factor=1): @@ -34,14 +36,13 @@ class WeightedAverage(nn.Module): local_difference_l = (local_l - l) ** 2 correlation = nn.functional.softmax(-1 * local_difference_l / alpha, dim=1) - weighted_ab = torch.cat( + return torch.cat( ( torch.sum(correlation * local_a, dim=1, keepdim=True), torch.sum(correlation * local_b, dim=1, keepdim=True), ), 1, ) - return weighted_ab class WeightedAverage_color(nn.Module): @@ -49,11 +50,13 @@ class WeightedAverage_color(nn.Module): smooth the image according to the color distance in the LAB space """ - def __init__(self,): + def __init__( + self, + ): super(WeightedAverage_color, self).__init__() def forward(self, x_lab, x_lab_predict, patch_size=3, alpha=1, scale_factor=1): - # alpha=0: less smooth; alpha=inf: smoother + """ alpha=0: less smooth; alpha=inf: smoother """ x_lab = F.interpolate(x_lab, scale_factor=scale_factor) l = uncenter_l(x_lab[:, 0:1, :, :]) a = x_lab[:, 1:2, :, :] @@ -71,18 +74,19 @@ class WeightedAverage_color(nn.Module): -1 * local_color_difference / alpha, dim=1 ) # so that sum of weights equal to 1 - weighted_ab = torch.cat( + return torch.cat( ( torch.sum(correlation * local_a_predict, dim=1, keepdim=True), torch.sum(correlation * local_b_predict, dim=1, keepdim=True), ), 1, ) - return weighted_ab class NonlocalWeightedAverage(nn.Module): - def __init__(self,): + def __init__( + self, + ): super(NonlocalWeightedAverage, self).__init__() def forward(self, x_lab, feature, patch_size=3, alpha=0.1, scale_factor=1): @@ -113,56 +117,36 @@ class CorrelationLayer(nn.Module): self.search_range = search_range def forward(self, x1, x2, alpha=1, raw_output=False, metric="similarity"): - # args = self.args - shape = list(x1.size()) shape[1] = (self.search_range * 2 + 1) ** 2 cv = torch.zeros(shape).to(torch.device("cuda")) - if metric == "similarity": - for i in range(-self.search_range, self.search_range + 1): - for j in range(-self.search_range, self.search_range + 1): - if i < 0: - slice_h, slice_h_r = slice(None, i), slice(-i, None) - elif i > 0: - slice_h, slice_h_r = slice(i, None), slice(None, -i) - else: - slice_h, slice_h_r = slice(None), slice(None) + for i in range(-self.search_range, self.search_range + 1): + for j in range(-self.search_range, self.search_range + 1): + if i < 0: + slice_h, slice_h_r = slice(None, i), slice(-i, None) + elif i > 0: + slice_h, slice_h_r = slice(i, None), slice(None, -i) + else: + slice_h, slice_h_r = slice(None), slice(None) - if j < 0: - slice_w, slice_w_r = slice(None, j), slice(-j, None) - elif j > 0: - slice_w, slice_w_r = slice(j, None), slice(None, -j) - else: - slice_w, slice_w_r = slice(None), slice(None) + if j < 0: + slice_w, slice_w_r = slice(None, j), slice(-j, None) + elif j > 0: + slice_w, slice_w_r = slice(j, None), slice(None, -j) + else: + slice_w, slice_w_r = slice(None), slice(None) - # storage sequence (eg. search_range=3): -24, -23, .., -1, 0, 1, ..., 23, 24 + if metric == "similarity": cv[:, (self.search_range * 2 + 1) * i + j, slice_h, slice_w] = ( x1[:, :, slice_h, slice_w] * x2[:, :, slice_h_r, slice_w_r] ).sum(1) - else: # patchwise subtraction - for i in range(-self.search_range, self.search_range + 1): - for j in range(-self.search_range, self.search_range + 1): - if i < 0: - slice_h, slice_h_r = slice(None, i), slice(-i, None) - elif i > 0: - slice_h, slice_h_r = slice(i, None), slice(None, -i) - else: - slice_h, slice_h_r = slice(None), slice(None) - - if j < 0: - slice_w, slice_w_r = slice(None, j), slice(-j, None) - elif j > 0: - slice_w, slice_w_r = slice(j, None), slice(None, -j) - else: - slice_w, slice_w_r = slice(None), slice(None) - - # storage sequence (eg. search_range=3): -24, -23, .., -1, 0, 1, ..., 23, 24 + else: # patchwise subtraction cv[:, (self.search_range * 2 + 1) * i + j, slice_h, slice_w] = -( (x1[:, :, slice_h, slice_w] - x2[:, :, slice_h_r, slice_w_r]) ** 2 ).sum(1) - ## TODO sigmoid? + # TODO sigmoid? if raw_output: return cv else: @@ -185,11 +169,11 @@ class Self_Attn(nn.Module): def forward(self, x): """ - inputs : - x : input feature maps(B X C X W X H) - returns : - out : self attention value + input feature - attention: B X N X N (N is Width*Height) + inputs : + x : input feature maps(B X C X W X H) + returns : + out : self attention value + input feature + attention: B X N X N (N is Width*Height) """ m_batchsize, C, width, height = x.size() proj_query = self.query_conv(x).view(m_batchsize, -1, width * height).permute(0, 2, 1) # B X N X C @@ -206,7 +190,7 @@ class Self_Attn(nn.Module): class VGG19_feature_color_torchversion(nn.Module): - """ + """ NOTE: no need to pre-process the input; input tensor should range in [0,1] """ @@ -242,7 +226,7 @@ class VGG19_feature_color_torchversion(nn.Module): self.pool5 = nn.AvgPool2d(kernel_size=2, stride=2) def forward(self, x, out_keys, preprocess=True): - """ + """ NOTE: input tensor should range in [0,1] """ out = {} @@ -283,8 +267,8 @@ class VGG19_feature_color(nn.Module): """Extract multiple convolutional feature maps.""" features = [] for name, layer in self.vgg._modules.items(): - x = layer(x) if name in self.select: + x = layer(x) features.append(x) return features @@ -301,70 +285,20 @@ class VGG19_feature(nn.Module): return A_relu3_1, A_relu4_1, A_relu5_1, B_relu3_1, B_relu4_1, B_relu5_1 -class VGG19_feature_new(nn.Module): - # input: [LLL] channels, range=[0,255] - def __init__(self, gpu_ids): - super(VGG19_feature_new, self).__init__() - self.vgg19_gray_new = vgg19_gray_new().cuda() - - def forward(self, A_l, B_l): - A_relu2_1, A_relu3_1, A_relu4_1, A_relu5_1 = self.vgg19_gray_new(A_l) - B_relu2_1, B_relu3_1, B_relu4_1, B_relu5_1 = self.vgg19_gray_new(B_l) - return A_relu2_1, A_relu3_1, A_relu4_1, A_relu5_1, B_relu2_1, B_relu3_1, B_relu4_1, B_relu5_1 - - -class WTA(torch.autograd.Function): - """ - We can implement our own custom autograd Functions by subclassing - torch.autograd.Function and implementing the forward and backward passes - which operate on Tensors. - """ - - @staticmethod - def forward(ctx, input): - """ - In the forward pass we receive a Tensor containing the input and return a - Tensor containing the output. You can cache arbitrary Tensors for use in the - backward pass using the save_for_backward method. - """ - height = input.shape[2] - width = input.shape[3] - activation_max, index_max = torch.topk(input, -1, keepdim=True) - input_zeros = input * 0 - output_max_only = torch.where(input == activation_max, input, input_zeros) - - mask = (output_max_only > 0).type(torch.float) - ctx.save_for_backward(input, mask) - return output_max_only - - @staticmethod - def backward(ctx, grad_output): - """ - In the backward pass we receive a Tensor containing the gradient of the loss - with respect to the output, and we need to compute the gradient of the loss - with respect to the input. - """ - # import pdb - # pdb.set_trace() - input, mask = ctx.saved_tensors - grad_input = grad_output.clone() * mask - return grad_input - - class WTA_scale(torch.autograd.Function): """ - We can implement our own custom autograd Functions by subclassing - torch.autograd.Function and implementing the forward and backward passes - which operate on Tensors. - """ + We can implement our own custom autograd Functions by subclassing + torch.autograd.Function and implementing the forward and backward passes + which operate on Tensors. + """ @staticmethod def forward(ctx, input, scale=1e-4): """ - In the forward pass we receive a Tensor containing the input and return a - Tensor containing the output. You can cache arbitrary Tensors for use in the - backward pass using the save_for_backward method. - """ + In the forward pass we receive a Tensor containing the input and return a + Tensor containing the output. You can cache arbitrary Tensors for use in the + backward pass using the save_for_backward method. + """ activation_max, index_max = torch.max(input, -1, keepdim=True) input_scale = input * scale # default: 1e-4 # input_scale = input * scale # default: 1e-4 @@ -377,10 +311,10 @@ class WTA_scale(torch.autograd.Function): @staticmethod def backward(ctx, grad_output): """ - In the backward pass we receive a Tensor containing the gradient of the loss - with respect to the output, and we need to compute the gradient of the loss - with respect to the input. - """ + In the backward pass we receive a Tensor containing the gradient of the loss + with respect to the output, and we need to compute the gradient of the loss + with respect to the input. + """ # import pdb # pdb.set_trace() input, mask = ctx.saved_tensors @@ -418,9 +352,9 @@ class ResidualBlock(nn.Module): return out -## NOTE: THIS IS THE FINAL VERSION class WarpNet(nn.Module): - # input is Al, Bl, channel = 1, range~[0,255] + """ input is Al, Bl, channel = 1, range~[0,255] """ + def __init__(self, batch_size): super(WarpNet, self).__init__() self.feature_channel = 64 @@ -549,12 +483,9 @@ class WarpNet(nn.Module): similarity_map = similarity_map.view(batch_size, 1, feature_height, feature_width) # f can be negative - if WTA_scale_weight == 1: - f_WTA = f - else: - f_WTA = WTA_scale.apply(f, WTA_scale_weight) + f_WTA = f if WTA_scale_weight == 1 else WTA_scale.apply(f, WTA_scale_weight) f_WTA = f_WTA / temperature - f_div_C = F.softmax(f_WTA.squeeze_(), dim=-1) # 2*1936*1936; softmax along the horizontal line (dim=-1) + f_div_C = F.softmax(f_WTA.squeeze_(), dim=-1) # 2*1936*1936; # downsample the reference color B_lab = F.avg_pool2d(B_lab_map, 4) diff --git a/models/guided_filter.py b/models/guided_filter.py deleted file mode 100644 index b2cab00..0000000 --- a/models/guided_filter.py +++ /dev/null @@ -1,125 +0,0 @@ -import torch -from torch import nn -from torch.autograd import Variable -from torch.nn import functional as F - -# from .box_filter import BoxFilter - - -def diff_x(input, r): - assert input.dim() == 4 - - left = input[:, :, r : 2 * r + 1] - middle = input[:, :, 2 * r + 1 :] - input[:, :, : -2 * r - 1] - right = input[:, :, -1:] - input[:, :, -2 * r - 1 : -r - 1] - - output = torch.cat([left, middle, right], dim=2) - - return output - - -def diff_y(input, r): - assert input.dim() == 4 - - left = input[:, :, :, r : 2 * r + 1] - middle = input[:, :, :, 2 * r + 1 :] - input[:, :, :, : -2 * r - 1] - right = input[:, :, :, -1:] - input[:, :, :, -2 * r - 1 : -r - 1] - - output = torch.cat([left, middle, right], dim=3) - - return output - - -class BoxFilter(nn.Module): - def __init__(self, r): - super(BoxFilter, self).__init__() - - self.r = r - - def forward(self, x): - assert x.dim() == 4 - - return diff_y(diff_x(x.cumsum(dim=2), self.r).cumsum(dim=3), self.r) - - -class FastGuidedFilter(nn.Module): - def __init__(self, r, eps=1e-8): - super(FastGuidedFilter, self).__init__() - - self.r = r - self.eps = eps - self.boxfilter = BoxFilter(r) - - def forward(self, lr_x, lr_y, hr_x): - n_lrx, c_lrx, h_lrx, w_lrx = lr_x.size() - n_lry, c_lry, h_lry, w_lry = lr_y.size() - n_hrx, c_hrx, h_hrx, w_hrx = hr_x.size() - - assert n_lrx == n_lry and n_lry == n_hrx - assert c_lrx == c_hrx and (c_lrx == 1 or c_lrx == c_lry) - assert h_lrx == h_lry and w_lrx == w_lry - assert h_lrx > 2 * self.r + 1 and w_lrx > 2 * self.r + 1 - - ## N - N = self.boxfilter(Variable(lr_x.data.new().resize_((1, 1, h_lrx, w_lrx)).fill_(1.0))) - - ## mean_x - mean_x = self.boxfilter(lr_x) / N - ## mean_y - mean_y = self.boxfilter(lr_y) / N - ## cov_xy - cov_xy = self.boxfilter(lr_x * lr_y) / N - mean_x * mean_y - ## var_x - var_x = self.boxfilter(lr_x * lr_x) / N - mean_x * mean_x - - ## A - A = cov_xy / (var_x + self.eps) - ## b - b = mean_y - A * mean_x - - ## mean_A; mean_b - mean_A = F.upsample(A, (h_hrx, w_hrx), mode="bilinear") - mean_b = F.upsample(b, (h_hrx, w_hrx), mode="bilinear") - - return mean_A * hr_x + mean_b - - -class GuidedFilter(nn.Module): - def __init__(self, r, eps=1e-8): - super(GuidedFilter, self).__init__() - - self.r = r - self.eps = eps - self.boxfilter = BoxFilter(r) - - def forward(self, x, y): - n_x, c_x, h_x, w_x = x.size() - n_y, c_y, h_y, w_y = y.size() - - assert n_x == n_y - assert c_x == 1 or c_x == c_y - assert h_x == h_y and w_x == w_y - assert h_x > 2 * self.r + 1 and w_x > 2 * self.r + 1 - - # N - N = self.boxfilter(Variable(x.data.new().resize_((1, 1, h_x, w_x)).fill_(1.0))) - - # mean_x - mean_x = self.boxfilter(x) / N - # mean_y - mean_y = self.boxfilter(y) / N - # cov_xy - cov_xy = self.boxfilter(x * y) / N - mean_x * mean_y - # var_x - var_x = self.boxfilter(x * x) / N - mean_x * mean_x - - # A - A = cov_xy / (var_x + self.eps) - # b - b = mean_y - A * mean_x - - # mean_A; mean_b - mean_A = self.boxfilter(A) / N - mean_b = self.boxfilter(b) / N - - return mean_A * x + mean_b diff --git a/models/spectral_normalization.py b/models/spectral_normalization.py index 30279fe..6801480 100644 --- a/models/spectral_normalization.py +++ b/models/spectral_normalization.py @@ -1,9 +1,6 @@ import torch -import torch.nn.functional as F -from torch import Tensor, nn -from torch.autograd import Variable +from torch import nn from torch.nn import Parameter -from torch.optim.optimizer import Optimizer, required def l2normalize(v, eps=1e-12): @@ -29,7 +26,6 @@ class SpectralNorm(nn.Module): v.data = l2normalize(torch.mv(torch.t(w.view(height, -1).data), u.data)) u.data = l2normalize(torch.mv(w.view(height, -1).data, v.data)) - # sigma = torch.dot(u.data, torch.mv(w.view(height,-1).data, v.data)) sigma = u.dot(w.view(height, -1).mv(v)) setattr(self.module, self.name, w / sigma.expand_as(w)) diff --git a/models/vgg19_gray.py b/models/vgg19_gray.py index a6b8315..44ad8e0 100644 --- a/models/vgg19_gray.py +++ b/models/vgg19_gray.py @@ -1,9 +1,7 @@ -from collections import OrderedDict, namedtuple from functools import reduce import torch import torch.nn as nn -from torch.autograd import Variable class LambdaBase(nn.Sequential): @@ -15,6 +13,7 @@ class LambdaBase(nn.Sequential): output = [] for module in self._modules.values(): output.append(module(input)) + return output if output else input @@ -126,13 +125,10 @@ model = nn.Sequential( # Sequential, ) -model.load_state_dict(torch.load("models/vgg19_gray.pth")) -# print(model) +model.load_state_dict(torch.load("data/vgg19_gray.pth")) vgg19_gray_net = torch.nn.Sequential() for (name, layer) in model._modules.items(): vgg19_gray_net.add_module(layer_names[int(name)], model[int(name)]) -# print(vgg19_gray_net) - for param in vgg19_gray_net.parameters(): param.requires_grad = False @@ -163,8 +159,6 @@ class vgg19_gray(torch.nn.Module): h_relu4_1 = h h = self.slice3(h) h_relu5_1 = h - # vgg_outputs = namedtuple("VggOutputs", ['h_relu3_1', 'h_relu4_1', 'h_relu5_1']) - # out = vgg_outputs(h_relu3_1, h_relu4_1, h_relu5_1) return h_relu3_1, h_relu4_1, h_relu5_1 @@ -197,40 +191,4 @@ class vgg19_gray_new(torch.nn.Module): h_relu4_1 = h h = self.slice3(h) h_relu5_1 = h - # vgg_outputs = namedtuple("VggOutputs", ['h_relu3_1', 'h_relu4_1', 'h_relu5_1']) - # out = vgg_outputs(h_relu3_1, h_relu4_1, h_relu5_1) return h_relu2_1, h_relu3_1, h_relu4_1, h_relu5_1 - - -# class vgg19_gray_conv(torch.nn.Module): -# def __init__(self, requires_grad=False): -# super(vgg19_gray_conv, self).__init__() -# vgg_pretrained_features = vgg19_gray_net -# self.slice0 = torch.nn.Sequential() -# self.slice1 = torch.nn.Sequential() -# self.slice2 = torch.nn.Sequential() -# self.slice3 = torch.nn.Sequential() -# for x in range(6): -# self.slice0.add_module(layer_names[x], vgg_pretrained_features[x]) -# for x in range(6, 11): -# self.slice1.add_module(layer_names[x], vgg_pretrained_features[x]) -# for x in range(11, 20): -# self.slice2.add_module(layer_names[x], vgg_pretrained_features[x]) -# for x in range(20, 29): -# self.slice3.add_module(layer_names[x], vgg_pretrained_features[x]) -# if not requires_grad: -# for param in self.parameters(): -# param.requires_grad = False - -# def forward(self, X): -# h = self.slice0(X) -# h_relu2_1 = h -# h = self.slice1(h) -# h_relu3_1 = h -# h = self.slice2(h) -# h_relu4_1 = h -# h = self.slice3(h) -# h_relu5_1 = h -# # vgg_outputs = namedtuple("VggOutputs", ['h_relu3_1', 'h_relu4_1', 'h_relu5_1']) -# # out = vgg_outputs(h_relu3_1, h_relu4_1, h_relu5_1) -# return h_relu2_1, h_relu3_1, h_relu4_1, h_relu5_1 diff --git a/test.py b/test.py index 4fcf8bf..1c5aaae 100644 --- a/test.py +++ b/test.py @@ -16,8 +16,7 @@ import lib.TestTransforms as transforms from models.ColorVidNet import ColorVidNet from models.FrameColor import frame_colorization from models.NonlocalNet import VGG19_feature_color_torchversion, WarpNet -from utils.tensor_lab2rgb import tensor_lab2rgb -from utils.util import batch_lab2rgb_transpose_mc, folder2vid, mkdir_if_not, save_frames, uncenter_l +from utils.util import batch_lab2rgb_transpose_mc, folder2vid, mkdir_if_not, save_frames, tensor_lab2rgb, uncenter_l from utils.util_distortion import CenterPad, Normalize, RGB2Lab, ToTensor os.environ["CUDA_DEVICE_ORDER"] = "PCI_BUS_ID" @@ -50,11 +49,10 @@ def colorize_video(opt, input_path, reference_path, output_path, nonlocal_net, c print("reference name:", ref_name) frame_ref = Image.open(ref_name) - iter_num = 0 total_time = 0 I_last_lab_predict = None - for index in range(file_count): + for iter_num, index in enumerate(range(file_count)): frame_name = filenames[index] print("input =", frame_name) print("reference =", ref_name) @@ -131,11 +129,9 @@ def colorize_video(opt, input_path, reference_path, output_path, nonlocal_net, c # save the frames save_frames(IA_predict_rgb, output_path, index) - iter_num = iter_num + 1 - # output video video_name = "video.avi" - folder2vid(image_folder=output_path, output_dir=".", filename=video_name) + folder2vid(image_folder=output_path, output_dir=output_path, filename=video_name) if __name__ == "__main__": diff --git a/utils/data_augmentation.py b/utils/data_augmentation.py deleted file mode 100644 index 1362ed4..0000000 --- a/utils/data_augmentation.py +++ /dev/null @@ -1,154 +0,0 @@ -import random - -import cv2 -import lib.functional as F -import numpy as np -from PIL import Image -from scipy.misc import imread, imresize, imsave -from scipy.ndimage.filters import gaussian_filter -from skimage import color - -cv2.setNumThreads(0) - - -# %% Dataloader -class RGB2Lab(object): - def __init__(self): - pass - - def __call__(self, inputs): - image_lab = color.rgb2lab(inputs) - return image_lab - - -class Guassian_noise(object): - """Elastic distortion - """ - - def __init__(self, noise_sigma=0.1): - self.noise_sigma = noise_sigma - - def __call__(self, inputs): - h = inputs.shape[0] - w = inputs.shape[1] - noisy_image = inputs - noise = np.random.randn(h, w) * self.noise_sigma - noisy_image[:, :, 0] = inputs[:, :, 0] + noise - - return noisy_image - - -class Distortion(object): - """Elastic distortion - """ - - def __init__(self, distortion_level=3, flip_probability=0.1): - self.alpha_max = distortion_level - self.flip_probability = flip_probability - - def __call__(self, inputs): - if np.random.rand() < self.flip_probability: - inputs = inputs.transpose(Image.FLIP_LEFT_RIGHT) - - inputs = np.array(inputs) - alpha = np.random.rand() * self.alpha_max - sigma = 50 - random_state = np.random.RandomState(None) - shape = inputs.shape[0], inputs.shape[1] - - dx = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode="constant", cval=0) * alpha * 1000 - dy = gaussian_filter((random_state.rand(*shape) * 2 - 1), sigma, mode="constant", cval=0) * alpha * 1000 - - x, y = np.meshgrid(np.arange(shape[0]), np.arange(shape[1]), indexing="ij") - remap_image = cv2.remap( - inputs, (dy + y).astype(np.float32), (dx + x).astype(np.float32), interpolation=cv2.INTER_LINEAR - ) - - return Image.fromarray(remap_image) - - -class Normalize(object): - def __init__(self): - pass - - def __call__(self, inputs): - inputs[0:1, :, :] = F.normalize(inputs[0:1, :, :], 50, 1) - inputs[1:3, :, :] = F.normalize(inputs[1:3, :, :], (0, 0), (1, 1)) - return inputs - - -class ToTensor(object): - def __init__(self): - pass - - def __call__(self, inputs): - outputs = F.to_mytensor(inputs) - return outputs - - -class RandomErasing(object): - """ - Class that performs Random Erasing in Random Erasing Data Augmentation by Zhong et al. - ------------------------------------------------------------------------------------- - probability: The probability that the operation will be performed. - sl: min erasing area - sh: max erasing area - r1: min aspect ratio - mean: erasing value - ------------------------------------------------------------------------------------- - """ - - def __init__(self, probability=0.6, sl=0.05, sh=0.6): - self.probability = probability - self.sl = sl - self.sh = sh - - def __call__(self, img): - img = np.array(img) - if random.uniform(0, 1) > self.probability: - return Image.fromarray(img) - - area = img.shape[0] * img.shape[1] - h0 = img.shape[0] - w0 = img.shape[1] - channel = img.shape[2] - - h = int(round(random.uniform(self.sl, self.sh) * h0)) - w = int(round(random.uniform(self.sl, self.sh) * w0)) - - if w < img.shape[1] and h < img.shape[0]: - x1 = random.randint(0, img.shape[0] - h) - y1 = random.randint(0, img.shape[1] - w) - img[x1 : x1 + h, y1 : y1 + w, :] = np.random.rand(h, w, channel) * 255 - return Image.fromarray(img) - - return Image.fromarray(img) - - -class CenteredPad(object): - """ - pad the frame with black border, - make square image for processing - """ - - def __init__(self, image_size): - self.image_size = image_size - - def __call__(self, img): - img = np.array(img) - width = np.size(img, 1) - height = np.size(img, 0) - old_size = [height, width] - - ratio = float(self.image_size) / max(height, width) - new_size = [int(x * ratio) for x in old_size] - I_resize = imresize(img, new_size) - width_new = np.size(I_resize, 1) - height_new = np.size(I_resize, 0) - - I_pad = np.zeros((self.image_size, self.image_size, 3)) - start_height = (self.image_size - new_size[0]) // 2 - start_width = (self.image_size - new_size[1]) // 2 - I_pad[start_height : (start_height + height_new), start_width : (start_width + width_new), :] = I_resize - - return Image.fromarray(I_pad.astype(np.uint8)) diff --git a/utils/flowlib.py b/utils/flowlib.py deleted file mode 100644 index f982829..0000000 --- a/utils/flowlib.py +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/python -""" -# ============================== -# flowlib.py -# library for optical flow processing -# Author: Ruoteng Li -# Date: 6th Aug 2016 -# ============================== -""" -import matplotlib.colors as cl -import matplotlib.pyplot as plt -import numpy as np -import png -from PIL import Image - -UNKNOWN_FLOW_THRESH = 1e7 -SMALLFLOW = 0.0 -LARGEFLOW = 1e8 - -""" -============= -Flow Section -============= -""" - - -def show_flow(filename): - """ - visualize optical flow map using matplotlib - :param filename: optical flow file - :return: None - """ - flow = read_flow(filename) - img = flow_to_image(flow) - plt.imshow(img) - plt.show() - - -def visualize_flow(flow, mode="Y"): - """ - this function visualize the input flow - :param flow: input flow in array - :param mode: choose which color mode to visualize the flow (Y: Ccbcr, RGB: RGB color) - :return: None - """ - if mode == "Y": - # Ccbcr color wheel - img = flow_to_image(flow) - plt.imshow(img) - plt.show() - elif mode == "RGB": - (h, w) = flow.shape[0:2] - du = flow[:, :, 0] - dv = flow[:, :, 1] - valid = flow[:, :, 2] - max_flow = max(np.max(du), np.max(dv)) - img = np.zeros((h, w, 3), dtype=np.float64) - # angle layer - img[:, :, 0] = np.arctan2(dv, du) / (2 * np.pi) - # magnitude layer, normalized to 1 - img[:, :, 1] = np.sqrt(du * du + dv * dv) * 8 / max_flow - # phase layer - img[:, :, 2] = 8 - img[:, :, 1] - # clip to [0,1] - small_idx = img[:, :, 0:3] < 0 - large_idx = img[:, :, 0:3] > 1 - img[small_idx] = 0 - img[large_idx] = 1 - # convert to rgb - img = cl.hsv_to_rgb(img) - # remove invalid point - img[:, :, 0] = img[:, :, 0] * valid - img[:, :, 1] = img[:, :, 1] * valid - img[:, :, 2] = img[:, :, 2] * valid - # show - plt.imshow(img) - plt.show() - - return None - - -def read_flow(filename): - """ - read optical flow from Middlebury .flo file - :param filename: name of the flow file - :return: optical flow data in matrix - """ - f = open(filename, "rb") - try: - magic = np.fromfile(f, np.float32, count=1)[0] # For Python3.x - except: - magic = np.fromfile(f, np.float32, count=1) # For Python2.x - data2d = None - - if 202021.25 != magic: - print("Magic number incorrect. Invalid .flo file") - else: - w = np.fromfile(f, np.int32, count=1)[0] - h = np.fromfile(f, np.int32, count=1)[0] - # print("Reading %d x %d flo file" % (h, w)) - data2d = np.fromfile(f, np.float32, count=2 * w * h) - # reshape data into 3D array (columns, rows, channels) - data2d = np.resize(data2d, (h, w, 2)) - f.close() - return data2d - - -def read_flow_png(flow_file): - """ - Read optical flow from KITTI .png file - :param flow_file: name of the flow file - :return: optical flow data in matrix - """ - flow_object = png.Reader(filename=flow_file) - flow_direct = flow_object.asDirect() - flow_data = list(flow_direct[2]) - (w, h) = flow_direct[3]["size"] - flow = np.zeros((h, w, 3), dtype=np.float64) - for i in range(len(flow_data)): - flow[i, :, 0] = flow_data[i][0::3] - flow[i, :, 1] = flow_data[i][1::3] - flow[i, :, 2] = flow_data[i][2::3] - - invalid_idx = flow[:, :, 2] == 0 - flow[:, :, 0:2] = (flow[:, :, 0:2] - 2 ** 15) / 64.0 - flow[invalid_idx, 0] = 0 - flow[invalid_idx, 1] = 0 - return flow - - -def write_flow(flow, filename): - """ - write optical flow in Middlebury .flo format - :param flow: optical flow map - :param filename: optical flow file path to be saved - :return: None - """ - f = open(filename, "wb") - magic = np.array([202021.25], dtype=np.float32) - (height, width) = flow.shape[0:2] - w = np.array([width], dtype=np.int32) - h = np.array([height], dtype=np.int32) - magic.tofile(f) - w.tofile(f) - h.tofile(f) - flow.tofile(f) - f.close() - - -def segment_flow(flow): - h = flow.shape[0] - w = flow.shape[1] - u = flow[:, :, 0] - v = flow[:, :, 1] - - idx = (abs(u) > LARGEFLOW) | (abs(v) > LARGEFLOW) - idx2 = abs(u) == SMALLFLOW - class0 = (v == 0) & (u == 0) - u[idx2] = 0.00001 - tan_value = v / u - - class1 = (tan_value < 1) & (tan_value >= 0) & (u > 0) & (v >= 0) - class2 = (tan_value >= 1) & (u >= 0) & (v >= 0) - class3 = (tan_value < -1) & (u <= 0) & (v >= 0) - class4 = (tan_value < 0) & (tan_value >= -1) & (u < 0) & (v >= 0) - class8 = (tan_value >= -1) & (tan_value < 0) & (u > 0) & (v <= 0) - class7 = (tan_value < -1) & (u >= 0) & (v <= 0) - class6 = (tan_value >= 1) & (u <= 0) & (v <= 0) - class5 = (tan_value >= 0) & (tan_value < 1) & (u < 0) & (v <= 0) - - seg = np.zeros((h, w)) - - seg[class1] = 1 - seg[class2] = 2 - seg[class3] = 3 - seg[class4] = 4 - seg[class5] = 5 - seg[class6] = 6 - seg[class7] = 7 - seg[class8] = 8 - seg[class0] = 0 - seg[idx] = 0 - - return seg - - -def flow_error(tu, tv, u, v): - """ - Calculate average end point error - :param tu: ground-truth horizontal flow map - :param tv: ground-truth vertical flow map - :param u: estimated horizontal flow map - :param v: estimated vertical flow map - :return: End point error of the estimated flow - """ - smallflow = 0.0 - """ - stu = tu[bord+1:end-bord,bord+1:end-bord] - stv = tv[bord+1:end-bord,bord+1:end-bord] - su = u[bord+1:end-bord,bord+1:end-bord] - sv = v[bord+1:end-bord,bord+1:end-bord] - """ - stu = tu[:] - stv = tv[:] - su = u[:] - sv = v[:] - - idxUnknow = (abs(stu) > UNKNOWN_FLOW_THRESH) | (abs(stv) > UNKNOWN_FLOW_THRESH) - stu[idxUnknow] = 0 - stv[idxUnknow] = 0 - su[idxUnknow] = 0 - sv[idxUnknow] = 0 - - ind2 = [(np.absolute(stu) > smallflow) | (np.absolute(stv) > smallflow)] - index_su = su[ind2] - index_sv = sv[ind2] - an = 1.0 / np.sqrt(index_su ** 2 + index_sv ** 2 + 1) - un = index_su * an - vn = index_sv * an - - index_stu = stu[ind2] - index_stv = stv[ind2] - tn = 1.0 / np.sqrt(index_stu ** 2 + index_stv ** 2 + 1) - tun = index_stu * tn - tvn = index_stv * tn - - """ - angle = un * tun + vn * tvn + (an * tn) - index = [angle == 1.0] - angle[index] = 0.999 - ang = np.arccos(angle) - mang = np.mean(ang) - mang = mang * 180 / np.pi - """ - - epe = np.sqrt((stu - su) ** 2 + (stv - sv) ** 2) - epe = epe[ind2] - mepe = np.mean(epe) - return mepe - - -def flow_to_image(flow, display=False): - """ - Convert flow into middlebury color code image - :param flow: optical flow map - :return: optical flow image in middlebury color - """ - u = flow[:, :, 0] - v = flow[:, :, 1] - - maxu = -999.0 - maxv = -999.0 - minu = 999.0 - minv = 999.0 - - idxUnknow = (abs(u) > UNKNOWN_FLOW_THRESH) | (abs(v) > UNKNOWN_FLOW_THRESH) - u[idxUnknow] = 0 - v[idxUnknow] = 0 - - maxu = max(maxu, np.max(u)) - minu = min(minu, np.min(u)) - - maxv = max(maxv, np.max(v)) - minv = min(minv, np.min(v)) - - rad = np.sqrt(u ** 2 + v ** 2) - maxrad = max(-1, np.max(rad)) - - if display: - print("max flow: %.4f\nflow range:\nu = %.3f .. %.3f\nv = %.3f .. %.3f" % (maxrad, minu, maxu, minv, maxv)) - - u = u / (maxrad + np.finfo(float).eps) - v = v / (maxrad + np.finfo(float).eps) - - img = compute_color(u, v) - - idx = np.repeat(idxUnknow[:, :, np.newaxis], 3, axis=2) - img[idx] = 0 - - return np.uint8(img) - - -def evaluate_flow_file(gt, pred): - """ - evaluate the estimated optical flow end point error according to ground truth provided - :param gt: ground truth file path - :param pred: estimated optical flow file path - :return: end point error, float32 - """ - # Read flow files and calculate the errors - gt_flow = read_flow(gt) # ground truth flow - eva_flow = read_flow(pred) # predicted flow - # Calculate errors - average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], eva_flow[:, :, 0], eva_flow[:, :, 1]) - return average_pe - - -def evaluate_flow(gt_flow, pred_flow): - """ - gt: ground-truth flow - pred: estimated flow - """ - average_pe = flow_error(gt_flow[:, :, 0], gt_flow[:, :, 1], pred_flow[:, :, 0], pred_flow[:, :, 1]) - return average_pe - - -""" -============== -Disparity Section -============== -""" - - -def read_disp_png(file_name): - """ - Read optical flow from KITTI .png file - :param file_name: name of the flow file - :return: optical flow data in matrix - """ - image_object = png.Reader(filename=file_name) - image_direct = image_object.asDirect() - image_data = list(image_direct[2]) - (w, h) = image_direct[3]["size"] - channel = len(image_data[0]) / w - flow = np.zeros((h, w, channel), dtype=np.uint16) - for i in range(len(image_data)): - for j in range(channel): - flow[i, :, j] = image_data[i][j::channel] - return flow[:, :, 0] / 256 - - -def disp_to_flowfile(disp, filename): - """ - Read KITTI disparity file in png format - :param disp: disparity matrix - :param filename: the flow file name to save - :return: None - """ - f = open(filename, "wb") - magic = np.array([202021.25], dtype=np.float32) - (height, width) = disp.shape[0:2] - w = np.array([width], dtype=np.int32) - h = np.array([height], dtype=np.int32) - empty_map = np.zeros((height, width), dtype=np.float32) - data = np.dstack((disp, empty_map)) - magic.tofile(f) - w.tofile(f) - h.tofile(f) - data.tofile(f) - f.close() - - -""" -============== -Image Section -============== -""" - - -def read_image(filename): - """ - Read normal image of any format - :param filename: name of the image file - :return: image data in matrix uint8 type - """ - img = Image.open(filename) - im = np.array(img) - return im - - -def warp_image(im, flow): - """ - Use optical flow to warp image to the next - :param im: image to warp - :param flow: optical flow - :return: warped image - """ - from scipy import interpolate - - image_height = im.shape[0] - image_width = im.shape[1] - flow_height = flow.shape[0] - flow_width = flow.shape[1] - n = image_height * image_width - (iy, ix) = np.float32(np.mgrid[0:image_height, 0:image_width]) - (fy, fx) = np.float32(np.mgrid[0:flow_height, 0:flow_width]) - fx += flow[:, :, 0].astype(np.float) - fy += flow[:, :, 1] - mask = np.logical_or(fx < 0, fx > flow_width) - mask = np.logical_or(mask, fy < 0) - mask = np.logical_or(mask, fy > flow_height) - fx = np.minimum(np.maximum(fx, 0), flow_width) - fy = np.minimum(np.maximum(fy, 0), flow_height) - points = np.concatenate((ix.reshape(n, 1), iy.reshape(n, 1)), axis=1) - xi = np.concatenate((fx.reshape(n, 1), fy.reshape(n, 1)), axis=1) - warp = np.zeros((image_height, image_width, im.shape[2])) - for i in range(im.shape[2]): - channel = im[:, :, i] - # plt.imshow(channel, cmap='gray') - values = channel.reshape(n, 1) - new_channel = interpolate.griddata(points, values, xi, method="cubic") - new_channel = np.reshape(new_channel, [flow_height, flow_width]) - new_channel[mask] = 1 - warp[:, :, i] = new_channel.astype(np.uint8) - - return warp.astype(np.uint8) - - -""" -============== -Others -============== -""" - - -def scale_image(image, new_range): - """ - Linearly scale the image into desired range - :param image: input image - :param new_range: the new range to be aligned - :return: image normalized in new range - """ - min_val = np.min(image).astype(np.float32) - max_val = np.max(image).astype(np.float32) - min_val_new = np.array(min(new_range), dtype=np.float32) - max_val_new = np.array(max(new_range), dtype=np.float32) - scaled_image = (image - min_val) / (max_val - min_val) * (max_val_new - min_val_new) + min_val_new - return scaled_image.astype(np.uint8) - - -def compute_color(u, v): - """ - compute optical flow color map - :param u: optical flow horizontal map - :param v: optical flow vertical map - :return: optical flow in color code - """ - [h, w] = u.shape - img = np.zeros([h, w, 3]) - nanIdx = np.isnan(u) | np.isnan(v) - u[nanIdx] = 0 - v[nanIdx] = 0 - - colorwheel = make_color_wheel() - ncols = np.size(colorwheel, 0) - - rad = np.sqrt(u ** 2 + v ** 2) - - a = np.arctan2(-v, -u) / np.pi - - fk = (a + 1) / 2 * (ncols - 1) + 1 - - k0 = np.floor(fk).astype(int) - - k1 = k0 + 1 - k1[k1 == ncols + 1] = 1 - f = fk - k0 - - for i in range(0, np.size(colorwheel, 1)): - tmp = colorwheel[:, i] - col0 = tmp[k0 - 1] / 255 - col1 = tmp[k1 - 1] / 255 - col = (1 - f) * col0 + f * col1 - - idx = rad <= 1 - col[idx] = 1 - rad[idx] * (1 - col[idx]) - notidx = np.logical_not(idx) - - col[notidx] *= 0.75 - img[:, :, i] = np.uint8(np.floor(255 * col * (1 - nanIdx))) - - return img - - -def make_color_wheel(): - """ - Generate color wheel according Middlebury color code - :return: Color wheel - """ - RY = 15 - YG = 6 - GC = 4 - CB = 11 - BM = 13 - MR = 6 - - ncols = RY + YG + GC + CB + BM + MR - - colorwheel = np.zeros([ncols, 3]) - - col = 0 - - # RY - colorwheel[0:RY, 0] = 255 - colorwheel[0:RY, 1] = np.transpose(np.floor(255 * np.arange(0, RY) / RY)) - col += RY - - # YG - colorwheel[col : col + YG, 0] = 255 - np.transpose(np.floor(255 * np.arange(0, YG) / YG)) - colorwheel[col : col + YG, 1] = 255 - col += YG - - # GC - colorwheel[col : col + GC, 1] = 255 - colorwheel[col : col + GC, 2] = np.transpose(np.floor(255 * np.arange(0, GC) / GC)) - col += GC - - # CB - colorwheel[col : col + CB, 1] = 255 - np.transpose(np.floor(255 * np.arange(0, CB) / CB)) - colorwheel[col : col + CB, 2] = 255 - col += CB - - # BM - colorwheel[col : col + BM, 2] = 255 - colorwheel[col : col + BM, 0] = np.transpose(np.floor(255 * np.arange(0, BM) / BM)) - col += +BM - - # MR - colorwheel[col : col + MR, 2] = 255 - np.transpose(np.floor(255 * np.arange(0, MR) / MR)) - colorwheel[col : col + MR, 0] = 255 - - return colorwheel diff --git a/utils/hist_match.py b/utils/hist_match.py deleted file mode 100644 index c9271ed..0000000 --- a/utils/hist_match.py +++ /dev/null @@ -1,101 +0,0 @@ -import numpy as np -from matplotlib import pyplot as plt - - -def hist_match_module(source, template): - """ - Adjust the pixel values of a grayscale image such that its histogram - matches that of a target image - - Arguments: - ----------- - source: np.ndarray - Image to transform; the histogram is computed over the flattened - array - template: np.ndarray - Template image; can have different dimensions to source - Returns: - ----------- - matched: np.ndarray - The transformed output image - """ - - oldshape = source.shape - source = source.ravel() - template = template.ravel() - - # get the set of unique pixel values and their corresponding indices and - # counts - s_values, bin_idx, s_counts = np.unique(source, return_inverse=True, return_counts=True) - t_values, t_counts = np.unique(template, return_counts=True) - - # take the cumsum of the counts and normalize by the number of pixels to - # get the empirical cumulative distribution functions for the source and - # template images (maps pixel value --> quantile) - s_quantiles = np.cumsum(s_counts).astype(np.float64) - s_quantiles /= s_quantiles[-1] - t_quantiles = np.cumsum(t_counts).astype(np.float64) - t_quantiles /= t_quantiles[-1] - - # interpolate linearly to find the pixel values in the template image - # that correspond most closely to the quantiles in the source image - interp_t_values = np.interp(s_quantiles, t_quantiles, t_values) - - return interp_t_values[bin_idx].reshape(oldshape) - - -def hist_match(image_name1, image_name2, output_name): - """ - Adjust the pixel values of a grayscale image such that its histogram - matches that of a target image - - Arguments: - ----------- - image name1, image name2 - Returns: - ----------- - output dir + imagename - """ - image1 = plt.imread(image_name1) - image2 = plt.imread(image_name2) - # source = face()[:,:,0] - # template = ascent() - # matched = hist_match(source, template) - - # def ecdf(x): - # """convenience function for computing the empirical CDF""" - # vals, counts = np.unique(x, return_counts=True) - # ecdf = np.cumsum(counts).astype(np.float64) - # ecdf /= ecdf[-1] - # return vals, ecdf - - # x1, y1 = ecdf(source.ravel()) - # x2, y2 = ecdf(template.ravel()) - # x3, y3 = ecdf(matched.ravel()) - - # fig = plt.figure() - # gs = plt.GridSpec(2, 3) - # ax1 = fig.add_subplot(gs[0, 0]) - # ax2 = fig.add_subplot(gs[0, 1], sharex=ax1, sharey=ax1) - # ax3 = fig.add_subplot(gs[0, 2], sharex=ax1, sharey=ax1) - # ax4 = fig.add_subplot(gs[1, :]) - # for aa in (ax1, ax2, ax3): - # aa.set_axis_off() - - # ax1.imshow(source, cmap=plt.cm.gray) - # ax1.set_title('Source') - # ax2.imshow(template, cmap=plt.cm.gray) - # ax2.set_title('template') - # ax3.imshow(matched, cmap=plt.cm.gray) - # ax3.set_title('Matched') - - # ax4.plot(x1, y1 * 100, '-r', lw=3, label='Source') - # ax4.plot(x2, y2 * 100, '-k', lw=3, label='Template') - # ax4.plot(x3, y3 * 100, '--r', lw=3, label='Matched') - # ax4.set_xlim(x1[0], x1[-1]) - # ax4.set_xlabel('Pixel value') - # ax4.set_ylabel('Cumulative %') - # ax4.legend(loc=5) - - # plt.show() - # print('finished testing') diff --git a/utils/parallel.py b/utils/parallel.py deleted file mode 100644 index af9cf5a..0000000 --- a/utils/parallel.py +++ /dev/null @@ -1,286 +0,0 @@ -##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -## Created by: Hang Zhang, Rutgers University, Email: zhang.hang@rutgers.edu -## Modified by Thomas Wolf, HuggingFace Inc., Email: thomas@huggingface.co -## Copyright (c) 2017-2018 -## -## This source code is licensed under the MIT-style license found in the -## LICENSE file in the root directory of this source tree -##+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ -"""Encoding Data Parallel""" -import functools -import threading - -import torch -import torch.cuda.comm as comm -from torch.autograd import Function, Variable -from torch.nn.parallel import DistributedDataParallel -from torch.nn.parallel._functions import Broadcast, ReduceAddCoalesced -from torch.nn.parallel.data_parallel import DataParallel -from torch.nn.parallel.parallel_apply import get_a_var -from torch.nn.parallel.scatter_gather import gather - -torch_ver = torch.__version__[:3] - -__all__ = ["allreduce", "DataParallelModel", "DataParallelCriterion", "patch_replication_callback"] - - -def allreduce(*inputs): - """Cross GPU all reduce autograd operation for calculate mean and - variance in SyncBN. - """ - return AllReduce.apply(*inputs) - - -class AllReduce(Function): - @staticmethod - def forward(ctx, num_inputs, *inputs): - ctx.num_inputs = num_inputs - ctx.target_gpus = [inputs[i].get_device() for i in range(0, len(inputs), num_inputs)] - inputs = [inputs[i : i + num_inputs] for i in range(0, len(inputs), num_inputs)] - # sort before reduce sum - inputs = sorted(inputs, key=lambda i: i[0].get_device()) - results = comm.reduce_add_coalesced(inputs, ctx.target_gpus[0]) - outputs = comm.broadcast_coalesced(results, ctx.target_gpus) - return tuple([t for tensors in outputs for t in tensors]) - - @staticmethod - def backward(ctx, *inputs): - inputs = [i.data for i in inputs] - inputs = [inputs[i : i + ctx.num_inputs] for i in range(0, len(inputs), ctx.num_inputs)] - results = comm.reduce_add_coalesced(inputs, ctx.target_gpus[0]) - outputs = comm.broadcast_coalesced(results, ctx.target_gpus) - return (None,) + tuple([Variable(t) for tensors in outputs for t in tensors]) - - -class Reduce(Function): - @staticmethod - def forward(ctx, *inputs): - ctx.target_gpus = [inputs[i].get_device() for i in range(len(inputs))] - inputs = sorted(inputs, key=lambda i: i.get_device()) - return comm.reduce_add(inputs) - - @staticmethod - def backward(ctx, gradOutput): - return Broadcast.apply(ctx.target_gpus, gradOutput) - - -class DistributedDataParallelModel(DistributedDataParallel): - """Implements data parallelism at the module level for the DistributedDataParallel module. - This container parallelizes the application of the given module by - splitting the input across the specified devices by chunking in the - batch dimension. - In the forward pass, the module is replicated on each device, - and each replica handles a portion of the input. During the backwards pass, - gradients from each replica are summed into the original module. - Note that the outputs are not gathered, please use compatible - :class:`encoding.parallel.DataParallelCriterion`. - The batch size should be larger than the number of GPUs used. It should - also be an integer multiple of the number of GPUs so that each chunk is - the same size (so that each GPU processes the same number of samples). - Args: - module: module to be parallelized - device_ids: CUDA devices (default: all devices) - Reference: - Hang Zhang, Kristin Dana, Jianping Shi, Zhongyue Zhang, Xiaogang Wang, Ambrish Tyagi, - Amit Agrawal. “Context Encoding for Semantic Segmentation. - *The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) 2018* - Example:: - >>> net = encoding.nn.DistributedDataParallelModel(model, device_ids=[0, 1, 2]) - >>> y = net(x) - """ - - def gather(self, outputs, output_device): - return outputs - - -class DataParallelModel(DataParallel): - """Implements data parallelism at the module level. - - This container parallelizes the application of the given module by - splitting the input across the specified devices by chunking in the - batch dimension. - In the forward pass, the module is replicated on each device, - and each replica handles a portion of the input. During the backwards pass, - gradients from each replica are summed into the original module. - Note that the outputs are not gathered, please use compatible - :class:`encoding.parallel.DataParallelCriterion`. - - The batch size should be larger than the number of GPUs used. It should - also be an integer multiple of the number of GPUs so that each chunk is - the same size (so that each GPU processes the same number of samples). - - Args: - module: module to be parallelized - device_ids: CUDA devices (default: all devices) - - Reference: - Hang Zhang, Kristin Dana, Jianping Shi, Zhongyue Zhang, Xiaogang Wang, Ambrish Tyagi, - Amit Agrawal. “Context Encoding for Semantic Segmentation. - *The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) 2018* - - Example:: - - >>> net = encoding.nn.DataParallelModel(model, device_ids=[0, 1, 2]) - >>> y = net(x) - """ - - def gather(self, outputs, output_device): - return outputs - - def replicate(self, module, device_ids): - modules = super(DataParallelModel, self).replicate(module, device_ids) - execute_replication_callbacks(modules) - return modules - - -class DataParallelCriterion(DataParallel): - """ - Calculate loss in multiple-GPUs, which balance the memory usage. - The targets are splitted across the specified devices by chunking in - the batch dimension. Please use together with :class:`encoding.parallel.DataParallelModel`. - - Reference: - Hang Zhang, Kristin Dana, Jianping Shi, Zhongyue Zhang, Xiaogang Wang, Ambrish Tyagi, - Amit Agrawal. “Context Encoding for Semantic Segmentation. - *The IEEE Conference on Computer Vision and Pattern Recognition (CVPR) 2018* - - Example:: - - >>> net = encoding.nn.DataParallelModel(model, device_ids=[0, 1, 2]) - >>> criterion = encoding.nn.DataParallelCriterion(criterion, device_ids=[0, 1, 2]) - >>> y = net(x) - >>> loss = criterion(y, target) - """ - - def forward(self, inputs, *targets, **kwargs): - # input should be already scatterd - # scattering the targets instead - if not self.device_ids: - return self.module(inputs, *targets, **kwargs) - targets, kwargs = self.scatter(targets, kwargs, self.device_ids) - if len(self.device_ids) == 1: - return self.module(inputs, *targets[0], **kwargs[0]) - replicas = self.replicate(self.module, self.device_ids[: len(inputs)]) - outputs = _criterion_parallel_apply(replicas, inputs, targets, kwargs) - # return Reduce.apply(*outputs) / len(outputs) - # return self.gather(outputs, self.output_device).mean() - return self.gather(outputs, self.output_device) - - -def _criterion_parallel_apply(modules, inputs, targets, kwargs_tup=None, devices=None): - assert len(modules) == len(inputs) - assert len(targets) == len(inputs) - if kwargs_tup: - assert len(modules) == len(kwargs_tup) - else: - kwargs_tup = ({},) * len(modules) - if devices is not None: - assert len(modules) == len(devices) - else: - devices = [None] * len(modules) - - lock = threading.Lock() - results = {} - if torch_ver != "0.3": - grad_enabled = torch.is_grad_enabled() - - def _worker(i, module, input, target, kwargs, device=None): - if torch_ver != "0.3": - torch.set_grad_enabled(grad_enabled) - if device is None: - device = get_a_var(input).get_device() - try: - with torch.cuda.device(device): - # this also avoids accidental slicing of `input` if it is a Tensor - if not isinstance(input, (list, tuple)): - input = (input,) - if not isinstance(target, (list, tuple)): - target = (target,) - output = module(*(input + target), **kwargs) - with lock: - results[i] = output - except Exception as e: - with lock: - results[i] = e - - if len(modules) > 1: - threads = [ - threading.Thread(target=_worker, args=(i, module, input, target, kwargs, device),) - for i, (module, input, target, kwargs, device) in enumerate( - zip(modules, inputs, targets, kwargs_tup, devices) - ) - ] - - for thread in threads: - thread.start() - for thread in threads: - thread.join() - else: - _worker(0, modules[0], inputs[0], kwargs_tup[0], devices[0]) - - outputs = [] - for i in range(len(inputs)): - output = results[i] - if isinstance(output, Exception): - raise output - outputs.append(output) - return outputs - - -########################################################################### -# Adapted from Synchronized-BatchNorm-PyTorch. -# https://github.com/vacancy/Synchronized-BatchNorm-PyTorch -# -class CallbackContext(object): - pass - - -def execute_replication_callbacks(modules): - """ - Execute an replication callback `__data_parallel_replicate__` on each module created - by original replication. - - The callback will be invoked with arguments `__data_parallel_replicate__(ctx, copy_id)` - - Note that, as all modules are isomorphism, we assign each sub-module with a context - (shared among multiple copies of this module on different devices). - Through this context, different copies can share some information. - - We guarantee that the callback on the master copy (the first copy) will be called ahead - of calling the callback of any slave copies. - """ - master_copy = modules[0] - nr_modules = len(list(master_copy.modules())) - ctxs = [CallbackContext() for _ in range(nr_modules)] - - for i, module in enumerate(modules): - for j, m in enumerate(module.modules()): - if hasattr(m, "__data_parallel_replicate__"): - m.__data_parallel_replicate__(ctxs[j], i) - - -def patch_replication_callback(data_parallel): - """ - Monkey-patch an existing `DataParallel` object. Add the replication callback. - Useful when you have customized `DataParallel` implementation. - - Examples: - > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) - > sync_bn = DataParallel(sync_bn, device_ids=[0, 1]) - > patch_replication_callback(sync_bn) - # this is equivalent to - > sync_bn = SynchronizedBatchNorm1d(10, eps=1e-5, affine=False) - > sync_bn = DataParallelWithCallback(sync_bn, device_ids=[0, 1]) - """ - - assert isinstance(data_parallel, DataParallel) - - old_replicate = data_parallel.replicate - - @functools.wraps(old_replicate) - def new_replicate(module, device_ids): - modules = old_replicate(module, device_ids) - execute_replication_callbacks(modules) - return modules - - data_parallel.replicate = new_replicate diff --git a/utils/tb_image_recorder.py b/utils/tb_image_recorder.py deleted file mode 100644 index 26d3ee1..0000000 --- a/utils/tb_image_recorder.py +++ /dev/null @@ -1,29 +0,0 @@ -import threading - -import numpy as np - - -class TBImageRecorder(threading.Thread): - """ - TBImageRecorder - """ - - def __init__(self, tb_writer, func, queue): - super(TBImageRecorder, self).__init__() - self._tb_writer = tb_writer - # self._tb_writer = SummaryWriter() - self._func = func - self._queue = queue - - def run(self): - while True: - msgs, iter_index = self._queue.get() - if msgs: - img_info = self._func(*msgs) - - print("logging the images") - for tag, images in img_info.items(): - if images is not None: - self._tb_writer.add_image(tag, np.clip(images, 0, 255).astype(np.uint8), iter_index) - else: - break diff --git a/utils/tensor_lab2rgb.py b/utils/tensor_lab2rgb.py index f162543..c5b5533 100755 --- a/utils/tensor_lab2rgb.py +++ b/utils/tensor_lab2rgb.py @@ -1,14 +1,9 @@ -""""xyz2rgb(lab2xyz(lab, illuminant, observer))""" import numpy as np import torch -from scipy import linalg -from torch.autograd import Variable xyz_from_rgb = np.array( [[0.412453, 0.357580, 0.180423], [0.212671, 0.715160, 0.072169], [0.019334, 0.119193, 0.950227]] ) - -# rgb_from_xyz = linalg.inv(xyz_from_rgb).T rgb_from_xyz = np.array( [[3.24048134, -0.96925495, 0.05564664], [-1.53715152, 1.87599, -0.20404134], [-0.49853633, 0.04155593, 1.05731107]] ) @@ -26,18 +21,12 @@ def tensor_lab2rgb(input): neg_mask = z.data < 0 z[neg_mask] = 0 - # if neg_mask.any(): - # z[neg_mask] = 0 xyz = torch.cat((x, y, z), dim=3) mask = xyz.data > 0.2068966 mask_xyz = xyz.clone() mask_xyz[mask] = torch.pow(xyz[mask], 3.0) mask_xyz[~mask] = (xyz[~mask] - 16.0 / 116.0) / 7.787 - # if mask.any(): - # mask_xyz[mask] = torch.pow(xyz[mask], 3.) - # if not mask.all(): - # mask_xyz[~mask] = (xyz[~mask] - 16.0 / 116.) / 7.787 mask_xyz[:, :, :, 0] = mask_xyz[:, :, :, 0] * 0.95047 mask_xyz[:, :, :, 2] = mask_xyz[:, :, :, 2] * 1.08883 @@ -50,18 +39,9 @@ def tensor_lab2rgb(input): mask_rgb = rgb.clone() mask_rgb[mask] = 1.055 * torch.pow(rgb[mask], 1 / 2.4) - 0.055 mask_rgb[~mask] = rgb[~mask] * 12.92 - # if mask.any(): - # mask_rgb[mask] = 1.055 * torch.pow(rgb[mask], 1 / 2.4) - 0.055 - # if not mask.all(): - # mask_rgb[~mask] = rgb[~mask] * 12.92 neg_mask = mask_rgb.data < 0 large_mask = mask_rgb.data > 1 mask_rgb[neg_mask] = 0 mask_rgb[large_mask] = 1 - # if neg_mask.any(): - # mask_rgb[neg_mask] = 0 - # if large_mask.any(): - # mask_rgb[large_mask] = 1 - return mask_rgb diff --git a/utils/tensorboard_logger.py b/utils/tensorboard_logger.py deleted file mode 100755 index 78acc37..0000000 --- a/utils/tensorboard_logger.py +++ /dev/null @@ -1,88 +0,0 @@ -# Code referenced from https://gist.github.com/gyglim/1f8dfb1b5c82627ae3efcfbbadb9f514 -import numpy as np -import scipy.misc -import tensorflow as tf - -try: - from StringIO import StringIO # Python 2.7 -except ImportError: - from io import BytesIO # Python 3.x - - -class TBLogger(object): - def __init__(self, log_dir): - """Create a summary writer logging to log_dir.""" - self.writer = tf.summary.FileWriter(log_dir) - - def scalar_summary(self, tag, value, step): - """Log a scalar variable.""" - summary = tf.Summary(value=[tf.Summary.Value(tag=tag, simple_value=value)]) - self.writer.add_summary(summary, step) - - def image_summaries(self, tag, images, step): - """Log a list of images.""" - - img_summaries = [] - for i, img in enumerate(images): - # Write the image to a string - try: - s = StringIO() - except: - s = BytesIO() - scipy.misc.toimage(img).save(s, format="png") - - # Create an Image object - img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), height=img.shape[0], width=img.shape[1]) - # Create a Summary value - img_summaries.append(tf.Summary.Value(tag="%s/%d" % (tag, i), image=img_sum)) - - # Create and write Summary - summary = tf.Summary(value=img_summaries) - self.writer.add_summary(summary, step) - - def image_summary(self, tag, image, step): - """Log a list of images.""" - - img_summaries = [] - - # Write the image to a string - try: - s = StringIO() - except: - s = BytesIO() - - scipy.misc.toimage(image).save(s, format="png") - - # Create an Image object - img_sum = tf.Summary.Image(encoded_image_string=s.getvalue(), height=image.shape[0], width=image.shape[1]) - # Create and write Summary - summary = tf.Summary(value=[tf.Summary.Value(tag=tag, image=img_sum)]) - self.writer.add_summary(summary, step) - - def histo_summary(self, tag, values, step, bins=1000): - """Log a histogram of the tensor of values.""" - - # Create a histogram using numpy - counts, bin_edges = np.histogram(values, bins=bins) - - # Fill the fields of the histogram proto - hist = tf.HistogramProto() - hist.min = float(np.min(values)) - hist.max = float(np.max(values)) - hist.num = int(np.prod(values.shape)) - hist.sum = float(np.sum(values)) - hist.sum_squares = float(np.sum(values ** 2)) - - # Drop the start of the first bin - bin_edges = bin_edges[1:] - - # Add bin edges and counts - for edge in bin_edges: - hist.bucket_limit.append(edge) - for c in counts: - hist.bucket.append(c) - - # Create and write Summary - summary = tf.Summary(value=[tf.Summary.Value(tag=tag, histo=hist)]) - self.writer.add_summary(summary, step) - self.writer.flush() diff --git a/utils/test_html.py b/utils/test_html.py deleted file mode 100644 index 1df27c5..0000000 --- a/utils/test_html.py +++ /dev/null @@ -1,66 +0,0 @@ -import os - -import dominate -from dominate.tags import * - - -class HTML: - def __init__(self, web_dir, title, reflesh=0): - self.title = title - self.web_dir = web_dir - self.img_dir = os.path.join(self.web_dir, "images") - if not os.path.exists(self.web_dir): - os.makedirs(self.web_dir) - if not os.path.exists(self.img_dir): - os.makedirs(self.img_dir) - # print(self.img_dir) - - self.doc = dominate.document(title=title) - if reflesh > 0: - with self.doc.head: - meta(http_equiv="reflesh", content=str(reflesh)) - - def get_image_dir(self): - return self.img_dir - - def add_header(self, str): - with self.doc: - h3(str) - - def add_table(self, border=1): - self.t = table(border=border, style="table-layout: fixed;") - self.doc.add(self.t) - - def add_images(self, ims, txts, links, width=400): - self.add_table() - with self.t: - with tr(): - for im, txt, link in zip(ims, txts, links): - with td(style="word-wrap: break-word;", halign="center", valign="top"): - with p(): - with a(href=os.path.join("images", link)): - img(style="width:%dpx" % width, src=os.path.join("images", im)) - br() - p(txt) - - def save(self): - html_file = "%s/index.html" % self.web_dir - f = open(html_file, "wt") - f.write(self.doc.render()) - f.close() - - -if __name__ == "__main__": - html = HTML("web/", "test_html") - html.add_header("hello world") - - ims = [] - txts = [] - links = [] - for n in range(4): - ims.append("image_%d.jpg" % n) - txts.append("text_%d" % n) - links.append("image_%d.jpg" % n) - html.add_images(ims, txts, links) - html.add_images(ims, txts, links) - html.save() diff --git a/utils/test_lms_edge.py b/utils/test_lms_edge.py deleted file mode 100644 index 6a04206..0000000 --- a/utils/test_lms_edge.py +++ /dev/null @@ -1,48 +0,0 @@ -import matplotlib.pyplot as plt -import numpy as np -import skimage -from skimage import color, feature, io, morphology -from skimage.filters import prewitt, roberts, scharr, sobel -from skimage.morphology import square - - -def rgb2lms(rgb_image): - lms_image = rgb_image - r = rgb_image[:, :, 0] - g = rgb_image[:, :, 1] - b = rgb_image[:, :, 2] - - l = 17.8824 * r + 43.5161 * g + 4.1194 * b - m = 3.4557 * r + 27.1554 * g + 3.8671 * b - s = 0.0300 * r + 0.1843 * g + 1.4671 * b - lms_image = np.stack((l, m, s), axis=2) - return l, m, s - - -def image_normalize(rg_image): - rg_image = rg_image - rg_image.min() - rg_image = rg_image / (rg_image.max() - rg_image.min()) * 255 - return rg_image - - -image_rgb = io.imread("ref5.jpg") -plt.imshow(image_rgb, cmap="gray") -L, M, S = rgb2lms(image_rgb) -image_lab = color.rgb2lab(image_rgb) -a_edge = sobel(image_lab[:, :, 1]) -b_edge = sobel(image_lab[:, :, 2]) -color_edge = np.sqrt(a_edge ** 2 + b_edge ** 2) -luminance_edge = sobel(image_lab[:, :, 0]) -canny_edge_a = feature.canny(image_lab[:, :, 1], sigma=3, low_threshold=3) -canny_edge_b = feature.canny(image_lab[:, :, 2], sigma=3, low_threshold=3) - -plt.figure() -# edge = morphology.opening(np.logical_and(color_edge > 8, luminance_edge > 4)) -edge = morphology.thin( - morphology.closing((morphology.opening(np.logical_and(color_edge > 10, luminance_edge > 2))), square(1)) -) -plt.imshow(edge, cmap="gray") - -plt.figure() -plt.imshow(np.logical_and(canny_edge_a, canny_edge_b), cmap="gray") -plt.show() diff --git a/utils/util.py b/utils/util.py index 0a30212..d510d50 100755 --- a/utils/util.py +++ b/utils/util.py @@ -1,8 +1,6 @@ -import numbers import os import shutil import sys -import time import cv2 import matplotlib.pyplot as plt @@ -55,8 +53,8 @@ class MovingAvg(object): ###### image normalization ###### -# normalization for l def center_l(l): + # normalization for l l_mc = (l - l_mean) / l_norm return l_mc @@ -68,31 +66,28 @@ def uncenter_l(l): # normalization for ab def center_ab(ab): - ab_mc = (ab - ab_mean) / ab_norm - return ab_mc + return (ab - ab_mean) / ab_norm # normalization for lab image def center_lab_img(img_lab): - img_lab_mc = ( + return ( img_lab / np.array((l_norm, ab_norm, ab_norm))[:, np.newaxis, np.newaxis] - np.array((l_mean / l_norm, ab_mean / ab_norm, ab_mean / ab_norm))[:, np.newaxis, np.newaxis] ) - return img_lab_mc ###### color space transformation ###### def rgb2lab_transpose(img_rgb): - img_lab = color.rgb2lab(img_rgb).transpose((2, 0, 1)) - return img_lab + return color.rgb2lab(img_rgb).transpose((2, 0, 1)) def lab2rgb(img_l, img_ab): - """ INPUTS - img_l XxXx1 [0,100] - img_ab XxXx2 [-100,100] - OUTPUTS - returned value is XxXx3 """ + """INPUTS + img_l XxXx1 [0,100] + img_ab XxXx2 [-100,100] + OUTPUTS + returned value is XxXx3""" pred_lab = np.concatenate((img_l, img_ab), axis=2).astype("float64") pred_rgb = color.lab2rgb(pred_lab) pred_rgb = (np.clip(pred_rgb, 0, 1) * 255).astype("uint8") @@ -107,14 +102,13 @@ def gray2rgb_batch(l): def lab2rgb_transpose(img_l, img_ab): - """ INPUTS - img_l 1xXxX [0,100] - img_ab 2xXxX [-100,100] - OUTPUTS - returned value is XxXx3 """ + """INPUTS + img_l 1xXxX [0,100] + img_ab 2xXxX [-100,100] + OUTPUTS + returned value is XxXx3""" pred_lab = np.concatenate((img_l, img_ab), axis=0).transpose((1, 2, 0)) - pred_rgb = (np.clip(color.lab2rgb(pred_lab), 0, 1) * 255).astype("uint8") - return pred_rgb + return (np.clip(color.lab2rgb(pred_lab), 0, 1) * 255).astype("uint8") def lab2rgb_transpose_mc(img_l_mc, img_ab_mc): @@ -134,8 +128,7 @@ def lab2rgb_transpose_mc(img_l_mc, img_ab_mc): img_ab = img_ab_mc * ab_norm + ab_mean pred_lab = torch.cat((img_l, img_ab), dim=0) grid_lab = pred_lab.numpy().astype("float64") - grid_rgb = (np.clip(color.lab2rgb(grid_lab.transpose((1, 2, 0))), 0, 1) * 255).astype("uint8") - return grid_rgb + return (np.clip(color.lab2rgb(grid_lab.transpose((1, 2, 0))), 0, 1) * 255).astype("uint8") def batch_lab2rgb_transpose_mc(img_l_mc, img_ab_mc, nrow=8): @@ -155,8 +148,7 @@ def batch_lab2rgb_transpose_mc(img_l_mc, img_ab_mc, nrow=8): img_ab = img_ab_mc * ab_norm + ab_mean pred_lab = torch.cat((img_l, img_ab), dim=1) grid_lab = vutils.make_grid(pred_lab, nrow=nrow).numpy().astype("float64") - grid_rgb = (np.clip(color.lab2rgb(grid_lab.transpose((1, 2, 0))), 0, 1) * 255).astype("uint8") - return grid_rgb + return (np.clip(color.lab2rgb(grid_lab.transpose((1, 2, 0))), 0, 1) * 255).astype("uint8") ###### loss functions ###### @@ -205,8 +197,7 @@ def calc_ab_gradient(input_ab): def calc_tv_loss(input): x_grad = input[:, :, :, 1:] - input[:, :, :, :-1] y_grad = input[:, :, 1:, :] - input[:, :, :-1, :] - loss = torch.sum(x_grad ** 2) / x_grad.nelement() + torch.sum(y_grad ** 2) / y_grad.nelement() - return loss + return torch.sum(x_grad ** 2) / x_grad.nelement() + torch.sum(y_grad ** 2) / y_grad.nelement() def calc_cosine_dist_loss(input, target): @@ -221,15 +212,13 @@ def calc_cosine_dist_loss(input, target): def weighted_mse_loss(input, target, weights): out = (input - target) ** 2 out = out * weights.expand_as(out) - loss = out.mean() - return loss + return out.mean() def weighted_l1_loss(input, target, weights): out = torch.abs(input - target) out = out * weights.expand_as(out) - loss = out.mean() - return loss + return out.mean() def colorfulness(input_ab): @@ -250,8 +239,7 @@ def colorfulness(input_ab): mean_a = torch.mean(a, dim=-1) mean_b = torch.mean(b, dim=-1) - colorfulness = torch.sqrt(sigma_a ** 2 + sigma_b ** 2) + 0.37 * torch.sqrt(mean_a ** 2 + mean_b ** 2) - return colorfulness + return torch.sqrt(sigma_a ** 2 + sigma_b ** 2) + 0.37 * torch.sqrt(mean_a ** 2 + mean_b ** 2) ###### video related ####### @@ -270,16 +258,14 @@ def folder2vid(image_folder, output_dir, filename): print(images) frame = cv2.imread(os.path.join(image_folder, images[0])) height, width, layers = frame.shape - # video = cv2.VideoWriter(os.path.join(output_dir, filename), 0x00000021, 24, (width, height)) print("writing to video " + os.path.join(output_dir, filename)) video = cv2.VideoWriter( os.path.join(output_dir, filename), cv2.VideoWriter_fourcc("D", "I", "V", "X"), 24, (width, height) ) - # video = cv2.VideoWriter(os.path.join(output_dir, filename), cv2.VideoWriter_fourcc('H', '2', '6', '4'), 24, (width, height)) + for image in images: video.write(cv2.imread(os.path.join(image_folder, image))) - # cv2.destroyAllWindows() video.release() # import imageio @@ -363,10 +349,8 @@ def vgg_preprocess(tensor): # input is RGB tensor which ranges in [0,1] # output is BGR tensor which ranges in [0,255] tensor_bgr = torch.cat((tensor[:, 2:3, :, :], tensor[:, 1:2, :, :], tensor[:, 0:1, :, :]), dim=1) - # tensor_bgr = tensor[:, [2, 1, 0], ...] tensor_bgr_ml = tensor_bgr - torch.Tensor([0.40760392, 0.45795686, 0.48501961]).type_as(tensor_bgr).view(1, 3, 1, 1) - tensor_rst = tensor_bgr_ml * 255 - return tensor_rst + return tensor_bgr_ml * 255 def torch_vgg_preprocess(tensor): @@ -375,48 +359,57 @@ def torch_vgg_preprocess(tensor): # input and output ranges in [0,1] # normalize the tensor with mean and variance tensor_mc = tensor - torch.Tensor([0.485, 0.456, 0.406]).type_as(tensor).view(1, 3, 1, 1) - tensor_mc_norm = tensor_mc / torch.Tensor([0.229, 0.224, 0.225]).type_as(tensor_mc).view(1, 3, 1, 1) - return tensor_mc_norm + return tensor_mc / torch.Tensor([0.229, 0.224, 0.225]).type_as(tensor_mc).view(1, 3, 1, 1) def network_gradient(net, gradient_on=True): - if gradient_on: - for param in net.parameters(): - param.requires_grad = True - else: - for param in net.parameters(): - param.requires_grad = False + for param in net.parameters(): + param.requires_grad = bool(gradient_on) return net -# def load_gray_image(img_path): -# img_path = utf8_str(img_path) -# img_gray = cv2.imread(img_path, cv2.IMREAD_GRAYSCALE) -# return img_gray +##### color space +xyz_from_rgb = np.array( + [[0.412453, 0.357580, 0.180423], [0.212671, 0.715160, 0.072169], [0.019334, 0.119193, 0.950227]] +) +rgb_from_xyz = np.array( + [[3.24048134, -0.96925495, 0.05564664], [-1.53715152, 1.87599, -0.20404134], [-0.49853633, 0.04155593, 1.05731107]] +) -# def load_rgb_image(img_path): -# img_path = utf8_str(img_path) -# im = cv2.imread(img_path, cv2.IMREAD_COLOR) -# if im.shape[2] == 1: -# print('find gray images: ', img_path) -# im = cv2.cvtColor(im, cv2.COLOR_GRAY2BGR) -# img_rgb = cv2.cvtColor(im, cv2.COLOR_BGR2RGB) -# return img_rgb -# def resize_img_longside(img_rgb, long_size, interpolation): -# h, w = img_rgb.shape[:2] -# scalar = long_size / max(h, w) -# resized_img_rgb = cv2.resize(img_rgb, (int(w * scalar), int(h * scalar)), interpolation=interpolation) -# return resized_img_rgb +def tensor_lab2rgb(input): + """ + n * 3* h *w + """ + input_trans = input.transpose(1, 2).transpose(2, 3) # n * h * w * 3 + L, a, b = input_trans[:, :, :, 0:1], input_trans[:, :, :, 1:2], input_trans[:, :, :, 2:] + y = (L + 16.0) / 116.0 + x = (a / 500.0) + y + z = y - (b / 200.0) -# def resize_img_shortside(img_rgb, short_size, interpolation): -# h, w = img_rgb.shape[:2] -# scalar = short_size / min(h, w) -# resized_img_rgb = cv2.resize(img_rgb, (int(w * scalar), int(h * scalar)), interpolation=interpolation) -# return resized_img_rgb + neg_mask = z.data < 0 + z[neg_mask] = 0 + xyz = torch.cat((x, y, z), dim=3) -# def resize_img(img_rgb, dsize, interpolation): -# if isinstance(dsize, numbers.Number): -# dsize = (int(dsize), int(dsize)) -# resized_img_rgb = cv2.resize(img_rgb, dsize, interpolation=interpolation) -# return resized_img_rgb + mask = xyz.data > 0.2068966 + mask_xyz = xyz.clone() + mask_xyz[mask] = torch.pow(xyz[mask], 3.0) + mask_xyz[~mask] = (xyz[~mask] - 16.0 / 116.0) / 7.787 + mask_xyz[:, :, :, 0] = mask_xyz[:, :, :, 0] * 0.95047 + mask_xyz[:, :, :, 2] = mask_xyz[:, :, :, 2] * 1.08883 + + rgb_trans = torch.mm(mask_xyz.view(-1, 3), torch.from_numpy(rgb_from_xyz).type_as(xyz)).view( + input.size(0), input.size(2), input.size(3), 3 + ) + rgb = rgb_trans.transpose(2, 3).transpose(1, 2) + + mask = rgb > 0.0031308 + mask_rgb = rgb.clone() + mask_rgb[mask] = 1.055 * torch.pow(rgb[mask], 1 / 2.4) - 0.055 + mask_rgb[~mask] = rgb[~mask] * 12.92 + + neg_mask = mask_rgb.data < 0 + large_mask = mask_rgb.data > 1 + mask_rgb[neg_mask] = 0 + mask_rgb[large_mask] = 1 + return mask_rgb diff --git a/utils/util_tensorboard.py b/utils/util_tensorboard.py deleted file mode 100644 index 2a30b60..0000000 --- a/utils/util_tensorboard.py +++ /dev/null @@ -1,14 +0,0 @@ -from utils.util import to_np - - -def histogram_logger(tb_writer, iter_index, net=None): - if net is not None: - for tag, value in net.named_parameters(): - tag = tag.replace(".", "/") - tb_writer.add_histogram(tag, to_np(value), iter_index) - tb_writer.add_histogram(tag + "/grad", to_np(value.grad), iter_index) - - -def value_logger(tb_writer, iter_index, loss_info): - for tag, value in loss_info.items(): - tb_writer.add_scalar(tag, value, iter_index) diff --git a/utils/vgg_util.py b/utils/vgg_util.py index dfe5a31..05652fd 100755 --- a/utils/vgg_util.py +++ b/utils/vgg_util.py @@ -119,18 +119,15 @@ class MaxScale(object): Returns: PIL.Image: Rescaled image. """ - if isinstance(self.size, int): - w, h = img.size - if (w <= h == self.size) or (h <= w == self.size): - return img - if w < h: - oh = self.size - ow = int(self.size * w / h) - return img.resize((ow, oh), self.interpolation) - else: - ow = self.size - oh = int(self.size * h / w) - return img.resize((ow, oh), self.interpolation) - else: + if not isinstance(self.size, int): return img.resize(self.size, self.interpolation) - + w, h = img.size + if (w <= h == self.size) or (h <= w == self.size): + return img + if w < h: + oh = self.size + ow = int(self.size * w / h) + else: + ow = self.size + oh = int(self.size * h / w) + return img.resize((ow, oh), self.interpolation) diff --git a/utils/visualize_analogy.py b/utils/visualize_analogy.py deleted file mode 100644 index 81d9241..0000000 --- a/utils/visualize_analogy.py +++ /dev/null @@ -1,105 +0,0 @@ -import glob -import os -import os.path as osp -import struct - -import cv2 -import numpy as np -import torch.utils.data as data -from PIL import Image - -cv2.setNumThreads(0) -import pdb - -import lib.functional as F -import matplotlib.pyplot as plt -import torch -import torchvision.transforms as transforms -from PIL import Image -from skimage import color - -from utils.util import ( - MovingAvg, - batch_lab2rgb_transpose_mc, - calc_ab_gradient, - calc_cosine_dist_loss, - calc_tv_loss, - clean_tensorboard, - feature_normalize, - imshow, - imshow_lab, - mkdir_if_not, - to_np, - uncenter_l, - vgg_preprocess, - weighted_l1_loss, -) - - -def combo5_loader_nonoptimize(path): - f = open(path, "rb") - - # width, height - d = f.read(4) - im_sz = struct.unpack("i", d) - h = im_sz[0] - - d = f.read(4) - im_sz = struct.unpack("i", d) - w = im_sz[0] - - # warp_ba_layer 4 - d = f.read(4) - im_sz = struct.unpack("i", d) - d = f.read(im_sz[0]) - file_bytes = np.asarray(bytearray(d), dtype=np.uint8) - img_data_ndarray = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) - img_data_ndarray = cv2.cvtColor(img_data_ndarray, cv2.COLOR_BGR2RGB) - warp_ba = Image.fromarray(img_data_ndarray) - # warp_ba.save('/home/v-mingmh/Documents/gray/test_input/warp_ba.PNG') - - # warp_aba_layer 4 - d = f.read(4) - im_sz = struct.unpack("i", d) - d = f.read(im_sz[0]) - file_bytes = np.asarray(bytearray(d), dtype=np.uint8) - img_data_ndarray = cv2.imdecode(file_bytes, cv2.IMREAD_COLOR) - img_data_ndarray = cv2.cvtColor(img_data_ndarray, cv2.COLOR_BGR2RGB) - warp_aba = Image.fromarray(img_data_ndarray) - - # 5 layers: err_aba, err_ba, err_ab - errs = [] - for l in range(5): - d = f.read(4) - im_sz = struct.unpack("i", d) - d = f.read(im_sz[0]) - file_bytes = np.asarray(bytearray(d), dtype=np.uint8) - img_data_ndarray = cv2.imdecode(file_bytes, cv2.IMREAD_GRAYSCALE) - err_ba = Image.fromarray(img_data_ndarray) - - # err_ba.save('/home/v-mingmh/Documents/gray/test_input/input_err_ba' + str(l) + '.PNG') - - d = f.read(4) - im_sz = struct.unpack("i", d) - d = f.read(im_sz[0]) - file_bytes = np.asarray(bytearray(d), dtype=np.uint8) - img_data_ndarray = cv2.imdecode(file_bytes, cv2.IMREAD_GRAYSCALE) - err_ab = Image.fromarray(img_data_ndarray) - - # err_ab.save('/home/v-mingmh/Documents/gray/test_input/input_err_ab' + str(l) + '.PNG') - - errs.append([err_ba, err_ab]) - - f.close() - - return errs, warp_ba, warp_aba - - -# combo_folder = '../../movie/seq1/combo_new/' -combo_folder = "../../movie/seq2/combo_new" -output_folder = "analogy_output2/" -path, dirs, filenames = os.walk(combo_folder).__next__() - -for filename in filenames: - errs, warp_ba, warp_aba = combo5_loader_nonoptimize(combo_folder + "/" + filename) - warp_ba.save(output_folder + filename.split(".")[0] + ".jpg") diff --git a/utils/warping.py b/utils/warping.py deleted file mode 100644 index 251586f..0000000 --- a/utils/warping.py +++ /dev/null @@ -1,34 +0,0 @@ -import torch -import torch.nn as nn -import torch.nn.functional as F -from torch.autograd import Variable - - -def get_grid(x): - torchHorizontal = ( - torch.linspace(-1.0, 1.0, x.size(3)).view(1, 1, 1, x.size(3)).expand(x.size(0), 1, x.size(2), x.size(3)) - ) - torchVertical = ( - torch.linspace(-1.0, 1.0, x.size(2)).view(1, 1, x.size(2), 1).expand(x.size(0), 1, x.size(2), x.size(3)) - ) - grid = torch.cat([torchHorizontal, torchVertical], 1).cuda() - - return grid - - -class WarpingLayer(nn.Module): - def __init__(self, device): - super(WarpingLayer, self).__init__() - self.device = device - - def forward(self, x, flow): - # WarpingLayer uses F.grid_sample, which expects normalized grid - # we still output unnormalized flow for the convenience of comparing EPEs with FlowNet2 and original code - # so here we need to denormalize the flow - flow_for_grip = torch.zeros_like(flow).cuda() - flow_for_grip[:, 0, :, :] = flow[:, 0, :, :] / ((flow.size(3) - 1.0) / 2.0) - flow_for_grip[:, 1, :, :] = flow[:, 1, :, :] / ((flow.size(2) - 1.0) / 2.0) - - grid = (get_grid(x) + flow_for_grip).permute(0, 2, 3, 1) - x_warp = F.grid_sample(x, grid) - return x_warp diff --git a/utils/frame2video.py b/utils/~frame2video.py similarity index 56% rename from utils/frame2video.py rename to utils/~frame2video.py index 8c5bb25..15cb7c1 100644 --- a/utils/frame2video.py +++ b/utils/~frame2video.py @@ -1,20 +1,6 @@ -import argparse -import gc -import glob -import math import os -import platform -import random -import shutil -import sys -import time -from datetime import datetime -from pdb import set_trace import cv2 -import matplotlib.pyplot as plt -import numpy as np -from skimage import color, io video_name = "video.mp4" frame_folder = "../../results/seq1_new/" @@ -26,7 +12,7 @@ images.sort() frame = cv2.imread(os.path.join(frame_folder, images[0])) height, width, layers = frame.shape video = cv2.VideoWriter("video.avi", cv2.VideoWriter_fourcc("X", "V", "I", "D"), 15, (width, height)) -# video = cv2.VideoWriter('video.avi', cv2.VideoWriter_fourcc('M', 'J', 'P', 'G'), 10, (width, height)) + for image in images: video.write(cv2.imread(os.path.join(frame_folder, image))) @@ -35,8 +21,6 @@ video.release() import imageio -frames = [] -for image in images: - frames.append(imageio.imread(os.path.join(frame_folder, image))) -imageio.mimsave("movie.gif", frames) +frames = [imageio.imread(os.path.join(frame_folder, image)) for image in images] +imageio.mimsave("movie.gif", frames) diff --git a/utils/model_util.py b/utils/~model_util.py similarity index 97% rename from utils/model_util.py rename to utils/~model_util.py index d00925a..1a824b9 100755 --- a/utils/model_util.py +++ b/utils/~model_util.py @@ -39,7 +39,7 @@ def _vgg_make_fc_layer_names(): class VGG19Before(nn.Module): def __init__(self, layer_name, part="conv"): super(VGG19Before, self).__init__() - assert part == "conv" or part == "fc", "invalid part name" + assert part in ["conv", "fc"], "invalid part name" vgg_19 = models.vgg19(pretrained=True) if part == "conv": @@ -59,4 +59,3 @@ class VGG19Before(nn.Module): if hasattr(self, "classifier"): x = self.classifier(x) return x - diff --git a/tensorboardX/__init__.py b/~tensorboardX/__init__.py similarity index 100% rename from tensorboardX/__init__.py rename to ~tensorboardX/__init__.py diff --git a/tensorboardX/crc32c.py b/~tensorboardX/crc32c.py similarity index 100% rename from tensorboardX/crc32c.py rename to ~tensorboardX/crc32c.py diff --git a/tensorboardX/embedding.py b/~tensorboardX/embedding.py similarity index 100% rename from tensorboardX/embedding.py rename to ~tensorboardX/embedding.py diff --git a/tensorboardX/event_file_writer.py b/~tensorboardX/event_file_writer.py similarity index 100% rename from tensorboardX/event_file_writer.py rename to ~tensorboardX/event_file_writer.py diff --git a/tensorboardX/graph.py b/~tensorboardX/graph.py similarity index 100% rename from tensorboardX/graph.py rename to ~tensorboardX/graph.py diff --git a/tensorboardX/graph_onnx.py b/~tensorboardX/graph_onnx.py similarity index 100% rename from tensorboardX/graph_onnx.py rename to ~tensorboardX/graph_onnx.py diff --git a/tensorboardX/record_writer.py b/~tensorboardX/record_writer.py similarity index 100% rename from tensorboardX/record_writer.py rename to ~tensorboardX/record_writer.py diff --git a/tensorboardX/src/__init__.py b/~tensorboardX/src/__init__.py similarity index 100% rename from tensorboardX/src/__init__.py rename to ~tensorboardX/src/__init__.py diff --git a/tensorboardX/src/attr_value_pb2.py b/~tensorboardX/src/attr_value_pb2.py similarity index 100% rename from tensorboardX/src/attr_value_pb2.py rename to ~tensorboardX/src/attr_value_pb2.py diff --git a/tensorboardX/src/event_pb2.py b/~tensorboardX/src/event_pb2.py similarity index 100% rename from tensorboardX/src/event_pb2.py rename to ~tensorboardX/src/event_pb2.py diff --git a/tensorboardX/src/graph_pb2.py b/~tensorboardX/src/graph_pb2.py similarity index 100% rename from tensorboardX/src/graph_pb2.py rename to ~tensorboardX/src/graph_pb2.py diff --git a/tensorboardX/src/node_def_pb2.py b/~tensorboardX/src/node_def_pb2.py similarity index 100% rename from tensorboardX/src/node_def_pb2.py rename to ~tensorboardX/src/node_def_pb2.py diff --git a/tensorboardX/src/plugin_pr_curve_pb2.py b/~tensorboardX/src/plugin_pr_curve_pb2.py similarity index 100% rename from tensorboardX/src/plugin_pr_curve_pb2.py rename to ~tensorboardX/src/plugin_pr_curve_pb2.py diff --git a/tensorboardX/src/resource_handle_pb2.py b/~tensorboardX/src/resource_handle_pb2.py similarity index 100% rename from tensorboardX/src/resource_handle_pb2.py rename to ~tensorboardX/src/resource_handle_pb2.py diff --git a/tensorboardX/src/summary_pb2.py b/~tensorboardX/src/summary_pb2.py similarity index 100% rename from tensorboardX/src/summary_pb2.py rename to ~tensorboardX/src/summary_pb2.py diff --git a/tensorboardX/src/tensor_pb2.py b/~tensorboardX/src/tensor_pb2.py similarity index 100% rename from tensorboardX/src/tensor_pb2.py rename to ~tensorboardX/src/tensor_pb2.py diff --git a/tensorboardX/src/tensor_shape_pb2.py b/~tensorboardX/src/tensor_shape_pb2.py similarity index 100% rename from tensorboardX/src/tensor_shape_pb2.py rename to ~tensorboardX/src/tensor_shape_pb2.py diff --git a/tensorboardX/src/types_pb2.py b/~tensorboardX/src/types_pb2.py similarity index 100% rename from tensorboardX/src/types_pb2.py rename to ~tensorboardX/src/types_pb2.py diff --git a/tensorboardX/src/versions_pb2.py b/~tensorboardX/src/versions_pb2.py similarity index 100% rename from tensorboardX/src/versions_pb2.py rename to ~tensorboardX/src/versions_pb2.py diff --git a/tensorboardX/summary.py b/~tensorboardX/summary.py similarity index 100% rename from tensorboardX/summary.py rename to ~tensorboardX/summary.py diff --git a/tensorboardX/writer.py b/~tensorboardX/writer.py similarity index 100% rename from tensorboardX/writer.py rename to ~tensorboardX/writer.py diff --git a/tensorboardX/x2num.py b/~tensorboardX/x2num.py similarity index 100% rename from tensorboardX/x2num.py rename to ~tensorboardX/x2num.py