From e0596a10465718d9f39008c8f4b5072314929057 Mon Sep 17 00:00:00 2001 From: AlexeyAB Date: Fri, 1 Mar 2019 16:56:00 +0300 Subject: [PATCH] Added cmd/sh scripts for downloading ILSVRC2012 train & valid for Windows --- scripts/datasets.txt | 36 +++++++++++++++++++ scripts/windows/win_cifar.cmd | 19 ++++++++++ .../win_get_imagenet_train_48hours.cmd | 25 +++++++++++++ scripts/windows/win_get_imagenet_valid.cmd | 36 +++++++++++++++++++ scripts/windows/win_install_cygwin.cmd | 12 +++++++ scripts/windows/windows_imagenet_label.sh | 16 +++++++++ scripts/windows/windows_imagenet_train.sh | 17 +++++++++ 7 files changed, 161 insertions(+) create mode 100644 scripts/datasets.txt create mode 100644 scripts/windows/win_cifar.cmd create mode 100644 scripts/windows/win_get_imagenet_train_48hours.cmd create mode 100644 scripts/windows/win_get_imagenet_valid.cmd create mode 100644 scripts/windows/win_install_cygwin.cmd create mode 100644 scripts/windows/windows_imagenet_label.sh create mode 100644 scripts/windows/windows_imagenet_train.sh diff --git a/scripts/datasets.txt b/scripts/datasets.txt new file mode 100644 index 00000000..8151c083 --- /dev/null +++ b/scripts/datasets.txt @@ -0,0 +1,36 @@ + + +Datasets: + +Pascal VOC: http://host.robots.ox.ac.uk/pascal/VOC/voc2012/index.html + +MS COCO: http://cocodataset.org/#download + +ImageNet: http://imagenet.stanford.edu/download.php + +ImageNet (ILSVRC2012): http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads + +Open Images: https://storage.googleapis.com/openimages/web/download.html + +Cityscapes: https://www.cityscapes-dataset.com/ + +Object Tracking Benchmark: http://cvlab.hanyang.ac.kr/tracker_benchmark/datasets.html + +MARS: http://www.liangzheng.com.cn/Project/project_mars.html + +Market-1501: http://www.liangzheng.org/Project/project_reid.html + +German Traffic Sign Recognition Benchmark: http://benchmark.ini.rub.de/ + +Labeled Faces in the Wild: http://vis-www.cs.umass.edu/lfw/ + +Core50: https://vlomonaco.github.io/core50/ + +Visual Question Answering: https://visualqa.org/download.html + +Large Movie Review Dataset: http://ai.stanford.edu/~amaas/data/sentiment/ + + +Other datasets (Music, Natural Images, Artificial Datasets, Faces, Text, Speech, Recommendation Systems, Misc): http://deeplearning.net/datasets/ + +25 datasets: https://www.analyticsvidhya.com/blog/2018/03/comprehensive-collection-deep-learning-datasets/ \ No newline at end of file diff --git a/scripts/windows/win_cifar.cmd b/scripts/windows/win_cifar.cmd new file mode 100644 index 00000000..a53f95af --- /dev/null +++ b/scripts/windows/win_cifar.cmd @@ -0,0 +1,19 @@ +echo Run install_cygwin.cmd before: + + +c:\cygwin64\bin\wget https://pjreddie.com/media/files/cifar.tgz + +c:\cygwin64\bin\gzip -d "%CD:\=/%/cifar.tgz" + +c:\cygwin64\bin\tar --force-local -xvf "%CD:\=/%/cifar.tar" + +c:\cygwin64\bin\cat "%CD:\=/%/labels.txt" + + +c:\cygwin64\bin\find "%CD:\=/%/cifar/train" -name \*.png > "%CD:\=/%/cifar/train.list" + +c:\cygwin64\bin\find "%CD:\=/%/cifar/test" -name \*.png > "%CD:\=/%/cifar/test.list" + + + +pause \ No newline at end of file diff --git a/scripts/windows/win_get_imagenet_train_48hours.cmd b/scripts/windows/win_get_imagenet_train_48hours.cmd new file mode 100644 index 00000000..2c560608 --- /dev/null +++ b/scripts/windows/win_get_imagenet_train_48hours.cmd @@ -0,0 +1,25 @@ +echo Run install_cygwin.cmd before: + +rem http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads +rem https://github.com/amd/OpenCL-caffe/wiki/Instructions-to-create-ImageNet-2012-data + + +c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; echo $PWD" + +echo Wait several hours... + +c:\cygwin64\bin\wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar + +c:\cygwin64\bin\mkdir -p "%CD:\=/%/ILSVRC2012_img_train" + +c:\cygwin64\bin\tar --force-local -xf "%CD:\=/%/ILSVRC2012_img_train.tar" -C "%CD:\=/%/ILSVRC2012_img_train" + + + +c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/windows_imagenet_train.sh" + +c:\cygwin64\bin\find "%CD:\=/%/ILSVRC2012_img_train" -name \*.JPEG > imagenet1k.train.list + + + +pause \ No newline at end of file diff --git a/scripts/windows/win_get_imagenet_valid.cmd b/scripts/windows/win_get_imagenet_valid.cmd new file mode 100644 index 00000000..1eefdf07 --- /dev/null +++ b/scripts/windows/win_get_imagenet_valid.cmd @@ -0,0 +1,36 @@ +echo Run install_cygwin.cmd before: + +rem http://www.image-net.org/challenges/LSVRC/2012/nonpub-downloads +rem https://github.com/amd/OpenCL-caffe/wiki/Instructions-to-create-ImageNet-2012-data + + +c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; echo $PWD" + + +c:\cygwin64\bin\wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_bbox_val_v3.tgz + +c:\cygwin64\bin\gzip -d "%CD:\=/%/ILSVRC2012_bbox_val_v3.tgz" + +c:\cygwin64\bin\tar --force-local -xvf "%CD:\=/%/ILSVRC2012_bbox_val_v3.tar" + + +c:\cygwin64\bin\wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_val.tar + +c:\cygwin64\bin\mkdir -p "%CD:\=/%/imgs" + +c:\cygwin64\bin\tar --force-local -xf "%CD:\=/%/ILSVRC2012_img_val.tar" -C "%CD:\=/%/imgs" + + +echo Wait a few hours... + +rem c:\cygwin64\bin\wget https://pjreddie.com/media/files/imagenet_label.sh + +c:\cygwin64\bin\dos2unix "%CD:\=/%/windows_imagenet_label.sh" + +c:\cygwin64\bin\bash -l -c "cd %CD:\=/%/; %CD:\=/%/windows_imagenet_label.sh" + +c:\cygwin64\bin\find "%CD:\=/%/labelled" -name \*.JPEG > inet.val.list + + + +pause \ No newline at end of file diff --git a/scripts/windows/win_install_cygwin.cmd b/scripts/windows/win_install_cygwin.cmd new file mode 100644 index 00000000..6a2bda4a --- /dev/null +++ b/scripts/windows/win_install_cygwin.cmd @@ -0,0 +1,12 @@ +echo Download file: https://www.cygwin.com/setup-x86_64.exe + + +setup-x86_64.exe -q -P dos2unix,wget,tar,untar,gzip,unzip,qawk,bzip2,git,vim,gcc-g++,make,grep,sed,find + + +# wget rawgit.com/transcode-open/apt-cyg/master/apt-cyg +# install apt-cyg /bin + +echo Finished + +pause \ No newline at end of file diff --git a/scripts/windows/windows_imagenet_label.sh b/scripts/windows/windows_imagenet_label.sh new file mode 100644 index 00000000..a371875d --- /dev/null +++ b/scripts/windows/windows_imagenet_label.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +mkdir -p labelled +wd=`pwd` + +for f in val/*.xml; +do +label=`grep -m1 "" $f | grep -oP '\K[^<]*'` +im=`echo $f | sed 's/val/imgs/; s/xml/JPEG/'` +out=`echo $im | sed 's/JPEG/'${label}'.JPEG/; s/imgs/labelled/'` +mv ${wd}/$im ${wd}/$out +#ln -s ${wd}/$im ${wd}/$out +done + +#find ${wd}/labelled -name \*.JPEG > inet.val.list + diff --git a/scripts/windows/windows_imagenet_train.sh b/scripts/windows/windows_imagenet_train.sh new file mode 100644 index 00000000..11e6430b --- /dev/null +++ b/scripts/windows/windows_imagenet_train.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +#wget http://www.image-net.org/challenges/LSVRC/2012/nnoupb/ILSVRC2012_img_train.tar +#mkdir -p ILSVRC2012_img_train +#tar --force-local -xf ILSVRC2012_img_train.tar -C ILSVRC2012_img_train + +wd=`pwd` + +for f in ILSVRC2012_img_train/*.tar; +do +name=$(echo "$f" | cut -f 1 -d '.') +mkdir "${wd}/${name}" +tar --force-local -xf "${wd}/${f}" -C "${wd}/${name}" +done + +#find "${wd}/ILSVRC2012_img_train" -name \*.JPEG > imagenet1k.train.list +