#;; This buffer is for notes you don't want to save, and for Lisp evaluation.
#;; If you want to create a file, visit that file with C-x C-f,
#;; then enter the text in that file's own 


##### Requires pillow and pandas
##### as well as the ImageMagick library

import csv
import urllib
import os
import subprocess
from PIL import Image
import pandas as pd
import re
import sys

def compute_file_size(image):
    if not os.path.exists(image):
        print('Error: Image does not exist')
        return
    _HD = 1920*1080
    _4K = 3480*2160    
    im = Image.open(image)
    W, H = im.size
    if W < H:
        temp = H
        H    = W
        W    = temp    
    if W*H >= _4K :
        return '4K'
    elif W*H > _HD :
        return 'HD'
    else:
        return 'SD'

# working directory. should contain the subdirectoires  ../imagesQ99/HD ../imagesQ99/4K ../imagesQ99/SD and ../tmp 
path = '/proj/video_data3/videosim/ejacstr/JVET/CEs/Gothenburg2019/downloadStill/'

# point your downloaded CSV file here:
csvPath = '/proj/video_data3/videosim/ejacstr/nn/data/2017_07/train/'

csvFile = csvPath + 'images.csv'
destFile= path + 'tmp/toTest99.jpg'

nBatch = 10
nTotal = 4000
imageURL = 2 # The second column is the image url.
saveNumber = 0
verbose = 1
skip_offset = 0

numHDfiles = 0

# This file contains all the images in the OpenImage database
# that has a quality setting of exactly 99.
fQ99 = open("Q99list.txt", "r")


for n in range (1000) :     # we are only interested in Q99 images stored in the logfile99_full.txt
    line = fQ99.readline()              # read one line
    row_num = int( line.split()[0] )      # read the line number in csv file
    save_num = int( line.split()[1] )
    print "Loading image from csv file on row ", row_num, " and saving it as file number ", save_num
    row = pd.read_csv(csvFile, delimiter = ',', nrows = 1, skiprows = row_num)    # read the corresponding row
    URL = row.iloc[0][2];
    print ( 'line %d image url is %s' % (n,URL) )

    # check url existence
    code = urllib.urlopen(URL).code
    # if the url does not exist does something
    if code <> 200:
        # if it is no farm, we cant do anything
        if URL.split('//')[1].split('.')[0][0:4] == 'farm':
            print('Image is no longer downloadable, skipping...')
            continue
        else:
            # trying another server.
            # converting all cx with x>3 to c1 in the server name; This may work!
            URL = re.sub("https://c?.","https://c1",URL)
            code = urllib.urlopen(URL).code
            print ( 'image url is updated %s' % URL)
    if code==200:
        urllib.urlretrieve (URL, destFile)
    else:
        print('Image is no longer downloadable, skipping...')
        continue
    if compute_file_size(destFile) == 'HD':
        Path = path + 'imagesQ99/HD/'
        numHDfiles += 1
        if numHDfiles > 301:
            print("Downloaded 300 HD files. Exiting.")
            sys.exit(0)
    elif compute_file_size(destFile) == '4K':
        Path = path + 'imagesQ99/4K/'
    else:
        Path = path + 'imagesQ99/SD/'
    storeName = Path + 'image_' + str(save_num).zfill(7) + '.jpg';
    cmd = 'cp ' + destFile + ' ' + storeName;
    print '*** Saving image as ', storeName
    os.system(cmd);
    
    
