#!/usr/bin/env python

import time
import os
import glob
import fnmatch
import time
import subprocess

import pysftp

# download AMSR2 data from JAXA

def connect():
    ''' try to (re)connect to the SFTP server'''
    tries = 5 # number of tries
    wait = 5  # delay between reconnetcs

    host = 'gcom-w1.jaxa.jp'
    username = "anton.korosov@nersc.no"
    password = "AMSR2014"
    port = 2051
    
    for n in range(tries):
        print 'Try to connect to JAXA [%d] ...' % n, 
        try:
            srv = pysftp.Connection(host=host, username=username, password=password, port=port)
        except:
            print 'ERROR: Cannot connect!'
            time.sleep(wait)
        else:
            print 'Connected!'
            return srv

    print 'ERROR: Could not connect to JAXA!'
    raise

fPattern = '*GW1AM2_*_01D_PNM?_L3SGT??HA1110110.h5*'

# get list of local files
lDir = '/Data/sat/downloads/AMSR2/L3B/'
lFiles = glob.glob(lDir + fPattern)
lFiles = [os.path.split(lFile)[1] for lFile in lFiles]

# get list of all remote files
freqs = [6, 7, 10, 18, 23, 36, 89]
years = [2014]
months = [5,6,7,8,9,10,11,12]

srv = connect()

rFiles = []
for yy in years:
    for mm in months:
        for ff in freqs:
            rPath = '/prd/standard/AMSR2/%04d/%04d.%02d/L3/TB%dGHz_10/1/' % (yy, yy, mm, ff)
            print rPath
            try:
                srv.chdir(rPath)
            except:
                print rPath, 'does not exist'
            else:
                rSubFiles = srv.listdir()
                rFiles += [rPath + rSubFile for rSubFile in rSubFiles]
srv.close()

# select remote files for download (not downloaded earlier)
#%cpaste
rFilesGood = []
rFiles = fnmatch.filter(rFiles, fPattern)
for rFile in rFiles:
    rFileName = str(os.path.split(os.path.splitext(rFile)[0])[1])
    print rFileName, 
    if rFileName not in lFiles:
        print ' - OK',
        rFilesGood.append(rFile)
    print ''

# download good remote files
delay = 2 # delay between downloads
for rFile in rFilesGood:

    try:
        # test connection
        rdir = srv.getcwd()
        print 'Connection - ok'
    except:
        # reconnect
        print 'Connecting ...'
        srv = connect()
        print 'OK!'

    # download the file
    rFileName = str(os.path.split(rFile)[1])
    print 'Downloading %s' % rFileName
    srv.get(rFile, lDir+rFileName)
    time.sleep(delay)

# gunzip all
gzFiles = glob.glob(lDir + '*.gz')
for gzFile in gzFiles:
    print gzFile
    subprocess.call(['gunzip', gzFile])