#!/usr/bin/env python #enable headless matplotlib import matplotlib matplotlib.use("Agg") import os import glob import logging import subprocess import datetime import urllib2 from sadcat.tools import Radarsat2Image from sadcat import CommonImagesCatalog, L2Catalog # load configuration from sadcat.config import download as download_config class DownloadRadarsat2(): '''Download Radarsat2 data ''' def __init__(self, zoneName): '''Make logger, get configuration''' self.zone = download_config[zoneName] self.zone['name'] = zoneName print self.zone def download_tails(self): '''Download tails of the tar files stored at ESA server''' # get list of already downloaded TAR files tailDir = str(self.zone['download'] + 'tail/' + self.zone['name'] + '/') localFiles = glob.glob(tailDir + 'RS02*.tar') localFiles = [os.path.split(path)[1] for path in localFiles] # get list of remote files remoteFiles = [] # list includes subdir and filename # get subdirs dwhURL = urllib2.urlopen(str(self.zone['url'])) subdirs = [sd.split()[-1] for sd in dwhURL.read().splitlines()] print 'File list read from remote server' for subdir in subdirs: # get list of files in subdir subURL = self.zone['url'] + '/' + subdir dwhSubURL = urllib2.urlopen(str(subURL)) filenames = [fn.split()[-1] for fn in dwhSubURL.read().splitlines()] for filename in filenames: remoteFiles.append((subdir, filename)) if len(remoteFiles) > 0: # download tails of remote files for rFile in remoteFiles: if rFile[1] not in localFiles: cmd = ['/usr/bin/curl', '-r', '-15000', '-m', '100', str(self.zone['url'] + '/' + rFile[0] + '/' + rFile[1]), '-o', os.path.join(tailDir, rFile[1])] print 'cmd: %s' % str(cmd) subprocess.call(cmd) # scan through TARs of remote files and extract name/lat/lon for i, rFile in enumerate(remoteFiles): try: rFileID = open(tailDir + rFile[1]) except: print 'Cannot find %s' % (tailDir + rFile[1]) else: rLines = rFileID.readlines() rFileID.close() rLatLon = ['0', '0'] # default value rFileName = '' # default value for rLine in rLines: if '' in rLine: #79.59 44.16 ==> ['79.59', '44.16'] rLatLon = rLine.split('gml:pos>')[1].replace('' in rLine: rFileName = rLine.split('urn:eop:RS02:DWH_MG1_CORE_11:')[1].split('')[0] # add [name,lon,lat] to the list of downloadable files remoteFiles[i] = [rFile[0], rFile[1], rFileName, float(rLatLon[1]), float(rLatLon[0])] return remoteFiles def download_new(self, db, remoteFiles): '''Download new TAR files from ESA 'rolling archive', untar''' # get lon/lat liits from config lons = [float(self.zone['lonlatbox'][0]), float(self.zone['lonlatbox'][2])] lats = [float(self.zone['lonlatbox'][1]), float(self.zone['lonlatbox'][3])] # get lits of files in the database dbRows, dbFields = db.get_sensor_list('radarsat2') dbNameI = dbFields.index('name') dbNames = [dbRow[dbNameI].strip() for dbRow in dbRows] downFiles = [] for rFile in remoteFiles: if len(rFile) == 5 and (rFile[2] + '.zip' not in dbNames and rFile[2] + '.ZIP' not in dbNames and rFile[3] >= lons[0] and rFile[3] <= lons[1] and rFile[4] >= lats[0] and rFile[4] <= lats[1]): downFiles.append((rFile[0], rFile[1])) for dFile in downFiles: # download cmd = ['wget', '-nc', '-P', str(self.zone['download']), str(self.zone['url'] + '/' + dFile[0] + '/' + dFile[1])] print 'WGET: %s' % str(cmd) subprocess.call(cmd) cmd = ['tar', '-C', str(self.zone['download']), '-xvf', str(self.zone['download'] + dFile[1])] print 'UNTAR: %s' % str(cmd) subprocess.call(cmd) print 'Empty: %s' % str(self.zone['download'] + dFile[1]) file(str(self.zone['download'] + dFile[1]), 'wb').close() def download_all(self): '''Download all files using wget recursively''' downloadCmd = str('wget -A %s -nc -nd -r -w 3 --connect-timeout=60 -P %s %s' % (self.zone['mask'], self.zone['download'], self.zone['url'])) print 'Download: %s ' % downloadCmd os.system(downloadCmd) def untar_all(self): '''Untar all downloaded tar files''' allTarFiles = glob.glob(self.zone['download'] + '*.tar') for tarFile in allTarFiles: tarFileSize = os.path.getsize(tarFile) if tarFileSize > 0: cmd = ['tar', '-C', str(self.zone['download']), '-xvf', str(tarFile)] print 'UNTAR: %s' % str(cmd) status = subprocess.call(cmd) print 'Status: ', status print 'Empty: %s' % str(tarFile) file(str(tarFile), 'wb').close() zoneNames = [ 'Eurarctic', 'Eurarctic_ENVreplacement', ] # check if radarsat2.py is already running ps = subprocess.Popen(['ps', '-ef'], stdout=subprocess.PIPE).communicate()[0] processes = ps.split('\n') selfProcNumber = 0 for proc in processes: if 'radarsat2.py' in proc: print proc selfProcNumber += 1 print 'Already running processes: ', selfProcNumber if True:#selfProcNumber <= 2: # ================ UPDATE, DOWNLOAD, UPDATE ================= #cic = CommonImagesCatalog() #cic.fill_database('radarsat2', '/Data/sat/downloads/Radarsat2/', 'RS2*[z,Z][i,I][p,P]', Radarsat2Image) for zoneName in zoneNames: rs2a = DownloadRadarsat2(zoneName) #remoteFiles = rs2a.download_tails() #rs2a.download_new(cic, remoteFiles) #cic.fill_database('radarsat2', '/Data/sat/downloads/Radarsat2/', 'RS2*[z,Z][i,I][p,P]', Radarsat2Image) rs2a.download_all() rs2a.untar_all() # add downloaded files to the database cic = CommonImagesCatalog() cic.fill_database('radarsat2', '/Data/sat/downloads/Radarsat2/', 'RS2*[z,Z][i,I][p,P]', Radarsat2Image) # perform default L2-processing (for web, for all images) rs2web = L2Catalog('radarsat2') rs2web.process(Radarsat2Image)