Send Large attachments with google API for gmail

The problem: How to send large size images using google's gmail API for python. The documentation is not very good and a 10MB size limit is not mentioned. Following their quickstart example, I use body-media instead of body to send large files.
See below for an example:

from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import logging
log = logging.getLogger("log")
import httplib2from StringIO import StringIO
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/gmail-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'# mail size limit. can get up to 35MB
MAILSIZELIMIT = 1024*1024*25 # set to 25MB
def get_credentials():
    """Gets valid user credentials from storage.

    If nothing has been stored, or if the stored credentials are invalid,
    the OAuth2 flow is completed to obtain the new credentials.

    Returns:
        Credentials, the obtained credential.
    """
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'gmail-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_known_args()
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        log.debug('Storing credentials to ' + credential_path)
    return credentials

def jpg2attachment(filename):
  """Creates an image message from file on disk.
       See other examples here: https://developers.google.com/gmail/api/guides/sending"""
  with open(filename,'rb') as f:
    img = MIMEImage(f.read())
    img.add_header('Content-Disposition', 'attachment', filename=os.path.split(filename)[1])
  return img

def send_gmail_with_attachments(sender,recipients,subject,txtcontent,attachmentfilelist=[]):
  """Sends email with attachments.
       sender: string of sender name/address
       recipients: a list of strings with recipients emails
       subject: string of the subject
       txtcontent: string with textual message body
       attachmentfilelist: a list of file names. in this example it is only for jpg files.
  """
  log.info('Sending email to: {}'.format(recipients))
  log.debug('Connecting to gmail server')
  try:
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
  except:
    log.error('Error with credentials')
    return False
  for i in range(3): # 3 attempts to connect to service.
    try:
      service = discovery.build('gmail', 'v1', http=http)
      break
    except httplib2.ServerNotFoundError,e:
      if i==2:
        log.error('{} of 3 attempts to connect to mail server failed.'.format(i+1))
        log.error(e)
        return False  
  COMMASPACE = ', '
  toaddrs = COMMASPACE.join(recipients) # get recipients list
  # Create the container (outer) email message.
  msg = MIMEMultipart()
  msg['Subject'] = subject
  msg['From'] = sender
  msg['To'] = toaddrs
  # add content
  txt = MIMEText('text', 'plain')
  txt.set_payload(txtcontent)
  msg.attach(txt)
  raw = msg.as_string()
  attachments = [jpg2attachment(a) for a in attachmentfilelist if a.endswith('jpg')]
  # send the message via gmail
  attempt = 1
  while len(attachments): # as long as we have attachements
    mediafile = StringIO() # placeholder for media body file
    while len(attachments) and len(raw)+len(attachments[0].as_string())<MAILSIZELIMIT: # make sure we are not exceeding size limit.
      msg.attach(attachments.pop(0)) # remove image attachment from list and put in message
      raw = msg.as_string() # serialize message
    mediafile.write(raw) # populate the media body file
    media = MediaIoBaseUpload(mediafile,mimetype='message/rfc822',chunksize=1024*1024,resumable=True) # pack as a media message
    if len(msg.get_payload())>1: # make sure message is not empty (in case a single attachment is over size limit
      try:  
        service.users().messages().send(userId='me',body={},media_body=media).execute() # send the message
      except Exception,e:
        log.error("attampt {}. Can't send email.\n\n{}\n\n".format(attempt,e))
        if attempt<=3:
          attempt+=1
        else:
          figures = [img.get_filename() for img in msg.get_payload() if img.get_content_maintype()=='image']+[img.get_filename() for img in attachments]
          log.error('Failed to send figures: {}'.format(figures))
          return
      else:
        sentimages = [img for img in msg.get_payload() if img.get_content_maintype()=='image']
        log.info('Sent figures: {}'.format([img.get_filename() for img in sentimages]))
        [msg.get_payload().remove(img) for img in sentimages]# remove figures from message
    elif len(attachments): # there is a too large image, send to last or remove.
      img = attachments.pop(0)
      if not len(img.as_string())>MAILSIZELIMIT:
        log.debug('Large size attachment {} ({}) pushed to end of line.'.format(img.get_filename(),len(img.as_string())))
        attachments.append(img) # push to end of line
      else:
        log.error('Failed to send figure: {}, size: {}'.format(img.get_filename(),len((img.as_string()))))
  return True

Batch Download ESA's Sentinel satellite images

The new ESA's satellite Sentinel  data is distributed free using an open data hub.
All you need to do is to register. The data hub provide tools for searching data and creating a cart of products for downloading. Since I'm very lazy, I created a Python client that would download the data once provided with the correct data link or a product list file (that can be easily downloaded from the hub online interface. The code is freely available on GitHub at:
https://github.com/rannof/SentinelDL

Next step (if time will allow it) is to add an automated data search based on parameters such as location, data type, dates etc.

FFT using pylab

from pylab import *

data,timestep = 10*sin(arange(1000)*5/2.0/pi)+5*sin(arange(1000)*10/2.0/pi),0.01
han = hanning(len(data))
handata = han*data
n = len(data)*50
FFT = abs(fftshift(fft(handata,n)))
freq = fftshift(fftfreq(n,timestep))
plot(freq,2*FFT/sum(han))

Python Event Listener - multiprocessing deamon

I wanted to have a file change listener such that if new data is added to a file I could run a code.
Following this post, I've written a modification so now the listener is working in multiprocessing.

#!/usr/bin/python
# A code to stream file as it grows
# by Ran Novitsky Nof Oct 25, 2012

import os,time
from multiprocessing import Process,Pipe

# change this file name
FileName = 'stream.txt'

# adapted from http://www.valuedlessons.com/2008/04/events-in-python.html


# the Event class take care of adding, removing and executing the functions
class Event:
    def __init__(self):
        self.handlers = set()

    def handle(self, handler):
        self.handlers.add(handler)
        return self

    def unhandle(self, handler):
        try:
            self.handlers.remove(handler)
        except:
            raise ValueError("Handler is not handling this event, so cannot unhandle it.")
        return self

    def fire(self, *args, **kargs):
        for handler in self.handlers:
            handler(*args, **kargs)

    def getHandlerCount(self):
        return len(self.handlers)

    __iadd__ = handle
    __isub__ = unhandle
    __call__ = fire
    __len__  = getHandlerCount
 
# the stream class takes care of the
# function to fire in case of an event
class Stream:
  def __init__(self,fname):
    self.fname = fname
    self.fd = open(fname,'r')
    self.get_stream()
  def get_stream(self):
    # this is the actual function to fire
    print self.fd.read(),
  def restart(self):
    self.fd.seek(0)
 
# the MockFileWatcher creats the multiprocessing deamon to alert on file change
class MockFileWatcher:
    def __init__(self,source_path,sleeptime=0.1):
        self.fileChanged = Event()
        self.source_path = source_path
        self.running=False
        self.deamon=False
        self.fd = None
        self.sleep = sleeptime
        # the next two lines can be set outside the class
        self.stream = Stream(source_path)
        self.fileChanged += self.stream.get_stream # add event handler
    def watchFiles(self):
        # open the file in non-blocking read mode
        self.fd = os.open(self.source_path,os.O_RDONLY | os.O_NONBLOCK)
        # go to the end
        os.lseek(self.fd,0,2)
        self.running=True
        self.deamon=False
        while self.running:
          # try to read
          if os.read(self.fd,1)!='':
            # New data was added
            os.lseek(self.fd,0,2) # go to the end of file
            self.fileChanged() # fire the event
          # see if the parent process has a message
          if self.chiled_conn.poll():
            # execute the message
            eval(self.chiled_conn.recv())
          # sleep for a while
          time.sleep(self.sleep)
    def watchFilesDeamon(self):
        # start the deamon
        self.parent_conn,self.chiled_conn = Pipe()
        p = Process(target=self.watchFiles)
        p.start()
        self.p = p  
        self.deamon=True
        return p
    def stop(self):
        # stop deamon/process
        if self.deamon:
          self.parent_conn.send('self.stop()')
          print "Hold while stopping child process..."
          # hold depends on the sleeping time
          if self.parent_conn.recv():
            pass
            #print "chiled is stopped"
          self.p.terminate() # terminate multiprocessing
          self.deamon=False
        # stop process
        self.running=False
        self.fd=None
        # if this is the process make sure parent know we are done
        if not self.deamon:  self.chiled_conn.send(True)      
        return True

if __name__=='__main__':
  watcher = MockFileWatcher(FileName)
  p = watcher.watchFilesDeamon()
  print 'from this point the listner is running at the background'
  print 'add more code here so you can see how it works.'

Try this by changing the file name at the top and run it. then add new lines to the file.

Howto write to stdout XY

how to use python to write to a certain location (x,y) on the stdout (terminal):

copied form here

import sys
def print_there(x, y, text):
     sys.stdout.write("\x1b7\x1b[%d;%df%s\x1b8" % (x, y, text))
     sys.stdout.flush()

HowTo Mosaic ASTER Global Digital Elevation Model (GDEM)

A second version of ASTER GDEM was released in Oct. 17 2011 by The Ministry of Economy, Trade and Industry of Japan (METI) and the National Aeronautics and Space Administration (NASA). The GDEM is in a 30m pixel resolution and 1x1 degree tiles. The data is distributed as zipped GeoTIFF files. When a larger than 1x1 deg. DEM is needed for Interferometry processing, you can use my GDEM.py script available from my script page. This python script will need a system with pylab and PIL python modules.

Simply:

Download the GDEM zipped files (see here on how to do that).

Unzip the zip files in the working directory.

The working directory should contain sub-directories containing the GeoTIFF files.

Run the script giving -h for help or the final DEM name as arguments.

The Final DEM is aimed for using as DEM in ROI-PAC interferometry processing.

If a -g is added as an argument, the produced DEM will be suitable for interferometric processing using Gamma software.

RoiView - Explore InSAR data and more

RoiView was originaly designed to replace DGX/MDX software which require bureaucracy to obtain. This software was used in order to view ROI-PAC interferometry processing results but producing an image from file was not so strait forward. In time support was added for ERDAS ER Mapper header files and for user provided parameters with or without header files.

Current RoiView version (0.75) is available for download at
See also RoiView's help files

Screenshot:

HowTo Parse HTML text using python.

I needed to to get the text of a specific div elemnt in an html file. I tried to use python's standard library modules for markup processing such as htmllib etc. but I couldn't figure how to use them. I've created my own module just for getting the text from an html element:

#!/bin/env python
from htmlentitydefs import entitydefs as ent
import string
# This module enable you to extract text from a certian HTML element
#
# by Ran Novitsky Nof, 2010
# ran.nof@gmail.com
#
# example of use:
# say we want to get the text in an element of type tag (e.g. 'div','a','span' etc.)
# who has an attribute key (e.g. "id","class","href" etc.) with a value of val
# for example in order to extract the text of a div element with id of textdiv from a file htmlfile.html:
#
# <html>
# <head>
#   :
# </head>
# <body>
#   :
# <div "id"="textdiv">I will not buy this <a href="spam">record</a> it is scratched. </div>
#   :
# </body>
# </html>
#
# use:
# from htmlparser import Parser
# htmlfile='htmlfile.html'
# tag,key,val = ('div','id','textdiv')
# text=Parser(htmlfile).getText(tag,key,val)
# print(text)
#
class Element():
  def __init__(self):
    self.startTag = -1
    self.endTag = -1
    self.attrib = {}
    self.keys = self.attrib.keys()
    self.innerHTML = ''
    self.tag = ''
    self.start = -1
    self.end = -1

class Parser():
  def __init__(self,infile):
    self._root = open(infile).read()
    self._root = self._root[self._root.find('<body'):self._root.find('</body')]
    self._root = self._root[self._root.find('>')+1:]
    self.tags = set()
    i,j=0,0
    self.tagstarts = {}
    self.tagends = {}
    self.elements = []
    while i<len(self._root): 
      i = self._root.find('<',i)
      j = self._root.find('>',i)
      if not j>i: break
      tag = self._root[i+1:j].split()[0]     
      if tag.startswith('/'):
        tag = tag[1:]
        self.tagends[tag][-1-self.tagends[tag][::-1].index(None)]=((i,j))
      else:
        self.tags.add(tag)
        if tag in self.tagstarts:
          self.tagstarts[tag].append((i,j))
          self.tagends[tag].append(None)
        else:
          self.tagstarts[tag]=[(i,j)]
          self.tagends[tag]=[None]
      i=j+1
    self.getElements()
  def getElements(self):
    for tag in self.tags:
      if not tag in ['img']:
        for i in range(len(self.tagstarts[tag])):
          element = Element()
          element.startTag = self.tagstarts[tag][i][0]
          element.endTag = self.tagstarts[tag][i][1]
          tagData = self._root[element.startTag+1:element.endTag].replace("\"","")
          element.tag = tag
          element.attrib=dict([a.split('=') for a in tagData.split() if '=' in a])
          element.start = element.startTag
          element.end = self.tagends[tag][i][1]
          element.innerHTML = (self.tagstarts[tag][i][1]+1,self.tagends[tag][i][0])
          self.elements.append(element)
  def getText(self,tag,key,val):
    element = [element for element in self.elements if element.tag==tag and element.attrib[key]==val]
    if len(element):
      element = element[0]
      start,end = element.innerHTML
      text = self._root[start:end]
      i,j=0,0
      while i<len(text):
        i  = text.find('<',i)
        j = text.find('>',i)
        if i<0 or j<0: break
        text = text[:i]+text[j+1:]
      i,j=-1,-1   
      while i<len(text):
        i  = text.find('&',i+1)
        j = text.find(';',i+1)
        if i<0 or j<0: break
        if text[i+1:j] in ent.keys(): text = text[:i]+ent[text[i+1:j]]+text[j+1:]
    else:
      text=None 
    return text

Note the code does not check if the html code is correct. it also work only for the body part, and ignore img tags.
   

Using hillshade image as intensity (improved matplotlib shade)

Matplotlib module enables hillshade method (shade) using a LightSource class (v 0.99).
The problem is it uses the data itself as intensity and data. It is very useful for viewing a DEM but sometimes you would like the DEM as intensity underlying some other data. Another problem is that the shade method is producing a very light colored image sometimes even white where intensity is high.
I used as an example a DEM derived from SRTM v4 data acquired at the International  Centre for Tropical  Agriculture (CIAT - http://srtm.csi.cgiar.org) the hillshade production was made using LightSource class with azimuth of 165 deg. and altitude of 45 deg.)

DEM - gist_earth color scheme

Hill-shade (azdeg-165,altdeg-45)

matplotlib shade method

My shade method

The difference in the shading colors derived from the method used to produce it. While the matplotlib method uses "hard light" method I use a "soft light" method. the matplotlib is converting the RGB colors to HSV and then calculate the new saturation and value according to the intensity. I use a formula based on the description of ImageMagick's pegtop_light.which is much faster as it is a single formula. Another advantage is the option to use a separate layer as the intensity and another as the data used for colors.
The modified functions are hillshade and set_shade as follows:

#!/bin/env python
from pylab import *
def set_shade(a,intensity=None,cmap=cm.jet,scale=10.0,azdeg=165.0,altdeg=45.0):
''' sets shading for data array based on intensity layer
  or the data's value itself.
inputs:
  a - a 2-d array or masked array
  intensity - a 2-d array of same size as a (no chack on that)
                    representing the intensity layer. if none is given
                    the data itself is used after getting the hillshade values
                    see hillshade for more details.
  cmap - a colormap (e.g matplotlib.colors.LinearSegmentedColormap
              instance)
  scale,azdeg,altdeg - parameters for hilshade function see there for
              more details
output:
  rgb - an rgb set of the Pegtop soft light composition of the data and 
           intensity can be used as input for imshow()
based on ImageMagick's Pegtop_light:
http://www.imagemagick.org/Usage/compose/#pegtoplight'''
  if intensity is None:
# hilshading the data
    intensity = hillshade(a,scale=10.0,azdeg=165.0,altdeg=45.0)
  else:
# or normalize the intensity
    intensity = (intensity - intensity.min())/(intensity.max() - intensity.min())
# get rgb of normalized data based on cmap
  rgb = cmap((a-a.min())/float(a.max()-a.min()))[:,:,:3]
# form an rgb eqvivalent of intensity
  d = intensity.repeat(3).reshape(rgb.shape)
# simulate illumination based on pegtop algorithm.
  rgb = 2*d*rgb+(rgb**2)*(1-2*d)
  return rgb

def hillshade(data,scale=10.0,azdeg=165.0,altdeg=45.0):
  ''' convert data to hillshade based on matplotlib.colors.LightSource class.
    input:
         data - a 2-d array of data
         scale - scaling value of the data. higher number = lower gradient
         azdeg - where the light comes from: 0 south ; 90 east ; 180 north ;
                      270 west
         altdeg - where the light comes from: 0 horison ; 90 zenith
    output: a 2-d array of normalized hilshade
'''
  # convert alt, az to radians
  az = azdeg*pi/180.0
  alt = altdeg*pi/180.0
  # gradient in x and y directions
  dx, dy = gradient(data/float(scale))
  slope = 0.5*pi - arctan(hypot(dx, dy))
  aspect = arctan2(dx, dy)
  intensity = sin(alt)*sin(slope) + cos(alt)*cos(slope)*cos(-az - aspect - 0.5*pi)
  intensity = (intensity - intensity.min())/(intensity.max() - intensity.min())
  return intensity

Example of use:
One can save the code to a file named say: shading.py
now say we have a 4 byte float DEM data in a 560 lines 420 samples binary file. in a python code:

from pylab import *
from shading import set_shade
from shading import hillshade
dem = fromfile('DEM.dem',dtype=float32).reshape(560,420)
rgb = set_shade(dem,cmap=cm.gist_earth)
imshow(rgb)

will produce the "my shade method" image as above.

say we have a data to be plot using the DEM data as intensity:

replace the line before last with:
rgb = set_shade(data,intensity=hillshade(dem),cmap=cm.gist_earth)

Reading GMT .GRD/NetCDF file using python

When reading GTM file in GRD format you actually read a NetCDF format.
using python and pylab to read it enable matplotlib to plot it.
say we have a grd file named z.grd which contains 3 variables: x,y and z.
reading z values in python is done by the following three lines:

from pylab import *
from scipy.io import netcdf_file as netcdf
data = netcdf('z.grd','r').variables['z'][::-1]

Now data is a numpy array containing z values in the file dimensions.
for more info look at: http://gfesuite.noaa.gov/developer/netCDFPythonInterface.html