Thursday, May 18, 2017

Send Large attachments with google API for gmail

The problem: How to send large size images using google's gmail API for python. The documentation is not very good and a 10MB size limit is not mentioned. Following their quickstart example, I use body-media instead of body to send large files.
See below for an example:
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText
import logging
log = logging.getLogger("log")
import httplib2from StringIO import StringIO
from apiclient import discovery
from oauth2client import client
from oauth2client import tools
from oauth2client.file import Storage
try:
import argparse
flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args()
except ImportError:
flags = None

# If modifying these scopes, delete your previously saved credentials
# at ~/.credentials/gmail-python-quickstart.json
SCOPES = 'https://www.googleapis.com/auth/gmail.readonly'
CLIENT_SECRET_FILE = 'client_secret.json'
APPLICATION_NAME = 'Gmail API Python Quickstart'# mail size limit. can get up to 35MB
MAILSIZELIMIT = 1024*1024*25 # set to 25MB
def get_credentials():
    """Gets valid user credentials from storage.

    If nothing has been stored, or if the stored credentials are invalid,
    the OAuth2 flow is completed to obtain the new credentials.

    Returns:
        Credentials, the obtained credential.
    """
    home_dir = os.path.expanduser('~')
    credential_dir = os.path.join(home_dir, '.credentials')
    if not os.path.exists(credential_dir):
        os.makedirs(credential_dir)
    credential_path = os.path.join(credential_dir,
                                   'gmail-python-quickstart.json')

    store = Storage(credential_path)
    credentials = store.get()
    if not credentials or credentials.invalid:
        flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES)
        flow.user_agent = APPLICATION_NAME
        flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_known_args()
        if flags:
            credentials = tools.run_flow(flow, store, flags)
        else: # Needed only for compatibility with Python 2.6
            credentials = tools.run(flow, store)
        log.debug('Storing credentials to ' + credential_path)
    return credentials

def jpg2attachment(filename):
  """Creates an image message from file on disk.
       See other examples here: https://developers.google.com/gmail/api/guides/sending"""
  with open(filename,'rb') as f:
    img = MIMEImage(f.read())
    img.add_header('Content-Disposition', 'attachment', filename=os.path.split(filename)[1])
  return img

def send_gmail_with_attachments(sender,recipients,subject,txtcontent,attachmentfilelist=[]):
  """Sends email with attachments.
       sender: string of sender name/address
       recipients: a list of strings with recipients emails
       subject: string of the subject
       txtcontent: string with textual message body
       attachmentfilelist: a list of file names. in this example it is only for jpg files.
  """
  log.info('Sending email to: {}'.format(recipients))
  log.debug('Connecting to gmail server')
  try:
    credentials = get_credentials()
    http = credentials.authorize(httplib2.Http())
  except:
    log.error('Error with credentials')
    return False
  for i in range(3): # 3 attempts to connect to service.
    try:
      service = discovery.build('gmail', 'v1', http=http)
      break
    except httplib2.ServerNotFoundError,e:
      if i==2:
        log.error('{} of 3 attempts to connect to mail server failed.'.format(i+1))
        log.error(e)
        return False  
  COMMASPACE = ', '
  toaddrs = COMMASPACE.join(recipients) # get recipients list
  # Create the container (outer) email message.
  msg = MIMEMultipart()
  msg['Subject'] = subject
  msg['From'] = sender
  msg['To'] = toaddrs
  # add content
  txt = MIMEText('text', 'plain')
  txt.set_payload(txtcontent)
  msg.attach(txt)
  raw = msg.as_string()
  attachments = [jpg2attachment(a) for a in attachmentfilelist if a.endswith('jpg')]
  # send the message via gmail
  attempt = 1
  while len(attachments): # as long as we have attachements
    mediafile = StringIO() # placeholder for media body file
    while len(attachments) and len(raw)+len(attachments[0].as_string())<MAILSIZELIMIT: # make sure we are not exceeding size limit.
      msg.attach(attachments.pop(0)) # remove image attachment from list and put in message
      raw = msg.as_string() # serialize message
    mediafile.write(raw) # populate the media body file
    media = MediaIoBaseUpload(mediafile,mimetype='message/rfc822',chunksize=1024*1024,resumable=True) # pack as a media message
    if len(msg.get_payload())>1: # make sure message is not empty (in case a single attachment is over size limit
      try:  
        service.users().messages().send(userId='me',body={},media_body=media).execute() # send the message
      except Exception,e:
        log.error("attampt {}. Can't send email.\n\n{}\n\n".format(attempt,e))
        if attempt<=3:
          attempt+=1
        else:
          figures = [img.get_filename() for img in msg.get_payload() if img.get_content_maintype()=='image']+[img.get_filename() for img in attachments]
          log.error('Failed to send figures: {}'.format(figures))
          return
      else:
        sentimages = [img for img in msg.get_payload() if img.get_content_maintype()=='image']
        log.info('Sent figures: {}'.format([img.get_filename() for img in sentimages]))
        [msg.get_payload().remove(img) for img in sentimages]# remove figures from message
    elif len(attachments): # there is a too large image, send to last or remove.
      img = attachments.pop(0)
      if not len(img.as_string())>MAILSIZELIMIT:
        log.debug('Large size attachment {} ({}) pushed to end of line.'.format(img.get_filename(),len(img.as_string())))
        attachments.append(img) # push to end of line
      else:
        log.error('Failed to send figure: {}, size: {}'.format(img.get_filename(),len((img.as_string()))))
  return True