diff options
| -rw-r--r-- | python/download_emails.py | 124 | ||||
| -rw-r--r-- | python/gmail_helpers.py | 176 | ||||
| -rw-r--r-- | python/process_queue.py | 9 | ||||
| -rw-r--r-- | python/send_email.py | 74 |
4 files changed, 182 insertions, 201 deletions
diff --git a/python/download_emails.py b/python/download_emails.py index 1ab7c0a5..2a124d2e 100644 --- a/python/download_emails.py +++ b/python/download_emails.py @@ -8,131 +8,9 @@ import sys from apiclient import errors from pathlib import Path from pytz import timezone -from send_email import get_gmail_service -from email.message import EmailMessage +from gmail_helpers import GmailMessage from email.utils import parsedate_to_datetime - -def ListMessagesWithLabels(service, user_id, label_ids=[]): - """List all Messages of the user's mailbox with label_ids applied. - - Args: - service: Authorized Gmail API service instance. - user_id: User's email address. The special value "me" - can be used to indicate the authenticated user. - label_ids: Only return Messages with these labelIds applied. - - Returns: - List of Messages that have all required Labels applied. Note that the - returned list contains Message IDs, you must use get with the - appropriate id to get the details of a Message. - """ - try: - response = service.users().messages().list(userId=user_id, - labelIds=label_ids).execute() - if 'messages' in response: - yield from response['messages'] - while 'nextPageToken' in response: - page_token = response['nextPageToken'] - response = service.users().messages().list(userId=user_id, - labelIds=label_ids, - pageToken=page_token).execute() - yield from response['messages'] - - except errors.HttpError as error: - print(json.loads(error.content.decode('utf-8'))['error']['message']) - - -def ListHistory(service, user_id, label_id=None, start_history_id=10000): - """List History of all changes to the user's mailbox. - - Args: - service: Authorized Gmail API service instance. - user_id: User's email address. The special value "me" - can be used to indicate the authenticated user. - start_history_id: Only return Histories at or after start_history_id. - - Returns: - A list of mailbox changes that occurred after the start_history_id. - """ - history = (service.users().history().list(userId=user_id, - startHistoryId=start_history_id, - historyTypes="messageAdded", - labelId=label_id) - .execute()) - changes = history['history'] if 'history' in history else [] - for change in changes: - if 'messagesAdded' in change: - for c in change['messagesAdded']: - yield c['message'] - - while 'nextPageToken' in history: - page_token = history['nextPageToken'] - history = (service.users().history().list(userId=user_id, - startHistoryId=start_history_id, - pageToken=page_token).execute()) - for change in history['history']: - if 'messagesAdded' in change: - for c in change['messagesAdded']: - yield c['message'] - -def labels_dict(service, user_id): - """Returns a dictionary mapping labels to labelids. - - Args: - service: Authorized Gmail API service instance. - user_id: User's email address. The special value "me" - - Returns: - dictionary mapping labels to labelids. - """ - try: - response = service.users().labels().list(userId=user_id).execute() - labels = response['labels'] - return {label['name']: label['id'] for label in labels} - except errors.HttpError as error: - print(json.loads(error.content.decode('utf-8'))['error']['message']) - -class GmailMessage(EmailMessage): - _labels = labels_dict(EmailMessage._service, 'me') - - def msgdict(self): - return {'raw': base64.urlsafe_b64encode(self.as_bytes()).decode()} - - def send(self): - try: - message = (self._service.users().messages(). - send(userId='me',body=self.msgdict()) - .execute()) - print('Message Id: %s' % message['id']) - except errors.HttpError as error: - print('An error occurred: %s' % error) - - @staticmethod - def list_msg_ids(label, start_history_id=None): - if start_history_id is not None: - return ListHistory(EmailMessage._service, - 'me', - label_id=GmailMessage._labels[label], - start_history_id=start_history_id) - else: - return ListMessagesWithLabels(EmailMessage._service, - 'me', - label_ids=[GmailMessage._labels[label]]) - - @classmethod - def from_id(cls, msg_id, user_id='me'): - try: - message = (cls._service.users().messages(). - get(userId=user_id, id=msg_id, format='raw').execute()) - instance = email.message_from_bytes( - base64.urlsafe_b64decode(message['raw']), - policy=email.policy.EmailPolicy()) - instance.history_id = message['historyId'] - return instance - except errors.HttpError as error: - print(json.loads(error.content.decode('utf-8'))['error']['message']) - def save_emails(update=True): """Download new emails that were labeled swaptions.""" DATA_DIR = Path(os.getenv("DATA_DIR")) diff --git a/python/gmail_helpers.py b/python/gmail_helpers.py new file mode 100644 index 00000000..650aad2b --- /dev/null +++ b/python/gmail_helpers.py @@ -0,0 +1,176 @@ +from apiclient.discovery import build +from apiclient import errors +from email.message import EmailMessage +from httplib2 import Http +from oauth2client import client, tools +import argparse +import base64 +import json +import oauth2client +import os + +SCOPES = 'https://www.googleapis.com/auth/gmail.modify' +CLIENT_SECRET_FILE = 'secret.json' +APPLICATION_NAME = 'Swaptions' + +def get_gmail_service(): + """Gets valid user credentials from storage. + + If nothing has been stored, or if the stored credentials are invalid, + the OAuth2 flow is completed to obtain the new credentials. + + Returns: + Credentials, the obtained credential. + """ + credential_dir = '.credentials' + if not os.path.exists(credential_dir): + os.makedirs(credential_dir) + credential_path = os.path.join(credential_dir, + 'guillaume.horel@serenitascapital.com.json') + + store = oauth2client.file.Storage(credential_path) + credentials = store.get() + if not credentials or credentials.invalid: + flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args() + flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES) + flow.user_agent = APPLICATION_NAME + credentials = tools.run_flow(flow, store, flags) + print('Storing credentials to ' + credential_path) + service = build('gmail', 'v1', http=credentials.authorize(Http())) + return service + +def ListMessagesWithLabels(service, user_id, label_ids=[]): + """List all Messages of the user's mailbox with label_ids applied. + + Args: + service: Authorized Gmail API service instance. + user_id: User's email address. The special value "me" + can be used to indicate the authenticated user. + label_ids: Only return Messages with these labelIds applied. + + Returns: + List of Messages that have all required Labels applied. Note that the + returned list contains Message IDs, you must use get with the + appropriate id to get the details of a Message. + """ + try: + response = service.users().messages().list(userId=user_id, + labelIds=label_ids).execute() + if 'messages' in response: + yield from response['messages'] + while 'nextPageToken' in response: + page_token = response['nextPageToken'] + response = service.users().messages().list(userId=user_id, + labelIds=label_ids, + pageToken=page_token).execute() + yield from response['messages'] + + except errors.HttpError as error: + print(json.loads(error.content.decode('utf-8'))['error']['message']) + + +def ListHistory(service, user_id, label_id=None, start_history_id=10000): + """List History of all changes to the user's mailbox. + + Args: + service: Authorized Gmail API service instance. + user_id: User's email address. The special value "me" + can be used to indicate the authenticated user. + start_history_id: Only return Histories at or after start_history_id. + + Returns: + A list of mailbox changes that occurred after the start_history_id. + """ + history = (service.users().history().list(userId=user_id, + startHistoryId=start_history_id, + historyTypes="messageAdded", + labelId=label_id) + .execute()) + changes = history['history'] if 'history' in history else [] + for change in changes: + if 'messagesAdded' in change: + for c in change['messagesAdded']: + yield c['message'] + + while 'nextPageToken' in history: + page_token = history['nextPageToken'] + history = (service.users().history().list(userId=user_id, + startHistoryId=start_history_id, + pageToken=page_token).execute()) + for change in history['history']: + if 'messagesAdded' in change: + for c in change['messagesAdded']: + yield c['message'] + +def labels_dict(service, user_id): + """Returns a dictionary mapping labels to labelids. + + Args: + service: Authorized Gmail API service instance. + user_id: User's email address. The special value "me" + + Returns: + dictionary mapping labels to labelids. + """ + try: + response = service.users().labels().list(userId=user_id).execute() + labels = response['labels'] + return {label['name']: label['id'] for label in labels} + except errors.HttpError as error: + print(json.loads(error.content.decode('utf-8'))['error']['message']) + +class GmailMessage(EmailMessage): + _service = get_gmail_service() + _labels = labels_dict(_service, 'me') + + def msgdict(self): + return {'raw': base64.urlsafe_b64encode(self.as_bytes()).decode()} + + def send(self): + try: + message = (self._service.users().messages(). + send(userId='me',body=self.msgdict()) + .execute()) + print('Message Id: %s' % message['id']) + except errors.HttpError as error: + print('An error occurred: %s' % error) + + @staticmethod + def list_msg_ids(label, start_history_id=None): + if start_history_id is not None: + return ListHistory(GmailMessage._service, + 'me', + label_id=GmailMessage._labels[label], + start_history_id=start_history_id) + else: + return ListMessagesWithLabels(GmailMessage._service, + 'me', + label_ids=[GmailMessage._labels[label]]) + + @classmethod + def from_id(cls, msg_id, user_id='me'): + try: + message = (cls._service.users().messages(). + get(userId=user_id, id=msg_id, format='raw').execute()) + instance = email.message_from_bytes( + base64.urlsafe_b64decode(message['raw']), + policy=email.policy.EmailPolicy()) + instance.history_id = message['historyId'] + return instance + except errors.HttpError as error: + print(json.loads(error.content.decode('utf-8'))['error']['message']) + +def main(): + """Shows basic usage of the Gmail API. + + Creates a Gmail API service object and outputs a list of label names + of the user's Gmail account. + """ + message = GmailMessage() + message.set_content("Hello everyone!") + message['To'] = 'guillaume.horel@gmail.com' + message['Subject'] = 'pomme' + message.send() + +if __name__ == '__main__': + message = main() diff --git a/python/process_queue.py b/python/process_queue.py index 95bce045..5b85892a 100644 --- a/python/process_queue.py +++ b/python/process_queue.py @@ -18,7 +18,7 @@ from common import get_redis_queue from pyisda.date import previous_twentieth from db import dbconn from quantlib.time.api import pydate_from_qldate, UnitedStates, Days, Date -from send_email import EmailMessage +from gmail_helpers import GmailMessage from tabulate import tabulate HEADERS = {'bond_trades': [ @@ -364,9 +364,10 @@ def bond_trade_process(conn, session, trade): conn.rollback() # send out email with trade content - email = EmailMessage(print_trade(trade)) - email['to'] = 'nyops@lmcg.com' - email['subject'] = email_subject(trade) + email = GmailMessage() + email.set_content(print_trade(trade)) + email['To'] = 'nyops@lmcg.com' + email['Subject'] = email_subject(trade) email.send() diff --git a/python/send_email.py b/python/send_email.py deleted file mode 100644 index ee107768..00000000 --- a/python/send_email.py +++ /dev/null @@ -1,74 +0,0 @@ -from apiclient.discovery import build -from apiclient import errors -from httplib2 import Http -import oauth2client -from oauth2client import client, tools -import os -import json -import base64 -import binascii -from email.mime.text import MIMEText -import sys - -import argparse - -SCOPES = 'https://www.googleapis.com/auth/gmail.modify' -CLIENT_SECRET_FILE = 'secret.json' -APPLICATION_NAME = 'Swaptions' - -def get_gmail_service(): - """Gets valid user credentials from storage. - - If nothing has been stored, or if the stored credentials are invalid, - the OAuth2 flow is completed to obtain the new credentials. - - Returns: - Credentials, the obtained credential. - """ - credential_dir = '.credentials' - if not os.path.exists(credential_dir): - os.makedirs(credential_dir) - credential_path = os.path.join(credential_dir, - 'guillaume.horel@serenitascapital.com.json') - - store = oauth2client.file.Storage(credential_path) - credentials = store.get() - if not credentials or credentials.invalid: - flags = argparse.ArgumentParser(parents=[tools.argparser]).parse_args() - flow = client.flow_from_clientsecrets(CLIENT_SECRET_FILE, SCOPES) - flow.user_agent = APPLICATION_NAME - credentials = tools.run_flow(flow, store, flags) - print('Storing credentials to ' + credential_path) - service = build('gmail', 'v1', http=credentials.authorize(Http())) - return service - -class EmailMessage(MIMEText): - _service = get_gmail_service() - - def msgdict(self): - if sys.version_info[0]==2: - return {'raw': base64.urlsafe_b64encode(self.as_string())} - else: - return {'raw': base64.urlsafe_b64encode(self.as_bytes()).decode()} - - def send(self): - try: - message = (self._service.users().messages().send(userId='me', body=self.msgdict()) - .execute()) - print('Message Id: %s' % message['id']) - except errors.HttpError as error: - print('An error occurred: %s' % error) - -def main(): - """Shows basic usage of the Gmail API. - - Creates a Gmail API service object and outputs a list of label names - of the user's Gmail account. - """ - message = EmailMessage('Hello everyone!') - message['to'] = 'guillaume.horel@gmail.com' - message['subject'] = 'pomme' - message.send() - -if __name__ == '__main__': - message = main() |