Skip to content

Commit

Permalink
Merge pull request #36 from gene1wood/python3
Browse files Browse the repository at this point in the history 
Update to Python 3.8 and bugfixes
  • Loading branch information
@gene1wood
gene1wood authored Nov 17, 2020
2 parents 5d1eea5 + f1b2757 commit fd7faef
Show file tree
Hide file tree
Showing 6 changed files with 111 additions and 94 deletions.
Binary file modified artifacts/birch-girder.zip
View file
Binary file not shown.
58 changes: 39 additions & 19 deletions birch_girder/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
from agithub.GitHub import GitHub # pypi install agithub
import yaml # pip install PyYAML
from dateutil import tz # sudo pip install python-dateutil
import email
import email.utils
from email_reply_parser import \
EmailReplyParser # pip install email_reply_parser
from email.mime.multipart import MIMEMultipart
Expand All @@ -22,16 +22,16 @@
import pyzmail
import bs4
import base64
import urllib
import urllib.parse

TIME_ZONE = tz.gettz('America/Los_Angeles')

# Example "Re: [examplecorp/support] Add myself to user list. (#2)"
# https://stackoverflow.com/questions/9153629/regex-code-for-removing-fwd-re-etc-from-email-subject/11640925#comment81160171_11640925
EMAIL_SUBJECT_PREFIX = re.compile(
'^([[(] *)?(RE?S?|FYI|RIF|I|FS|VB|RV|ENC|ODP|PD|YNT'
'|ILT|SV|VS|VL|AW|WG|ΑΠ|ΣΧΕΤ|ΠΡΘ|תגובה|הועבר|主题|转发|FWD?)'
' *([-:;)\]][ :;\])-]*|$)|\]+ *$',
r'^([\[(] *)?(RE?S?|FYI|RIF|I|FS|VB|RV|ENC|ODP|PD|YNT'
r'|ILT|SV|VS|VL|AW|WG|ΑΠ|ΣΧΕΤ|ΠΡΘ|תגובה|הועבר|主题|转发|FWD?)'
r' *([-:;)\]][ :;\])-]*|$)|]+ *$',
re.IGNORECASE)

# "[examplecorp/support] Add myself to user list. (#2)"
Expand Down Expand Up @@ -401,11 +401,12 @@ def parse_email(self):
if len(self.record['ses']['mail']['commonHeaders']['from']) == 1
else ', '.join(
self.record['ses']['mail']['commonHeaders']['from']))
possible_recipients = list(self.config['recipient_list'].keys())

logger.debug(
'Multiple email destinations found. Looking for an applicable one '
': %s' % self.record['ses']['mail']['destination'])
for possible_recipient in self.config['recipient_list'].keys():
for possible_recipient in possible_recipients:
# Assign the first matching address in recipient_list to
# to_address
# Note : It's possible we could determine the actual correct
Expand All @@ -421,7 +422,7 @@ def parse_email(self):
break

if not self.to_address:
self.to_address = self.config['recipient_list'].keys()[0].lower()
self.to_address = possible_recipients[0].lower()
logger.debug(
'No applicable email was found in destination list so we will '
'use %s : %s' % (self.to_address,
Expand All @@ -435,7 +436,7 @@ def parse_email(self):
"the reply email back to the submitter may come from a "
"different email address than they sent the request to." % (
self.record['ses']['mail']['destination'],
self.config['recipient_list'].keys(),
possible_recipients,
self.to_address
))

Expand Down Expand Up @@ -577,7 +578,7 @@ def parse_email_payload(self):
soup = bs4.BeautifulSoup(
msg.html_part.get_payload(), 'html.parser')
self.email_body = ''.join(
unicode(x) for x in (
str(x) for x in (
soup.body.contents
if soup.body is not None else soup.contents)
if not isinstance(x, bs4.Comment))
Expand All @@ -596,7 +597,7 @@ def parse_email_payload(self):
storage_filename = "%s-%s" % (self.timestamp, filename)
logger.info('Adding attachment %s to repo' % filename)
if not self.dryrun:
path = 'attachments/%s' % urllib.quote(storage_filename)
path = 'attachments/%s' % urllib.parse.quote(storage_filename)
status, data = (
self.gh.repos[self.github_owner]
[self.github_repo].contents[path].put(
Expand Down Expand Up @@ -656,8 +657,8 @@ def update_issue(self, body, message_id, new_attachment_urls):
"couldn't add these new attachment links to the table. "
"Though this attachment has been saved to the repo at %s the "
"issue has not been updated to reflect this" % (
new_attachment_urls.keys(),
new_attachment_urls.values()
list(new_attachment_urls.keys()),
list(new_attachment_urls.values())
)
)
return body
Expand Down Expand Up @@ -900,6 +901,15 @@ def incoming_email(self):
else '''Thanks for contacting us. We will get back to you as soon
as possible. You can reply to this email if you have additional information
to add to your request.''')
status, html_body = self.gh.markdown.post(
body={
'text': body,
'mode': 'gfm',
'context': '%s/%s' % (
parsed_email.github_owner,
parsed_email.github_repo)
}
)
text_url = 'https://github.com/%s/%s/issues/%s' % (
parsed_email.github_owner,
parsed_email.github_repo,
Expand Down Expand Up @@ -936,21 +946,31 @@ def incoming_email(self):
in_reply_to=parsed_email.message_id,
references=parsed_email.message_id,
html=EMAIL_HTML_TEMPLATE.substitute(
html_body=body.format(html_url),
html_body=html_body.decode('utf-8').format(html_url),
**template_args),
text=EMAIL_TEXT_TEMPLATE.substitute(
text_body=body.format(text_url),
**template_args))

# Add a reaction to the issue indicating the sender has been
# replied to
repo = (
self.gh.repos[parsed_email.github_owner][parsed_email.github_repo])
issue = repo.issues[issue_data['number']]
status, reaction_data = issue.reactions.post(
body={'content':'heart'},
body={'content': 'rocket'},
headers={
'Accept': 'application/vnd.github.squirrel-girl-preview+json'})
logger.info('Just added a reaction to issue #%s after sending an email' %
issue_data['number'])
if int(status / 100) == 2:
logger.info(
'Just added a reaction to issue #%s after sending '
'an email' % issue_data['number'])
else:
logger.error(
'Unable to add reaction to issue #%s after %s : %s' % (
issue_data['number'],
status,
reaction_data))
else:
message_id = '1'
logger.debug(
Expand Down Expand Up @@ -1064,7 +1084,7 @@ def github_hook(self):
html_email_body = (
'<a href="https://github.com/{username}">@{username}</a> writes :'
'<br>\n{html_comment}'.format(username=author,
html_comment=html_comment))
html_comment=html_comment.decode('utf-8')))

issue_reference = '%s/%s#%s' % (
message['repository']['owner']['login'],
Expand Down Expand Up @@ -1102,7 +1122,7 @@ def github_hook(self):
comment = (self.gh.repos[message['repository']['full_name']].
issues.comments[message['comment']['id']])
status, reaction_data = comment.reactions.post(
body={'content':'heart'},
body={'content': 'rocket'},
headers={
'Accept': 'application/vnd.github.squirrel-girl-preview+json'})
logger.info('Just added a reaction to a comment in issue #%s after '
Expand All @@ -1121,7 +1141,7 @@ def lambda_handler(event, context):
"""
# logger.debug('got event {}'.format(event))
with open('config.yaml') as f:
config = yaml.load(f.read())
config = yaml.load(f.read(), Loader=yaml.SafeLoader)
handler = EventHandler(config, event, context)
handler.process_event()

Expand Down
33 changes: 15 additions & 18 deletions birch_girder/deploy.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,6 @@
import yaml # pip install PyYAML
import boto3
from agithub.GitHub import GitHub # pip install agithub
import agithub.base
from base64 import b64encode
from nacl import encoding, public

Expand All @@ -24,12 +23,6 @@
GREEN_COLOR = '\033[92m'
BLUE_COLOR = '\033[94m'

try:
# Python 2
prompt = lambda x: raw_input('%s%s%s : ' % (BLUE_COLOR, x, END_COLOR))
except NameError:
# Python 3
prompt = lambda x: input('%s%s%s : ' % (BLUE_COLOR, x, END_COLOR))

class Config(collections.MutableMapping):
def __init__(self, filename, *args, **kwargs):
Expand Down Expand Up @@ -67,6 +60,10 @@ def load(self):
pass


def prompt(message):
return input('%s%s%s : ' % (BLUE_COLOR, message, END_COLOR))


def plugin_path_type(path):
if not os.path.isdir(path):
raise argparse.ArgumentTypeError("%s isn't a directory" % path)
Expand All @@ -81,7 +78,7 @@ def get_two_factor_code():


def green_print(data):
print(GREEN_COLOR + data + END_COLOR)
print('%s%s%s' % (GREEN_COLOR, data, END_COLOR))


def color_getpass(prompt):
Expand Down Expand Up @@ -339,7 +336,7 @@ def main():
'Version': '2008-10-17',
'Statement': []
}
if (statement_id not in [x['Sid'] for x in policy['Statement']]):
if statement_id not in [x['Sid'] for x in policy['Statement']]:
policy['Statement'].append(
{
'Sid': statement_id,
Expand Down Expand Up @@ -412,7 +409,7 @@ def main():
identities = get_paginated_results('ses', 'list_identities', 'Identities')
verifications_initiated = False
identities_that_matter = []
for recipient in [x.lower() for x in config['recipient_list'].keys()]:
for recipient in [x.lower() for x in list(config['recipient_list'].keys())]:
domain = recipient.split('@')[1]
if recipient not in identities and domain not in identities:
print(
Expand Down Expand Up @@ -662,7 +659,7 @@ def main():
layers = get_paginated_results('lambda', 'list_layers', 'Layers')
layer_name = '%s-layer' % args.lambda_function_name
publish_layer = False
with open(zip_file_name) as f:
with open(zip_file_name, mode='rb') as f:
try:
hash_map_file = open(hash_version_map_filename)
hash_map = json.load(hash_map_file)
Expand Down Expand Up @@ -699,7 +696,7 @@ def main():
LayerName=layer_name,
Description='Birch Girder supporting python packages',
Content={'ZipFile': f.read()},
CompatibleRuntimes=['python2.7'])
CompatibleRuntimes=['python3.8'])
layer_version_arn = response['LayerVersionArn']
green_print('AWS Lambda layer published : %s'
% layer_version_arn)
Expand All @@ -724,7 +721,7 @@ def main():
try:
response = client.create_function(
FunctionName=args.lambda_function_name,
Runtime='python2.7',
Runtime='python3.8',
Role=lambda_iam_role_arn,
Handler='__init__.lambda_handler',
Code={'ZipFile': in_memory_data.getvalue()},
Expand Down Expand Up @@ -771,7 +768,7 @@ def main():
with zipfile.ZipFile(in_memory_data, 'w') as zip_file:
config_file = zipfile.ZipInfo('config.yaml', time.localtime()[:6])
config_file.compress_type = zipfile.ZIP_DEFLATED
config_file.external_attr = 0644 << 16L
config_file.external_attr = 0o644 << 16
zip_file.writestr(config_file, open(args.config).read())

zip_file.write(
Expand Down Expand Up @@ -856,7 +853,7 @@ def main():
Rule={
'Name': args.ses_rule_name,
'Enabled': True,
'Recipients': config['recipient_list'].keys(),
'Recipients': list(config['recipient_list'].keys()),
'Actions': [
{
'S3Action': {
Expand Down Expand Up @@ -1015,17 +1012,17 @@ def main():
status, _ = gh.repos[owner][repo].contents['.github']['workflows'][args.github_action_filename].put(
body={
'message': 'Adding Birch Girder GitHub Actions workflow\n\nhttps://github.com/gene1wood/birch-girder',
'content': base64.b64encode(content)})
'content': base64.b64encode(content.encode('ascii'))})
if 200 <= status < 300:
green_print(' New GitHub Actions workflow %s deployed'
% (args.github_action_filename))
else:
print("result from add %s and %s" % (status, _))
elif base64.b64decode(workflow_data['content']) != content:
elif base64.b64decode(workflow_data['content']).decode('ascii') != content:
status, _ = gh.repos[owner][repo].contents['.github']['workflows'][args.github_action_filename].put(
body={
'message': 'Updating Birch Girder GitHub Actions workflow\n\nhttps://github.com/gene1wood/birch-girder',
'content': base64.b64encode(content),
'content': base64.b64encode(content.encode('ascii')),
'sha': workflow_data['sha']})
if 200 <= status < 300:
green_print(' GitHub Actions workflow %s updated'
Expand Down
Loading

0 comments on commit fd7faef

Please sign in to comment.