diff --git a/gratipay/chomp/__init__.py b/gratipay/chomp/__init__.py index 8a51c02668..39d3c507d1 100644 --- a/gratipay/chomp/__init__.py +++ b/gratipay/chomp/__init__.py @@ -1,20 +1,18 @@ -"""Process npm artifacts. +"""Process package artifacts. The main function is exposed as a console script named `chomp` via setup.py. """ from __future__ import absolute_import, division, print_function, unicode_literals from collections import OrderedDict +from io import StringIO +from gratipay.utils import markdown import argparse import sys - import psycopg2 - import requests -from gratipay.utils import markdown -from io import StringIO class NPM(object): """Represent npm. @@ -22,12 +20,12 @@ class NPM(object): def __init__(self): self.name = 'npm' - def parse(self, package, package_manager_id, package_id): + def parse(self, package, package_id): """Given an npm package dict, return a dict of info and a list of emails to be imported. """ out = OrderedDict() out['id'] = str(package_id) - out['package_manager_id'] = str(package_manager_id) + out['package_manager_id'] = str(self.id) out['name'] = package['name'] out['description'] = package['description'] if 'description' in package else '' if 'time' in package: @@ -50,44 +48,55 @@ def parse(self, package, package_manager_id, package_id): return out, emails def fetch_catalog(self): + # import json + # return json.loads('{"a11y-announcer":{"name":"a11y-announcer","description":"An accessible ember route change announcer","dist-tags":{"latest":"1.0.2"},"maintainers":[{"name":"robdel12","email":"Robertdeluca19@gmail.com"}],"homepage":"https://github.com/ember-a11y/a11y-announcer#readme","keywords":["ember-addon","ember accessibility","ember router","a11y-announcer"],"repository":{"type":"git","url":"git+https://github.com/ember-a11y/a11y-announcer.git"},"author":{"name":"Robert DeLuca"},"bugs":{"url":"https://github.com/ember-a11y/a11y-announcer/issues"},"license":"MIT","readmeFilename":"README.md","users":{"jalcine":true,"unwiredbrain":true},"time":{"modified":"2016-08-13T23:03:37.135Z"},"versions":{"1.0.2":"latest"}}}') r = requests.get('https://registry.npmjs.com/-/all', verify=False) r.raise_for_status() return r.json() def stringify(obj): - return '\t'.join([v for k, v in obj.iteritems()]) + return '\t'.join([v for k, v in obj.iteritems()]) + '\n' -def insert_catalog_for(pm): +def insert_package_manager(pm, cursor): + """Insert a new package manager returning it's id. + """ + cursor.execute(""" + INSERT INTO package_managers (name) + VALUES (%s) RETURNING id + """, (pm.name,)) + return cursor.fetchone()[0] + +def package_manager_exists(pm, cursor): + cursor.execute(""" + SELECT package_managers.id + FROM package_managers + WHERE package_managers.name = (%s) + """, (pm.name,)) + return cursor.fetchone() + +def insert_catalog_for(pm, cursor): + """Insert all packages for a given package manager. + """ catalog = pm.fetch_catalog() - conn = psycopg2.connect(host='localhost', database='gratipay') if catalog: - cursor = conn.cursor() - cursor.execute(""" - INSERT INTO package_managers (name) - VALUES (%s) RETURNING id - """, (pm.name,)) - package_manager_id = cursor.fetchone()[0] package_id = 0 # Mass assign ids so they are usable during email insertion - package_data = StringIO() - email_data = StringIO() + package_stream, email_stream = StringIO(), StringIO() for k, v in catalog.iteritems(): package_id += 1 - package, emails = pm.parse(v, package_manager_id, package_id) - package_data.write(stringify(package) + '\n') + package, emails = pm.parse(v, package_id) + package_stream.write(stringify(package)) for email in emails: - email_data.write(stringify(email) + '\n') - package_data.seek(0) - email_data.seek(0) + email_stream.write(stringify(email)) + package_stream.seek(0) + email_stream.seek(0) - cursor.copy_from(package_data, 'packages', columns=["id", "package_manager_id", "name", "description"]) - cursor.copy_from(email_data, 'package_emails', columns=["package_id", "email"]) - conn.commit() + cursor.copy_from(package_stream, 'packages', columns=["id", "package_manager_id", "name", "description", "mtime"]) + cursor.copy_from(email_stream, 'package_emails', columns=["package_id", "email"]) def update_database(SQL): pass - def parse_args(argv): p = argparse.ArgumentParser() p.add_argument( 'if_modified_since' @@ -98,9 +107,19 @@ def parse_args(argv): ) return p.parse_args(argv) - def main(argv=sys.argv): pm_arg = parse_args(argv[1:]).package_manager if pm_arg == 'npm': pm = NPM() - insert_catalog_for(pm) + + conn = psycopg2.connect(host='localhost', database='gratipay') + cursor = conn.cursor() + + if package_manager_exists(pm, cursor): + print("This package manager already exists!") + else: + package_manager_id = insert_package_manager(pm, cursor) + setattr(pm, 'id', package_manager_id) + insert_catalog_for(pm, cursor) + + conn.commit() diff --git a/sql/branch.sql b/sql/branch.sql index bb173f1e6b..b37ca54c54 100644 --- a/sql/branch.sql +++ b/sql/branch.sql @@ -22,6 +22,6 @@ BEGIN; , package_id bigint NOT NULL REFERENCES packages ON UPDATE CASCADE ON DELETE RESTRICT , email text NOT NULL , UNIQUE (package_id, email) - ) + ); END;