Merge pull request #108 from kkamkou/issue-103

Issue 103

Readme.md

@@ -25,7 +25,8 @@ py.test --pep8 --clearcache --cov gitmostwanted tests/unit
 ```
 
 ## Docker
+The first time [it'll fail](https://github.com/docker/compose/issues/374)
 
 ```bash
-[sudo] docker-compose up -d # The first time it'll fail
+[sudo] docker-compose up -d
 ```

docker-compose.yml

@@ -1,7 +1,7 @@
 web:
  build: .
  user: gitmostwanted
- entrypoint: ["/opt/entrypoint.sh"]
+ entrypoint: ["/opt/entrypoint.sh", "web"]
  ports:
  - "5000:5000"
  volumes:
@@ -15,7 +15,7 @@ web:
 celery:
  build: .
  user: gitmostwanted
- command: celery -A gitmostwanted.app.celery worker -B -s /tmp/celerybeat-schedule.dat --autoreload --loglevel=DEBUG
+ entrypoint: ["/opt/entrypoint.sh", "celery"]
  environment:
  - GMW_APP_SETTINGS=/opt/instance.cfg
  volumes_from:

entrypoint.sh

@@ -1,6 +1,18 @@
 #!/bin/bash
 set -e
 
-(cd ..; alembic upgrade head)
+case "$1" in
+ web)
+ (cd ..; alembic upgrade head)
+ python -m gitmostwanted.web
+ ;;
 
-python -m gitmostwanted.web
+ celery)
+ rm -f /tmp/celerybeat-schedule.dat
+ celery -A gitmostwanted.app.celery worker -B -s /tmp/celerybeat-schedule.dat --autoreload --loglevel=DEBUG
+ ;;
+
+ *)
+ echo $"Usage: $0 {web|celery}"
+ exit 1
+esac

gitmostwanted/blueprints/repo_rating.py

@@ -1,6 +1,5 @@
 from flask import Blueprint, g, request, render_template
 from gitmostwanted.app import db
-from gitmostwanted.blueprints.mixin import repository_filtered
 from gitmostwanted.models.user import UserAttitude
 from gitmostwanted.models.repo import Repo
 
@@ -24,11 +23,21 @@ def top(page):
  .order_by(Repo.worth.desc()) \
  .order_by(Repo.created_at.asc())
 
- l = Repo.language_distinct()
- q = repository_filtered(request, l, Repo, q)
+ lngs = Repo.language_distinct()
+
+ lang = request.args.get('lang')
+ if lang != 'All' and (lang,) in lngs:
+ q = q.filter(Repo.language == lang)
+
+ status = request.args.get('status')
+ if status in ('promising', 'hopeless'):
+ q = q.filter(Repo.status == status)
+
+ if bool(request.args.get('mature')):
+ q = q.filter(Repo.mature.is_(True))
 
  entries = q.paginate(page if page > 0 else 1, per_page=20, error_out=False)
  if entries.pages and entries.pages < entries.page:
  return top(entries.pages)
 
- return render_template('repository/top.html', repos=entries, page=page, languages=l)
+ return render_template('repository/top.html', repos=entries, page=page, languages=lngs)

gitmostwanted/blueprints/repo_trending.py

@@ -0,0 +1,42 @@
+from flask import Blueprint, g, request, render_template
+from gitmostwanted.models.repo import Repo
+from gitmostwanted.models.user import UserAttitude
+from gitmostwanted.models import report
+from gitmostwanted.app import db
+
+repo_trending = Blueprint('repo_trending', __name__)
+
+
+@repo_trending.route('/', defaults={'rng': 'day'})
+@repo_trending.route('/trending/<rng>/')
+def list_by_range(rng):
+ map_list = {'day': 'ReportAllDaily', 'week': 'ReportAllWeekly', 'month': 'ReportAllMonthly'}
+ model = getattr(report, map_list.get(rng, map_list.get('day')))
+
+ query = model.query.join(Repo).order_by(model.cnt_watch.desc())
+ if not g.user:
+ return list_by_range_filtered(query.add_columns(db.null()))
+
+ return list_by_range_filtered(
+ query.add_columns(UserAttitude.attitude).outerjoin(
+ UserAttitude,
+ (UserAttitude.user_id == g.user.id) & (UserAttitude.repo_id == Repo.id)
+ )
+ )
+
+
+def list_by_range_filtered(query):
+ langs = Repo.language_distinct()
+
+ lang = request.args.get('lang')
+ if lang != 'All' and (lang,) in langs:
+ query = query.filter(Repo.language == lang)
+
+ status = request.args.get('status')
+ if status in ('promising', 'hopeless'):
+ query = query.filter(Repo.status == status)
+
+ if bool(request.args.get('mature')):
+ query = query.filter(Repo.mature.is_(True))
+
+ return render_template('index.html', entries=query, languages=langs)

gitmostwanted/blueprints/user_attitude.py

@@ -1,6 +1,5 @@
-from flask import\
- abort, Blueprint, g, make_response, request, render_template_string, render_template
-from gitmostwanted.blueprints.mixin import repository_filtered
+from flask import abort, Blueprint, g, make_response, request, render_template_string,\
+ render_template
 from gitmostwanted.models.user import UserAttitude
 from gitmostwanted.models.repo import Repo
 from gitmostwanted.app import db
@@ -56,7 +55,16 @@ def list_by_attitude(attitude, page):
 
  languages = Repo.language_distinct()
 
- q = repository_filtered(request, languages, Repo, q)
+ lang = request.args.get('lang')
+ if lang != 'All' and (lang,) in languages:
+ q = q.filter(Repo.language == lang)
+
+ status = request.args.get('status')
+ if status in ('promising', 'hopeless'):
+ q = q.filter(Repo.status == status)
+
+ if bool(request.args.get('mature')):
+ q = q.filter(Repo.mature.is_(True))
 
  entries = q.paginate(page if page > 0 else 1, per_page=20, error_out=False)
  if entries.pages and entries.pages < entries.page:

gitmostwanted/blueprints/user_oauth.py

@@ -0,0 +1,62 @@
+from flask import Blueprint, g, redirect, request, session, url_for
+from gitmostwanted.services import oauth as service_oauth
+from gitmostwanted.models.user import User
+from gitmostwanted.app import app, db
+
+user_oauth = Blueprint('user_oauth', __name__)
+oauth = service_oauth.instance(app)
+
+
+@app.before_request
+def load_user_from_session():
+ if str(request.url_rule) in ['/logout']:
+ return None
+ g.user = User.query.get(session['user_id']) if 'user_id' in session else None
+
+
+@user_oauth.route('/logout')
+def logout():
+ session.pop('github_token', None)
+ session.pop('user_id', None)
+ return redirect('/')
+
+
+@user_oauth.route('/oauth/login')
+def login():
+ return oauth.github\
+ .authorize(callback=url_for('user_oauth.authorized', next=url_next(), _external=True))
+
+
+@user_oauth.route('/oauth/authorized')
+def authorized():
+ next_url = url_next() or url_for('/')
+
+ resp = oauth.github.authorized_response()
+ if resp is None:
+ return redirect(next_url)
+
+ session.permanent = True
+ session['github_token'] = (resp['access_token'], '')
+ me = oauth.github.get('user')
+ session['user_id'] = user_get_or_create(me.data['id'], me.data['email'], me.data['login']).id
+
+ return redirect(next_url)
+
+
+@oauth.github.tokengetter
+def github_tokengetter():
+ return session.get('github_token')
+
+
+def user_get_or_create(uid, uemail, uname):
+ entity = User.query.filter_by(github_id=uid).first()
+ if entity:
+ return entity
+ entity = User(github_id=uid, username=uname, email=uemail or None)
+ db.session.add(entity)
+ db.session.commit()
+ return entity
+
+
+def url_next():
+ return request.args.get('next') or request.referrer or None

gitmostwanted/bigquery/job.py → gitmostwanted/lib/bigquery/job.py

@@ -1,5 +1,5 @@
-from gitmostwanted.bigquery.service import ServiceGmw
-from gitmostwanted.bigquery.result import ResultJob
+from gitmostwanted.lib.bigquery.service import ServiceGmw
+from gitmostwanted.lib.bigquery.result import ResultJob
 from itertools import chain
 from uuid import uuid4
 
@@ -9,9 +9,27 @@ class Job:
  num_retries = 5
 
  def __init__(self, api: ServiceGmw, query_str: str, batch: bool=False):
- self.__complete = False
+ self.__complete = self.__id = False
+ self.__query_str = query_str
+ self.__batch = batch
  self.__api = api
- self.__id = self.__insert(query_str, batch)
+
+ def __insert(self, query_str: str, batch: bool=False):
+ body = {
+ 'jobReference': {
+ 'projectId': self.__api.project_id,
+ 'job_id': str(uuid4())
+ },
+ 'configuration': {
+ 'query': {
+ 'query': query_str,
+ 'priority': 'BATCH' if batch else 'INTERACTIVE'
+ }
+ }
+ }
+ return self.__api.jobs()\
+ .insert(projectId=self.__api.project_id, body=body)\
+ .execute(num_retries=self.num_retries)['jobReference']['jobId']
 
  @property
  def id(self):
@@ -25,6 +43,8 @@ def complete(self):
 
  @property
  def info(self):
+ if not self.id:
+ raise JobNotStartedException('The job is not started yet')
  return self.__api.jobs()\
  .get(projectId=self.__api.project_id, jobId=self.id)\
  .execute(num_retries=self.num_retries)
@@ -51,23 +71,14 @@ def pages(self):
 
  yield ResultJob(response)
 
- def __insert(self, query_str, batch: bool=False):
- body = {
- 'jobReference': {
- 'projectId': self.__api.project_id,
- 'job_id': str(uuid4())
- },
- 'configuration': {
- 'query': {
- 'query': query_str,
- 'priority': 'BATCH' if batch else 'INTERACTIVE'
- }
- }
- }
- return self.__api.jobs()\
- .insert(projectId=self.__api.project_id, body=body)\
- .execute(num_retries=self.num_retries)['jobReference']['jobId']
+ def execute(self):
+ self.__id = self.__insert(self.__query_str, self.__batch)
+ return self.id
 
 
 class JobIncompleteException(Exception):
  pass
+
+
+class JobNotStartedException(Exception):
+ pass

gitmostwanted/services/bigquery.py

@@ -1,4 +1,4 @@
-from gitmostwanted.bigquery.service import ServiceGmw
+from gitmostwanted.lib.bigquery.service import ServiceGmw
 
 
 def instance(app):

gitmostwanted/tasks/repo_metadata.py

@@ -1,6 +1,6 @@
 from gitmostwanted.app import app, db, celery
 from gitmostwanted.models.repo import Repo
-from gitmostwanted.github import api
+from gitmostwanted.lib.github import api
 from datetime import datetime, timedelta
 
 

gitmostwanted/tasks/repo_most_starred.py

@@ -1,14 +1,14 @@
-from gitmostwanted.github.api import repo_info
+from gitmostwanted.lib.github.api import repo_info
+from gitmostwanted.lib.bigquery.job import Job
 from gitmostwanted.app import app, db, celery
 from gitmostwanted.services import bigquery
-from gitmostwanted.bigquery.job import Job
 from gitmostwanted.models.repo import Repo
 from gitmostwanted.models import report
 from datetime import date, datetime
 from time import sleep
 
 
-def job_results(j: Job):
+def results_of(j: Job):
  while not j.complete:
  app.logger.debug('The job is not complete, waiting...')
  sleep(10)
@@ -76,12 +76,15 @@ def most_starred_month():
 def most_starred_sync(model_name: str, query: str):
  app.logger.info('Importing repos of %s', model_name)
 
- service = bigquery.instance(app)
  model = getattr(report, model_name)
+ service = bigquery.instance(app)
 
  db.session.query(model).delete()
 
- for row in job_results(Job(service, query)):
+ job = Job(service, query)
+ job.execute()
+
+ for row in results_of(job):
  info, code = repo_info(row[1])
  if not info:
  continue

gitmostwanted/tasks/repo_stars.py

@@ -1,12 +1,12 @@
 from gitmostwanted.models.repo import Repo, RepoStars
 from gitmostwanted.app import app, db, celery
 from gitmostwanted.services import bigquery
-from gitmostwanted.bigquery.job import Job
+from gitmostwanted.lib.bigquery.job import Job
 from datetime import datetime, timedelta
 from time import sleep
 
 
-def job_results(j: Job):
+def results_of(j: Job):
  while not j.complete:
  app.logger.debug('The job is not complete, waiting...')
  sleep(10)
@@ -31,10 +31,12 @@ def stars_mature(num_days):
 
  repos = Repo.query.filter(Repo.mature.is_(True)).filter(Repo.status == 'new')
  for repo in repos:
- jobs.append((Job(service, query.format(id=repo.id, date_from=date_from), batch=True), repo))
+ job = Job(service, query.format(id=repo.id, date_from=date_from), batch=True)
+ job.execute()
+ jobs.append((job, repo))
 
  for job in jobs:
- for row in job_results(job[0]):
+ for row in results_of(job[0]):
  db.session.add(RepoStars(repo_id=job[1].id, stars=row[0], year=row[1], day=row[2]))
 
  job[1].status = 'unknown'