Local administrators

.gitignore

@@ -6,3 +6,4 @@ config_local.py
 *.db
 *.swp
 *.log
+*.backup

www/audit.py

@@ -46,16 +46,25 @@ def get_user():
     return None
 
 
-def is_admin(user):
-    return user and user.uid in config.ADMINS
+def is_admin(user, project=None):
+    if not user:
+        return False
+    if user.uid in config.ADMINS:
+        return True
+    if not project:
+        return user.admin
+    return user == project.owner
 
 
 @app.route('/')
 def front():
     user = get_user()
     projects = Project.select().order_by(Project.updated.desc())
+
+    def local_is_admin(proj):
+        return is_admin(user, proj)
     return render_template('index.html', user=user, projects=projects,
-                           admin=is_admin(user))
+                           admin=is_admin(user), is_admin=local_is_admin)
 
 
 @app.route('/login')
@@ -121,9 +130,15 @@ def project(name):
         Feature.project == project, Feature.audit.is_null(False), Feature.audit != '').count()
     skipped = Feature.select(Feature.id).where(
         Feature.project == project, Feature.audit.contains('"skip": true')).count()
-    return render_template('project.html', project=project, admin=is_admin(get_user()),
+    user = get_user()
+    if user:
+        has_skipped = Task.select().join(Feature).where(
+            Task.user == user, Task.skipped == True, Feature.project == project).count() > 0
+    else:
+        has_skipped = False
+    return render_template('project.html', project=project, admin=is_admin(user, project),
                            desc=desc, val1=val1, val2=val2, corrected=corrected,
-                           skipped=skipped)
+                           skipped=skipped, has_skipped=has_skipped)
 
 
 @app.route('/browse/<name>')
@@ -271,7 +286,7 @@ def add_project(pid=None):
     return render_template('newproject.html', project=project)
 
 
-def update_features(project, features):
+def update_features(project, features, audit):
     curfeats = Feature.select(Feature).where(Feature.project == project)
     ref2feat = {f.ref: f for f in curfeats}
     deleted = set(ref2feat.keys())
@@ -282,6 +297,7 @@ def update_features(project, features):
         md5 = hashlib.md5()
         md5.update(data.encode('utf-8'))
         md5_hex = md5.hexdigest()
+
         coord = f['geometry']['coordinates']
         if coord[0] < minlon:
             minlon = coord[0]
@@ -291,10 +307,12 @@ def update_features(project, features):
             minlat = coord[1]
         if coord[1] > maxlat:
             maxlat = coord[1]
+
         if 'ref_id' in f['properties']:
             ref = f['properties']['ref_id']
         else:
             ref = '{}{}'.format(f['properties']['osm_type'], f['properties']['osm_id'])
+
         update = False
         if ref in ref2feat:
             deleted.remove(ref)
@@ -305,13 +323,20 @@ def update_features(project, features):
             feat = Feature(project=project, ref=ref)
             feat.validates_count = 0
             update = True
+
+        f_audit = audit.get(ref)
+        if f_audit and f_audit != feat.audit:
+            feat.audit = f_audit
+            update = True
+
         if update:
             feat.feature = data
             feat.feature_md5 = md5_hex
             feat.lon = round(coord[0] * 1e7)
             feat.lat = round(coord[1] * 1e7)
             feat.action = f['properties']['action'][0]
             feat.save()
+
     if deleted:
         q = Feature.delete().where(Feature.ref << list(deleted))
         q.execute()
@@ -326,12 +351,15 @@ def add_flash(pid, msg):
         flash(msg)
         return redirect(url_for('add_project', pid=pid))
 
-    if not is_admin(get_user()):
+    user = get_user()
+    if not is_admin(user):
         return redirect(url_for('front'))
     pid = request.form['pid']
     if pid:
         pid = int(pid)
         project = Project.get(Project.id == pid)
+        if not is_admin(user, project):
+            return redirect(url_for('front'))
     else:
         pid = None
         project = Project()
@@ -348,6 +376,10 @@ def add_flash(pid, msg):
         project.url = None
     project.description = request.form['description'].strip()
     project.can_validate = request.form.get('validate') is not None
+    project.hidden = request.form.get('is_hidden') is not None
+    if not project.owner or user.uid not in config.ADMINS:
+        project.owner = user
+
     if 'json' not in request.files or request.files['json'].filename == '':
         if not pid:
             return add_flash(pid, 'Would not create a project without features')
@@ -356,17 +388,27 @@ def add_flash(pid, msg):
         try:
             features = json.load(codecs.getreader('utf-8')(request.files['json']))
         except ValueError as e:
-            return add_flash(pid, 'Error in the uploaded file: {}'.format(e))
+            return add_flash(pid, 'Error in the uploaded features file: {}'.format(e))
         if 'features' not in features or not features['features']:
             return add_flash(pid, 'No features found in the JSON file')
         features = features['features']
 
-    project.updated = datetime.datetime.utcnow().date()
+    audit = None
+    if 'audit' in request.files and request.files['audit'].filename:
+        try:
+            audit = json.load(codecs.getreader('utf-8')(request.files['audit']))
+        except ValueError as e:
+            return add_flash(pid, 'Error in the uploaded audit file: {}'.format(e))
+        if not audit:
+            return add_flash(pid, 'No features found in the audit JSON file')
+
+    if features or audit or not project.updated:
+        project.updated = datetime.datetime.utcnow().date()
     project.save()
 
     if features:
         with database.atomic():
-            update_features(project, features)
+            update_features(project, features, audit or {})
 
     if project.feature_count == 0:
         project.delete_instance()
@@ -375,20 +417,33 @@ def add_flash(pid, msg):
     return redirect(url_for('project', name=project.name))
 
 
+@app.route('/clear_skipped/<int:pid>')
+def clear_skipped(pid):
+    project = Project.get(Project.id == pid)
+    user = get_user()
+    if user:
+        features = Feature.select().where(Feature.project == project)
+        query = Task.delete().where(
+            Task.user == user, Task.skipped == True,
+            Task.feature.in_(features))
+        query.execute()
+    return redirect(url_for('project', name=project.name))
+
+
 @app.route('/delete/<int:pid>')
 def delete_project(pid):
-    if not is_admin(get_user()):
-        return redirect(url_for('front'))
     project = Project.get(Project.id == pid)
+    if not is_admin(get_user(), project):
+        return redirect(url_for('front'))
     project.delete_instance(recursive=True)
     return redirect(url_for('front'))
 
 
 @app.route('/export_audit/<int:pid>')
 def export_audit(pid):
-    if not is_admin(get_user()):
-        return redirect(url_for('front'))
     project = Project.get(Project.id == pid)
+    if not is_admin(get_user(), project):
+        return redirect(url_for('front'))
     query = Feature.select(Feature.ref, Feature.audit).where(
         Feature.project == project, Feature.audit.is_null(False)).tuples()
     audit = {}
@@ -400,6 +455,24 @@ def export_audit(pid):
         headers={'Content-Disposition': 'attachment;filename=audit_{}.json'.format(project.name)})
 
 
+@app.route('/admin')
+def admin():
+    user = get_user()
+    if not user or user.uid not in config.ADMINS:
+        return redirect(url_for('front'))
+    admin_uids = User.select(User.uid).where(User.admin == True).tuples()
+    uids = '\n'.join([str(u[0]) for u in admin_uids])
+    return render_template('admin.html', uids=uids)
+
+
+@app.route('/admin_users', methods=['POST'])
+def admin_users():
+    uids = [int(x.strip()) for x in request.form['uids'].split()]
+    User.update(admin=False).where(User.uid.not_in(uids)).execute()
+    User.update(admin=True).where(User.uid.in_(uids)).execute()
+    return redirect(url_for('admin'))
+
+
 @app.route('/profile', methods=['GET', 'POST'])
 def profile():
     user = get_user()
@@ -453,11 +526,12 @@ def api_feature(pid):
     if user and request.method == 'POST' and project.can_validate:
         ref_and_audit = request.get_json()
         if ref_and_audit and len(ref_and_audit) == 2:
+            skipped = ref_and_audit[1] is None
             feat = Feature.get(Feature.project == project, Feature.ref == ref_and_audit[0])
             user_did_it = Task.select(Task.id).where(
                 Task.user == user, Task.feature == feat).count() > 0
-            Task.create(user=user, feature=feat)
-            if ref_and_audit[1] is not None:
+            Task.create(user=user, feature=feat, skipped=skipped)
+            if not skipped:
                 if len(ref_and_audit[1]):
                     new_audit = json.dumps(ref_and_audit[1], sort_keys=True, ensure_ascii=False)
                 else:

www/db.py

@@ -23,15 +23,24 @@ class Meta:
         database = database
 
 
+class User(BaseModel):
+    uid = IntegerField(primary_key=True)
+    admin = BooleanField(default=False)
+    bboxes = TextField(null=True)
+
+
 class Project(BaseModel):
     name = CharField(max_length=32, index=True, unique=True)
     title = CharField(max_length=250)
     description = TextField()
     url = CharField(max_length=1000, null=True)
     feature_count = IntegerField()
     can_validate = BooleanField()
+    hidden = BooleanField(default=False)
     bbox = CharField(max_length=60)
     updated = DateField()
+    owner = ForeignKeyField(User, related_name='projects')
+    overlays = TextField(null=True)
 
 
 class Feature(BaseModel):
@@ -46,29 +55,26 @@ class Feature(BaseModel):
     validates_count = IntegerField(default=0)
 
 
-class User(BaseModel):
-    uid = IntegerField(primary_key=True)
-    bboxes = TextField(null=True)
-
-
 class Task(BaseModel):
     user = ForeignKeyField(User, index=True, related_name='tasks')
     feature = ForeignKeyField(Feature, index=True, on_delete='CASCADE')
+    skipped = BooleanField(default=False)
 
 
-LAST_VERSION = 0
+LAST_VERSION = 1
 
 
 class Version(BaseModel):
     version = IntegerField()
 
 
+@database.atomic()
 def migrate():
     database.create_tables([Version], safe=True)
     try:
         v = Version.select().get()
     except Version.DoesNotExist:
-        database.create_tables([User, Project, Feature, Task], safe=True)
+        database.create_tables([User, Project, Feature, Task])
         v = Version(version=LAST_VERSION)
         v.save()
 
@@ -83,9 +89,19 @@ def migrate():
         migrator = PostgresqlMigrator(database)
 
     if v.version == 0:
+        # Making a copy of Project.owner field, because it's not nullable
+        # and we need to migrate a default value.
+        admin = User.select(User.uid).where(User.uid == list(config.ADMINS)[0]).get()
+        owner = ForeignKeyField(User, related_name='projects', to_field=User.uid, default=admin)
+
         peewee_migrate(
-            # TODO
-            migrator.add_column(User._meta.db_table, User.lang.name, User.lang)
+            migrator.add_column(User._meta.db_table, User.admin.db_column, User.admin),
+            migrator.add_column(Project._meta.db_table, Project.owner.db_column, owner),
+            migrator.add_column(Project._meta.db_table, Project.hidden.db_column, Project.hidden),
+            migrator.add_column(Project._meta.db_table, Project.overlays.db_column,
+                                Project.overlays),
+            migrator.add_column(Task._meta.db_table, Task.skipped.db_column, Task.skipped),
+            migrator.drop_column(Project._meta.db_table, 'validated_count'),
         )
         v.version = 1
         v.save()

www/templates/admin.html

@@ -0,0 +1,12 @@
+{% extends "layout.html" %}
+{% block title %}Administration — {% endblock %}
+{% block content %}
+<h1>Administration</h1>
+<form action="{{ url_for('admin_users') }}" method="POST">
+    Type user ids to allow uploading projects:<br>
+    <textarea name="uids" cols="20" rows="10">{{ admin_uids or '' }}</textarea><br>
+    <br>
+    <input type="submit">
+</form>
+<p><a href="{{ url_for('front') }}">Return</a></p>
+{% endblock %}

www/templates/index.html

@@ -4,24 +4,14 @@ <h1>Auditing Tool for OSM Conflator</h1>
 {% if admin %}
 <p><a href="{{ url_for('add_project') }}">Create a project</a></p>
 {% endif %}
-<p>Current imports that need validating:</p>
+<p>Imports that need validating:</p>
 <ul>
   {% for proj in projects %}
-  {% if not proj.complete %}
-  <li><a href="{{ url_for('project', name=proj.name) }}">{{ proj.title }}</a> ({{ proj.feature_count }})</li>
+  {% if not proj.hidden or is_admin(proj) %}
+  <li>{% if proj.hidden %}🕶️ {% endif %}<a href="{{ url_for('project', name=proj.name) }}">{{ proj.title }}</a> ({{ proj.feature_count }})</li>
   {% endif %}
   {% endfor %}
 </ul>
-{% if admin %}
-<p>Archived validated imports:</p>
-<ul>
-  {% for proj in projects %}
-  {% if proj.complete %}
-  <li><a href="{{ url_for('project', name=proj.name) }}">{{ proj.title }}</a> ({{ proj.feature_count }})</li>
-  {% endif %}
-  {% endfor %}
-</ul>
-{% endif %}
 {% if not user %}
 <p><a href="{{ url_for('login') }}">Login to validate imports</a></p>
 {% else %}

www/templates/newproject.html

@@ -11,6 +11,8 @@ <h1>{% if project.name %}Update{% else %}Create{% endif %} a project</h1>
     <textarea name="description" cols="100" rows="10">{{ project.description or '' }}</textarea><br>
     <br>
     JSON: <input type="file" name="json"><br>
+    Audit: <input type="file" name="audit"><br>
+    <input type="checkbox" id="is_hidden" name="is_hidden" {% if project.hidden %}checked="checked"{% endif %}><label for="is_hidden"> Hide from the projects list</label><br>
     <input type="checkbox" id="validate" name="validate" {% if project.can_validate %}checked="checked"{% endif %}><label for="validate"> Enable validation</label><br>
     <br>
     <input type="submit">

www/templates/project.html

@@ -45,5 +45,8 @@ <h1>{{ project.title }}</h1>
     <li>Have corrections: {{ corrected }}</li>
     <li>To be ignored: {{ skipped }}</li>
 </ul>
+{% if has_skipped %}
+<p><a href="{{ url_for('clear_skipped', pid=project.id) }}">Put skipped items back</a> on your review list</p>
+{% endif %}
 <p><a href="{{ url_for('front') }}">Return</a></p>
 {% endblock %}