diff --git a/.gitignore b/.gitignore index a95aaac..ead6316 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ config_local.py *.db *.swp *.log +*.backup diff --git a/www/audit.py b/www/audit.py index 7150d2f..37f6e1d 100644 --- a/www/audit.py +++ b/www/audit.py @@ -46,16 +46,25 @@ def get_user(): return None -def is_admin(user): - return user and user.uid in config.ADMINS +def is_admin(user, project=None): + if not user: + return False + if user.uid in config.ADMINS: + return True + if not project: + return user.admin + return user == project.owner @app.route('/') def front(): user = get_user() projects = Project.select().order_by(Project.updated.desc()) + + def local_is_admin(proj): + return is_admin(user, proj) return render_template('index.html', user=user, projects=projects, - admin=is_admin(user)) + admin=is_admin(user), is_admin=local_is_admin) @app.route('/login') @@ -121,9 +130,15 @@ def project(name): Feature.project == project, Feature.audit.is_null(False), Feature.audit != '').count() skipped = Feature.select(Feature.id).where( Feature.project == project, Feature.audit.contains('"skip": true')).count() - return render_template('project.html', project=project, admin=is_admin(get_user()), + user = get_user() + if user: + has_skipped = Task.select().join(Feature).where( + Task.user == user, Task.skipped == True, Feature.project == project).count() > 0 + else: + has_skipped = False + return render_template('project.html', project=project, admin=is_admin(user, project), desc=desc, val1=val1, val2=val2, corrected=corrected, - skipped=skipped) + skipped=skipped, has_skipped=has_skipped) @app.route('/browse/') @@ -271,7 +286,7 @@ def add_project(pid=None): return render_template('newproject.html', project=project) -def update_features(project, features): +def update_features(project, features, audit): curfeats = Feature.select(Feature).where(Feature.project == project) ref2feat = {f.ref: f for f in curfeats} deleted = set(ref2feat.keys()) @@ -282,6 +297,7 @@ def update_features(project, features): md5 = hashlib.md5() md5.update(data.encode('utf-8')) md5_hex = md5.hexdigest() + coord = f['geometry']['coordinates'] if coord[0] < minlon: minlon = coord[0] @@ -291,10 +307,12 @@ def update_features(project, features): minlat = coord[1] if coord[1] > maxlat: maxlat = coord[1] + if 'ref_id' in f['properties']: ref = f['properties']['ref_id'] else: ref = '{}{}'.format(f['properties']['osm_type'], f['properties']['osm_id']) + update = False if ref in ref2feat: deleted.remove(ref) @@ -305,6 +323,12 @@ def update_features(project, features): feat = Feature(project=project, ref=ref) feat.validates_count = 0 update = True + + f_audit = audit.get(ref) + if f_audit and f_audit != feat.audit: + feat.audit = f_audit + update = True + if update: feat.feature = data feat.feature_md5 = md5_hex @@ -312,6 +336,7 @@ def update_features(project, features): feat.lat = round(coord[1] * 1e7) feat.action = f['properties']['action'][0] feat.save() + if deleted: q = Feature.delete().where(Feature.ref << list(deleted)) q.execute() @@ -326,12 +351,15 @@ def add_flash(pid, msg): flash(msg) return redirect(url_for('add_project', pid=pid)) - if not is_admin(get_user()): + user = get_user() + if not is_admin(user): return redirect(url_for('front')) pid = request.form['pid'] if pid: pid = int(pid) project = Project.get(Project.id == pid) + if not is_admin(user, project): + return redirect(url_for('front')) else: pid = None project = Project() @@ -348,6 +376,10 @@ def add_flash(pid, msg): project.url = None project.description = request.form['description'].strip() project.can_validate = request.form.get('validate') is not None + project.hidden = request.form.get('is_hidden') is not None + if not project.owner or user.uid not in config.ADMINS: + project.owner = user + if 'json' not in request.files or request.files['json'].filename == '': if not pid: return add_flash(pid, 'Would not create a project without features') @@ -356,17 +388,27 @@ def add_flash(pid, msg): try: features = json.load(codecs.getreader('utf-8')(request.files['json'])) except ValueError as e: - return add_flash(pid, 'Error in the uploaded file: {}'.format(e)) + return add_flash(pid, 'Error in the uploaded features file: {}'.format(e)) if 'features' not in features or not features['features']: return add_flash(pid, 'No features found in the JSON file') features = features['features'] - project.updated = datetime.datetime.utcnow().date() + audit = None + if 'audit' in request.files and request.files['audit'].filename: + try: + audit = json.load(codecs.getreader('utf-8')(request.files['audit'])) + except ValueError as e: + return add_flash(pid, 'Error in the uploaded audit file: {}'.format(e)) + if not audit: + return add_flash(pid, 'No features found in the audit JSON file') + + if features or audit or not project.updated: + project.updated = datetime.datetime.utcnow().date() project.save() if features: with database.atomic(): - update_features(project, features) + update_features(project, features, audit or {}) if project.feature_count == 0: project.delete_instance() @@ -375,20 +417,33 @@ def add_flash(pid, msg): return redirect(url_for('project', name=project.name)) +@app.route('/clear_skipped/') +def clear_skipped(pid): + project = Project.get(Project.id == pid) + user = get_user() + if user: + features = Feature.select().where(Feature.project == project) + query = Task.delete().where( + Task.user == user, Task.skipped == True, + Task.feature.in_(features)) + query.execute() + return redirect(url_for('project', name=project.name)) + + @app.route('/delete/') def delete_project(pid): - if not is_admin(get_user()): - return redirect(url_for('front')) project = Project.get(Project.id == pid) + if not is_admin(get_user(), project): + return redirect(url_for('front')) project.delete_instance(recursive=True) return redirect(url_for('front')) @app.route('/export_audit/') def export_audit(pid): - if not is_admin(get_user()): - return redirect(url_for('front')) project = Project.get(Project.id == pid) + if not is_admin(get_user(), project): + return redirect(url_for('front')) query = Feature.select(Feature.ref, Feature.audit).where( Feature.project == project, Feature.audit.is_null(False)).tuples() audit = {} @@ -400,6 +455,24 @@ def export_audit(pid): headers={'Content-Disposition': 'attachment;filename=audit_{}.json'.format(project.name)}) +@app.route('/admin') +def admin(): + user = get_user() + if not user or user.uid not in config.ADMINS: + return redirect(url_for('front')) + admin_uids = User.select(User.uid).where(User.admin == True).tuples() + uids = '\n'.join([str(u[0]) for u in admin_uids]) + return render_template('admin.html', uids=uids) + + +@app.route('/admin_users', methods=['POST']) +def admin_users(): + uids = [int(x.strip()) for x in request.form['uids'].split()] + User.update(admin=False).where(User.uid.not_in(uids)).execute() + User.update(admin=True).where(User.uid.in_(uids)).execute() + return redirect(url_for('admin')) + + @app.route('/profile', methods=['GET', 'POST']) def profile(): user = get_user() @@ -453,11 +526,12 @@ def api_feature(pid): if user and request.method == 'POST' and project.can_validate: ref_and_audit = request.get_json() if ref_and_audit and len(ref_and_audit) == 2: + skipped = ref_and_audit[1] is None feat = Feature.get(Feature.project == project, Feature.ref == ref_and_audit[0]) user_did_it = Task.select(Task.id).where( Task.user == user, Task.feature == feat).count() > 0 - Task.create(user=user, feature=feat) - if ref_and_audit[1] is not None: + Task.create(user=user, feature=feat, skipped=skipped) + if not skipped: if len(ref_and_audit[1]): new_audit = json.dumps(ref_and_audit[1], sort_keys=True, ensure_ascii=False) else: diff --git a/www/db.py b/www/db.py index f4cdcf8..7bf620e 100644 --- a/www/db.py +++ b/www/db.py @@ -23,6 +23,12 @@ class Meta: database = database +class User(BaseModel): + uid = IntegerField(primary_key=True) + admin = BooleanField(default=False) + bboxes = TextField(null=True) + + class Project(BaseModel): name = CharField(max_length=32, index=True, unique=True) title = CharField(max_length=250) @@ -30,8 +36,11 @@ class Project(BaseModel): url = CharField(max_length=1000, null=True) feature_count = IntegerField() can_validate = BooleanField() + hidden = BooleanField(default=False) bbox = CharField(max_length=60) updated = DateField() + owner = ForeignKeyField(User, related_name='projects') + overlays = TextField(null=True) class Feature(BaseModel): @@ -46,29 +55,26 @@ class Feature(BaseModel): validates_count = IntegerField(default=0) -class User(BaseModel): - uid = IntegerField(primary_key=True) - bboxes = TextField(null=True) - - class Task(BaseModel): user = ForeignKeyField(User, index=True, related_name='tasks') feature = ForeignKeyField(Feature, index=True, on_delete='CASCADE') + skipped = BooleanField(default=False) -LAST_VERSION = 0 +LAST_VERSION = 1 class Version(BaseModel): version = IntegerField() +@database.atomic() def migrate(): database.create_tables([Version], safe=True) try: v = Version.select().get() except Version.DoesNotExist: - database.create_tables([User, Project, Feature, Task], safe=True) + database.create_tables([User, Project, Feature, Task]) v = Version(version=LAST_VERSION) v.save() @@ -83,9 +89,19 @@ def migrate(): migrator = PostgresqlMigrator(database) if v.version == 0: + # Making a copy of Project.owner field, because it's not nullable + # and we need to migrate a default value. + admin = User.select(User.uid).where(User.uid == list(config.ADMINS)[0]).get() + owner = ForeignKeyField(User, related_name='projects', to_field=User.uid, default=admin) + peewee_migrate( - # TODO - migrator.add_column(User._meta.db_table, User.lang.name, User.lang) + migrator.add_column(User._meta.db_table, User.admin.db_column, User.admin), + migrator.add_column(Project._meta.db_table, Project.owner.db_column, owner), + migrator.add_column(Project._meta.db_table, Project.hidden.db_column, Project.hidden), + migrator.add_column(Project._meta.db_table, Project.overlays.db_column, + Project.overlays), + migrator.add_column(Task._meta.db_table, Task.skipped.db_column, Task.skipped), + migrator.drop_column(Project._meta.db_table, 'validated_count'), ) v.version = 1 v.save() diff --git a/www/templates/admin.html b/www/templates/admin.html new file mode 100644 index 0000000..1465c3c --- /dev/null +++ b/www/templates/admin.html @@ -0,0 +1,12 @@ +{% extends "layout.html" %} +{% block title %}Administration — {% endblock %} +{% block content %} +

Administration

+
+ Type user ids to allow uploading projects:
+
+
+ +
+

Return

+{% endblock %} diff --git a/www/templates/index.html b/www/templates/index.html index c0c6b43..52919cf 100644 --- a/www/templates/index.html +++ b/www/templates/index.html @@ -4,24 +4,14 @@

Auditing Tool for OSM Conflator

{% if admin %}

Create a project

{% endif %} -

Current imports that need validating:

+

Imports that need validating:

    {% for proj in projects %} - {% if not proj.complete %} -
  • {{ proj.title }} ({{ proj.feature_count }})
    • + {% if not proj.hidden or is_admin(proj) %} +
  • {% if proj.hidden %}🕶️ {% endif %}{{ proj.title }} ({{ proj.feature_count }})
    • {% endif %} {% endfor %}
-{% if admin %} -

Archived validated imports:

-
    - {% for proj in projects %} - {% if proj.complete %} -
  • {{ proj.title }} ({{ proj.feature_count }})
    • - {% endif %} - {% endfor %} -
-{% endif %} {% if not user %}

Login to validate imports

{% else %} diff --git a/www/templates/newproject.html b/www/templates/newproject.html index acca970..7774ef0 100644 --- a/www/templates/newproject.html +++ b/www/templates/newproject.html @@ -11,6 +11,8 @@

{% if project.name %}Update{% else %}Create{% endif %} a project



JSON:
+ Audit:
+


diff --git a/www/templates/project.html b/www/templates/project.html index 8b27a90..5e3c695 100644 --- a/www/templates/project.html +++ b/www/templates/project.html @@ -45,5 +45,8 @@

{{ project.title }}

  • Have corrections: {{ corrected }}
  • To be ignored: {{ skipped }}
    • +{% if has_skipped %} +

    Put skipped items back on your review list

    +{% endif %}

    Return

    {% endblock %}