diff --git a/requirements.txt b/requirements.txt index 66d30670..64a6d170 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -sqlalchemy==1.4.45 +sqlalchemy==2.0.32 python-dateutil==2.8.2 colorama==0.4.6 pyreadline3==3.4.1; platform_system == 'Windows' diff --git a/scripts/coverage b/scripts/coverage index 9cbfc7f1..62e5b650 100755 --- a/scripts/coverage +++ b/scripts/coverage @@ -5,3 +5,4 @@ cd $(dirname $0)/.. coverage run --source=yokadi --omit="yokadi/tests/*" -m pytest yokadi/tests/tests.py coverage report +coverage html diff --git a/setup.py b/setup.py index b15eddc0..8147e159 100755 --- a/setup.py +++ b/setup.py @@ -78,7 +78,7 @@ def createFileList(sourceDir, *patterns): # distutils does not support install_requires, but pip needs it to be # able to automatically install dependencies install_requires=[ - "sqlalchemy ~= 1.4.45", + "sqlalchemy ~= 2.0.32", "python-dateutil ~= 2.8.2", "colorama ~= 0.4.6", "pyreadline3 ~= 3.4.1 ; platform_system == 'Windows'", diff --git a/yokadi/core/db.py b/yokadi/core/db.py index ea58e317..60da232e 100644 --- a/yokadi/core/db.py +++ b/yokadi/core/db.py @@ -12,9 +12,8 @@ from uuid import uuid1 from sqlalchemy import create_engine, inspect -from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.ext.associationproxy import association_proxy -from sqlalchemy.orm import scoped_session, sessionmaker, relationship +from sqlalchemy.orm import scoped_session, sessionmaker, relationship, declarative_base from sqlalchemy.orm.exc import NoResultFound from sqlalchemy.exc import IntegrityError from sqlalchemy import Column, Integer, Boolean, Unicode, DateTime, Enum, ForeignKey, UniqueConstraint @@ -54,7 +53,7 @@ class Project(Base): uuid = Column(Unicode, unique=True, default=uuidGenerator, nullable=False) name = Column(Unicode, unique=True) active = Column(Boolean, default=True) - tasks = relationship("Task", cascade="all", backref="project") + tasks = relationship("Task", cascade="all", backref="project", cascade_backrefs=False) def __repr__(self): return self.name @@ -83,7 +82,7 @@ class Keyword(Base): id = Column(Integer, primary_key=True) name = Column(Unicode, unique=True) tasks = association_proxy("taskKeywords", "task") - taskKeywords = relationship("TaskKeyword", cascade="all", backref="keyword") + taskKeywords = relationship("TaskKeyword", cascade="all", backref="keyword", cascade_backrefs=False) def __repr__(self): return self.name @@ -91,6 +90,7 @@ def __repr__(self): class TaskKeyword(Base): __tablename__ = "task_keyword" + __mapper_args__ = {"confirm_deleted_rows": False} id = Column(Integer, primary_key=True) taskId = Column("task_id", Integer, ForeignKey("task.id"), nullable=False) keywordId = Column("keyword_id", Integer, ForeignKey("keyword.id"), nullable=False) @@ -138,8 +138,8 @@ class Task(Base): status = Column(Enum("new", "started", "done"), default="new") recurrence = Column(RecurrenceRuleColumnType, nullable=False, default=RecurrenceRule()) projectId = Column("project_id", Integer, ForeignKey("project.id"), nullable=False) - taskKeywords = relationship("TaskKeyword", cascade="all", backref="task") - lock = relationship("TaskLock", cascade="all", backref="task") + taskKeywords = relationship("TaskKeyword", cascade="all", backref="task", cascade_backrefs=False) + lock = relationship("TaskLock", cascade="all", backref="task", cascade_backrefs=False) def setKeywordDict(self, dct): """ @@ -350,8 +350,7 @@ def createTables(self): Base.metadata.create_all(self.engine) def getVersion(self): - inspector = inspect(self.engine) - if not inspector.has_table("config"): + if not self._hasConfigTable(): # There was no Config table in v1 return 1 @@ -361,12 +360,16 @@ def getVersion(self): raise YokadiException("Configuration key '%s' does not exist. This should not happen!" % DB_VERSION_KEY) def setVersion(self, version): - assert self.engine.has_table("config") + assert self._hasConfigTable() instance = self.session.query(Config).filter_by(name=DB_VERSION_KEY).one() instance.value = str(version) self.session.add(instance) self.session.commit() + def _hasConfigTable(self): + inspector = inspect(self.engine) + return inspector.has_table("config") + def checkVersion(self): """Check version and exit if it is not suitable""" version = self.getVersion() diff --git a/yokadi/tests/bugtestcase.py b/yokadi/tests/bugtestcase.py index bf28a63f..1670e3da 100644 --- a/yokadi/tests/bugtestcase.py +++ b/yokadi/tests/bugtestcase.py @@ -33,7 +33,7 @@ def testAdd(self): expected = ["t1"] self.assertEqual(result, expected) - kwDict = self.session.query(Task).get(1).getKeywordDict() + kwDict = self.session.get(Task, 1).getKeywordDict() self.assertEqual(kwDict, dict(_severity=2, _likelihood=4, _bug=123)) for bad_input in ("", # No project diff --git a/yokadi/tests/dbtestcase.py b/yokadi/tests/dbtestcase.py new file mode 100644 index 00000000..fc898ca0 --- /dev/null +++ b/yokadi/tests/dbtestcase.py @@ -0,0 +1,20 @@ +""" +@author: Aurélien Gâteau +@license: GPL v3 or later +""" + +import unittest + +from yokadi.core import db + + +class DbTestCase(unittest.TestCase): + def setUp(self): + db.connectDatabase("", memoryDatabase=True) + self.session = db.getSession() + + def testSetVersion(self): + newVersion = db.DB_VERSION + 1 + db._database.setVersion(newVersion) + version = db._database.getVersion() + self.assertEqual(version, newVersion) diff --git a/yokadi/tests/icaltestcase.py b/yokadi/tests/icaltestcase.py index 8083ecdb..046feabe 100644 --- a/yokadi/tests/icaltestcase.py +++ b/yokadi/tests/icaltestcase.py @@ -4,7 +4,11 @@ @author: Sébastien Renard @license: GPL v3 or later """ +from datetime import datetime, timedelta + +import icalendar from yokadi.ycli import tui +from yokadi.ycli.projectcmd import getProjectFromName from yokadi.yical import yical from yokadi.core import dbutils from yokadi.core import db @@ -111,8 +115,87 @@ def testKeywordMapping(self): def testTaskDoneMapping(self): tui.addInputAnswers("y") t1 = dbutils.addTask("x", "t1", {}) - yical.createVTodoFromTask(t1) + v1 = yical.createVTodoFromTask(t1) + + completed = datetime.now() + v1.add("COMPLETED", completed) + yical.updateTaskFromVTodo(t1, v1) + self.assertEqual(t1.status, "done") + self.assertEqual(t1.doneDate, completed) + + def testGenerateCal(self): + # Add an inactive project + t1 = dbutils.addTask("p1", "t1", interactive=False) + project = getProjectFromName("p1") + project.active = False + + # And an active project with 3 tasks, one of them is done + t2new = dbutils.addTask("p2", "t2new", interactive=False) + + t2started = dbutils.addTask("p2", "t2started", interactive=False) + t2started.setStatus("started") + + t2done = dbutils.addTask("p2", "t2done", interactive=False) + t2done.setStatus("done") + + self.session.commit() + + # Generate the calendar + cal = yical.generateCal() + + # It should contain only "p2", "t1" and "t2new" and "t2started" + # I am not sure that it should contain "t1" (since its project is not active), but that's the current behavior + summaries = sorted(str(x["SUMMARY"]) for x in cal.subcomponents) + expected = sorted(["p2", f"t1 ({t1.id})", f"t2new ({t2new.id})", f"t2started ({t2started.id})"]) + + self.assertEqual(summaries, expected) + + def testHandlerProcessVTodoModifyTask(self): + # Create a task + task = dbutils.addTask("p1", "t1", interactive=False) + self.session.commit() - # v1["completed"] = datetime.datetime.now() - # yical.updateTaskFromVTodo(t1, v1) - # self.assertEqual(t1.status, "done") + # Create a vTodo to modify the task + modified = datetime.now() + created = modified + timedelta(hours=-1) + vTodo = icalendar.Todo() + vTodo["UID"] = yical.TASK_UID % str(task.id) + vTodo.add("CREATED", created) + vTodo.add("LAST-MODIFIED", modified) + vTodo.add("summary", "new title") + + # Process the vTodo + newTaskDict = {} + yical.IcalHttpRequestHandler.processVTodo(newTaskDict, vTodo) + + # The task title must have changed + task = dbutils.getTaskFromId(task.id) + self.assertEqual(task.title, "new title") + + # newTaskDict must not have changed + self.assertEqual(newTaskDict, {}) + + def testHandlerProcessVTodoCreateTask(self): + # Create a vTodo to add a new task + modified = datetime.now() + created = modified + timedelta(hours=-1) + vTodo = icalendar.Todo() + vTodo["UID"] = "zogzog" + vTodo.add("summary", "new task") + + # Process the vTodo + newTaskDict = {} + yical.IcalHttpRequestHandler.processVTodo(newTaskDict, vTodo) + + # The task should be in newTaskDict + newTaskList = list(newTaskDict.items()) + self.assertEqual(len(newTaskList), 1) + + (uid, taskId) = newTaskList[0] + + # And the task can be retrieved + task = dbutils.getTaskFromId(taskId) + self.assertEqual(task.title, "new task") + + # And there is only one task + self.assertEqual(self.session.query(db.Task).count(), 1) diff --git a/yokadi/tests/tasktestcase.py b/yokadi/tests/tasktestcase.py index 9a03016a..5c288aec 100644 --- a/yokadi/tests/tasktestcase.py +++ b/yokadi/tests/tasktestcase.py @@ -6,6 +6,7 @@ @license: GPL v3 or later """ from yokadi.tests.yokaditestcase import YokadiTestCase +from unittest.mock import patch import testutils @@ -43,7 +44,7 @@ def testAdd(self): for x in tasks: self.assert_(x.uuid) - kwDict = self.session.query(Task).get(2).getKeywordDict() + kwDict = self.session.get(Task, 2).getKeywordDict() self.assertEqual(kwDict, dict(kw1=None, kw2=12)) for bad_input in ("", # No project @@ -57,7 +58,7 @@ def testEdit(self): tui.addInputAnswers("newtxt") self.cmd.do_t_edit("1") - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertEqual(task.title, "newtxt") self.assertEqual(task.getKeywordDict(), {"_note": None}) @@ -68,7 +69,7 @@ def testEditAddKeyword(self): tui.addInputAnswers("txt @kw", "y") self.cmd.do_t_edit("1") - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertEqual(task.title, "txt") self.assertEqual(task.getKeywordDict(), {"kw": None}) @@ -79,7 +80,7 @@ def testEditRemoveKeyword(self): tui.addInputAnswers("txt") self.cmd.do_t_edit("1") - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertEqual(task.title, "txt") self.assertEqual(task.getKeywordDict(), {}) @@ -114,7 +115,7 @@ def testRemove(self): def testMark(self): tui.addInputAnswers("y") self.cmd.do_t_add("x t1") - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertEqual(task.status, "new") self.cmd.do_t_mark_started("1") self.assertEqual(task.status, "started") @@ -126,7 +127,7 @@ def testMark(self): def testAddKeywords(self): tui.addInputAnswers("y") self.cmd.do_t_add("x t1") - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) tui.addInputAnswers("y", "y") self.cmd.do_t_add_keywords("1 @kw1 @kw2=12") @@ -144,17 +145,17 @@ def testSetProject(self): self.cmd.do_t_add("x t1") tui.addInputAnswers("y") self.cmd.do_t_project("1 y") - task1 = self.session.query(Task).get(1) + task1 = self.session.get(Task, 1) self.assertEqual(task1.project.name, "y") self.cmd.do_t_add("x t2") self.cmd.do_t_project("1 _") - task1 = self.session.query(Task).get(1) + task1 = self.session.get(Task, 1) self.assertEqual(task1.project.name, "x") tui.addInputAnswers("n") self.cmd.do_t_project("1 doesnotexist") - task1 = self.session.query(Task).get(1) + task1 = self.session.get(Task, 1) self.assertEqual(task1.project.name, "x") def testLastTaskId(self): @@ -163,11 +164,11 @@ def testLastTaskId(self): tui.addInputAnswers("y") self.cmd.do_t_add("x t1") - task1 = self.session.query(Task).get(1) + task1 = self.session.get(Task, 1) self.assertEqual(self.cmd.getTaskFromId("_"), task1) self.cmd.do_t_add("x t2") - task2 = self.session.query(Task).get(2) + task2 = self.session.get(Task, 2) self.assertEqual(self.cmd.getTaskFromId("_"), task2) self.cmd.do_t_mark_started("1") @@ -178,15 +179,15 @@ def testLastProjectName(self): self.assertRaises(YokadiException, self.cmd.do_t_add, "_ t1") tui.addInputAnswers("y") self.cmd.do_t_add("x t1") - task1 = self.session.query(Task).get(1) + task1 = self.session.get(Task, 1) self.cmd.do_t_add("_ t2") - task2 = self.session.query(Task).get(2) + task2 = self.session.get(Task, 2) self.assertEqual(task1.project, task2.project) def testRecurs(self): tui.addInputAnswers("y") self.cmd.do_t_add("x t1") - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.cmd.do_t_recurs("1 daily 10:00") self.assertTrue(task.recurrence) @@ -315,7 +316,7 @@ def testTApply(self): ids = [1, 2, 4, 5, 6, 9] self.cmd.do_t_apply("1 2,4-6 9 t_add_keywords @lala") for taskId in range(1, 10): - kwDict = self.session.query(Task).get(taskId).getKeywordDict() + kwDict = self.session.get(Task, taskId).getKeywordDict() if taskId in ids: self.assertEqual(kwDict, dict(lala=None)) else: @@ -327,13 +328,13 @@ def testTApply(self): self.cmd.do_t_list("@lala") self.cmd.do_t_apply("__ t_add_keywords @toto") for taskId in range(1, 10): - kwDict = self.session.query(Task).get(taskId).getKeywordDict() + kwDict = self.session.get(Task, taskId).getKeywordDict() if taskId in ids: self.assertEqual(kwDict, dict(lala=None, toto=None)) else: self.assertNotEqual(kwDict, dict(lala=None, toto=None)) - def testReorder(self): + def testReorderFailsOnInvalidInputs(self): self.assertRaises(BadUsageException, self.cmd.do_t_reorder, "unknown_project") self.assertRaises(BadUsageException, self.cmd.do_t_reorder, "too much args") @@ -350,12 +351,12 @@ def testToNote(self): self.cmd.do_t_add("x t1") self.cmd.do_t_to_note(1) - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertTrue(task.isNote(self.session)) # Doing it twice should not fail self.cmd.do_t_to_note(1) - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertTrue(task.isNote(self.session)) def testToTask(self): @@ -363,12 +364,26 @@ def testToTask(self): self.cmd.do_n_add("x t1") self.cmd.do_n_to_task(1) - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertFalse(task.isNote(self.session)) # Doing it twice should not fail self.cmd.do_n_to_task(1) - task = self.session.query(Task).get(1) + task = self.session.get(Task, 1) self.assertFalse(task.isNote(self.session)) + @patch("yokadi.ycli.tui.editText") + def testReorder(self, editTextMock): + t1, t2, t3 = [dbutils.addTask("x", f"t{x}", interactive=False) for x in range(1, 4)] + + # Simulate moving t3 from 3rd line to the 1st + editTextMock.return_value = "3,t3\n1,t1\n2,t2" + + self.cmd.do_t_reorder("x") + editTextMock.assert_called_with("1,t1\n2,t2\n3,t3") + + self.assertEqual(t3.urgency, 2) + self.assertEqual(t1.urgency, 1) + self.assertEqual(t2.urgency, 0) + # vi: ts=4 sw=4 et diff --git a/yokadi/tests/tests.py b/yokadi/tests/tests.py index 6019aa55..ae65862e 100755 --- a/yokadi/tests/tests.py +++ b/yokadi/tests/tests.py @@ -51,6 +51,7 @@ from syncmanagertestcase import SyncManagerTestCase # noqa: F401, E402 from argstestcase import ArgsTestCase # noqa: F401, E402 from dbs13ntestcase import Dbs13nTestCase # noqa: F401, E402 +from dbtestcase import DbTestCase # noqa: F401, E402 def main(): diff --git a/yokadi/update/update9to10.py b/yokadi/update/update9to10.py index 5025d5a1..21703276 100644 --- a/yokadi/update/update9to10.py +++ b/yokadi/update/update9to10.py @@ -35,7 +35,11 @@ def createByweekdayValue(rule): def createJsonStringFromRule(pickledRule): - rule = pickle.loads(pickledRule) + try: + rule = pickle.loads(pickledRule) + except UnicodeDecodeError: + # Some rules fails to unpickle if we don't set encoding to latin1 + rule = pickle.loads(pickledRule, encoding="latin1") dct = {} dct["freq"] = rule._freq dct["bymonth"] = tuplify(rule._bymonth) diff --git a/yokadi/ycli/taskcmd.py b/yokadi/ycli/taskcmd.py index cb81009f..b307add6 100644 --- a/yokadi/ycli/taskcmd.py +++ b/yokadi/ycli/taskcmd.py @@ -648,7 +648,7 @@ def do_t_reorder(self, line): ids.reverse() for urgency, taskId in enumerate(ids): - task = self.session.query(Task).get(taskId) + task = self.session.get(Task, taskId) task.urgency = urgency self.session.commit() complete_t_reorder = ProjectCompleter(1) diff --git a/yokadi/yical/yical.py b/yokadi/yical/yical.py index 1cf54d1e..68642e07 100644 --- a/yokadi/yical/yical.py +++ b/yokadi/yical/yical.py @@ -168,7 +168,7 @@ def do_PUT(self): for vTodo in cal.walk(): if "UID" in vTodo: try: - self._processVTodo(vTodo) + IcalHttpRequestHandler.processVTodo(self.newTask, vTodo) except YokadiException as e: self.send_response(503, e) @@ -176,20 +176,24 @@ def do_PUT(self): self.send_response(200) self.end_headers() - def _processVTodo(self, vTodo): + # This is static method to make it easier to test + @staticmethod + def processVTodo(newTaskDict, vTodo): session = db.getSession() - if vTodo["UID"] in self.newTask: + uid = vTodo["UID"] + if uid in newTaskDict: # This is a recent new task but remote ical calendar tool is not # aware of new Yokadi UID. Update it here to avoid duplicate new tasks print("update UID to avoid duplicate task") - vTodo["UID"] = TASK_UID % self.newTask[vTodo["UID"]] + uid = TASK_UID % newTaskDict[uid] + vTodo["UID"] = uid - if vTodo["UID"].startswith(icalutils.UID_PREFIX): + if uid.startswith(icalutils.UID_PREFIX): # This is a yokadi Task. if vTodo["LAST-MODIFIED"].dt > vTodo["CREATED"].dt: # Task has been modified - print("Modified task: %s" % vTodo["UID"]) - result = TASK_RE.match(vTodo["UID"]) + print("Modified task: %s" % uid) + result = TASK_RE.match(uid) if result: id = result.group(1) task = dbutils.getTaskFromId(id) @@ -197,10 +201,10 @@ def _processVTodo(self, vTodo): updateTaskFromVTodo(task, vTodo) session.commit() else: - raise YokadiException("Task %s does exist in yokadi db " % id) + raise YokadiException("Task %s does exist in yokadi db " % uid) else: # This is a new task - print("New task %s (%s)" % (vTodo["summary"], vTodo["UID"])) + print("New task %s (%s)" % (vTodo["summary"], uid)) keywordDict = {} task = dbutils.addTask(INBOX_PROJECT, vTodo["summary"], keywordDict, interactive=False) @@ -210,7 +214,7 @@ def _processVTodo(self, vTodo): # if user update it right after creation without reloading the # yokadi UID # TODO: add purge for old UID - self.newTask[vTodo["UID"]] = task.id + newTaskDict[uid] = task.id class YokadiIcalServer(Thread):