diff --git a/master/buildbot/test/fake/fakemaster.py b/master/buildbot/test/fake/fakemaster.py index 318e8492d04..9080059b702 100644 --- a/master/buildbot/test/fake/fakemaster.py +++ b/master/buildbot/test/fake/fakemaster.py @@ -36,6 +36,7 @@ from buildbot.test.fake.botmaster import FakeBotMaster from buildbot.test.fake.machine import FakeMachineManager from buildbot.test.fake.secrets import FakeSecretStorage +from buildbot.test.util.db import resolve_test_db_url from buildbot.util import service from buildbot.util.twisted import async_to_deferred @@ -167,7 +168,7 @@ async def make_master( wantGraphql=False, with_secrets: dict | None = None, url=None, - db_url='sqlite://', + db_url=None, sqlite_memory=True, auto_upgrade=True, auto_shutdown=True, @@ -202,17 +203,14 @@ async def make_master( ) master._test_want_db = True - if db_url == 'sqlite://' and not sqlite_memory: - db_url = 'sqlite:///tmp.sqlite' - if not os.path.exists(master.basedir): - os.makedirs(master.basedir) - if auto_shutdown: # Add before setup so that failed database setup would still be closed and wouldn't # affect further tests testcase.addCleanup(master.test_shutdown) - master.db.configured_url = db_url + master.db.configured_url = resolve_test_db_url(db_url, sqlite_memory) + if not os.path.exists(master.basedir): + os.makedirs(master.basedir) await master.db.set_master(master) await master.db.setup() diff --git a/master/buildbot/test/integration/test_upgrade.py b/master/buildbot/test/integration/test_upgrade.py index 28621d244ce..93752eee793 100644 --- a/master/buildbot/test/integration/test_upgrade.py +++ b/master/buildbot/test/integration/test_upgrade.py @@ -71,20 +71,22 @@ def setUpUpgradeTest(self): # get the top-level dir from the tarball assert len(prefixes) == 1, "tarball has multiple top-level dirs!" self.basedir = prefixes.pop() + db_url = 'sqlite:///' + os.path.abspath(os.path.join(self.basedir, 'state.sqlite')) else: if not os.path.exists("basedir"): os.makedirs("basedir") self.basedir = os.path.abspath("basedir") + db_url = None self.master = yield fakemaster.make_master( self, basedir=self.basedir, wantDb=True, - db_url='sqlite:///' + os.path.abspath(os.path.join(self.basedir, 'state.sqlite')), + db_url=db_url, sqlite_memory=False, auto_upgrade=False, check_version=False, - auto_clean=False, + auto_clean=False if self.source_tarball else True, ) self._sql_log_handler = querylog.start_log_queries() diff --git a/master/buildbot/test/unit/process/test_users_manual.py b/master/buildbot/test/unit/process/test_users_manual.py index 8665b6284bb..c8db4514cdf 100644 --- a/master/buildbot/test/unit/process/test_users_manual.py +++ b/master/buildbot/test/unit/process/test_users_manual.py @@ -139,8 +139,8 @@ def test_perspective_commandline_remove(self): yield self.call_perspective_commandline( 'add', None, None, None, [{'identifier': 'h@c', 'git': 'hi '}] ) - yield self.call_perspective_commandline('remove', None, None, ['x'], None) - res = yield self.master.db.users.getUser('x') + yield self.call_perspective_commandline('remove', None, None, ['h@c'], None) + res = yield self.master.db.users.getUser(1) self.assertEqual(res, None) @defer.inlineCallbacks diff --git a/master/buildbot/test/unit/reporters/test_utils.py b/master/buildbot/test/unit/reporters/test_utils.py index 9ca28bb8f91..74859eb81e0 100644 --- a/master/buildbot/test/unit/reporters/test_utils.py +++ b/master/buildbot/test/unit/reporters/test_utils.py @@ -31,6 +31,10 @@ from buildbot.test.util import logging +def sort_builds(builds): + return sorted(builds, key=lambda key: key['buildid']) + + class TestDataUtils(TestReactorMixin, unittest.TestCase, logging.LoggingMixin): LOGCONTENT = textwrap.dedent("""\ line zero @@ -174,6 +178,9 @@ def test_getDetailsForBuildset(self): self.master, 98, want_properties=True, want_steps=True, want_previous_build=True ) self.assertEqual(len(res['builds']), 2) + + res['builds'] = sort_builds(res['builds']) + build1 = res['builds'][0] build2 = res['builds'][1] buildset = res['buildset'] @@ -239,7 +246,7 @@ def test_getDetailsForBuildsetWithLogs(self): want_logs_content=True, ) - build1 = res['builds'][0] + build1 = sort_builds(res['builds'])[0] self.assertEqual( build1['steps'][0]['logs'][0]['content']['content'], self.LOGCONTENT + "\n" ) @@ -262,6 +269,8 @@ def test_get_details_for_buildset_all(self): want_logs_content=True, ) + res['builds'] = sort_builds(res['builds']) + self.assertEqual( res, { diff --git a/master/buildbot/test/unit/www/test_oauth.py b/master/buildbot/test/unit/www/test_oauth.py index 053a1a2af32..f4438c67bdd 100644 --- a/master/buildbot/test/unit/www/test_oauth.py +++ b/master/buildbot/test/unit/www/test_oauth.py @@ -58,7 +58,6 @@ def raise_for_status(self): class OAuth2Auth(TestReactorMixin, www.WwwTestMixin, ConfigErrorsMixin, unittest.TestCase): - @defer.inlineCallbacks def setUp(self): self.setup_test_reactor(auto_tear_down=False) if requests is None: @@ -68,50 +67,85 @@ def setUp(self): self.patch(requests, 'post', mock.Mock(spec=requests.post)) self.patch(requests, 'get', mock.Mock(spec=requests.get)) - self.googleAuth = oauth2.GoogleAuth("ggclientID", "clientSECRET") - self.githubAuth = oauth2.GitHubAuth("ghclientID", "clientSECRET") - self.githubAuth_v4 = oauth2.GitHubAuth("ghclientID", "clientSECRET", apiVersion=4) - self.githubAuth_v4_teams = oauth2.GitHubAuth( + @defer.inlineCallbacks + def setup_google_auth(self): + auth = oauth2.GoogleAuth("ggclientID", "clientSECRET") + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_github_auth(self): + auth = oauth2.GitHubAuth("ghclientID", "clientSECRET") + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_github_auth_v4(self): + auth = oauth2.GitHubAuth("ghclientID", "clientSECRET", apiVersion=4) + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_github_auth_v4_secret(self): + auth = oauth2.GitHubAuth(Secret("client-id"), Secret("client-secret"), apiVersion=4) + master = yield self.make_master(url='h:/a/b/', auth=auth) + fake_storage_service = FakeSecretStorage() + fake_storage_service.reconfigService( + secretdict={"client-id": "secretClientId", "client-secret": "secretClientSecret"} + ) + secret_service = SecretManager() + secret_service.services = [fake_storage_service] + yield secret_service.setServiceParent(master) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_github_auth_v4_teams(self): + auth = oauth2.GitHubAuth( "ghclientID", "clientSECRET", apiVersion=4, getTeamsMembership=True ) - self.githubAuthEnt = oauth2.GitHubAuth( + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_github_auth_enterprise(self): + auth = oauth2.GitHubAuth( "ghclientID", "clientSECRET", serverURL="https://git.corp.fakecorp.com" ) - self.githubAuthEnt_v4 = oauth2.GitHubAuth( + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_github_auth_enterprise_v4(self): + auth = oauth2.GitHubAuth( "ghclientID", "clientSECRET", apiVersion=4, getTeamsMembership=True, serverURL="https://git.corp.fakecorp.com", ) - self.gitlabAuth = oauth2.GitLabAuth("https://gitlab.test/", "glclientID", "clientSECRET") - self.bitbucketAuth = oauth2.BitbucketAuth("bbclientID", "clientSECRET") - - for auth in [ - self.googleAuth, - self.githubAuth, - self.githubAuth_v4, - self.githubAuth_v4_teams, - self.githubAuthEnt, - self.gitlabAuth, - self.bitbucketAuth, - self.githubAuthEnt_v4, - ]: - self._master = master = yield self.make_master(url='h:/a/b/', auth=auth) - auth.reconfigAuth(master, master.config) - - self.githubAuth_secret = oauth2.GitHubAuth( - Secret("client-id"), Secret("client-secret"), apiVersion=4 - ) - self._master = master = yield self.make_master(url='h:/a/b/', auth=auth) - fake_storage_service = FakeSecretStorage() - fake_storage_service.reconfigService( - secretdict={"client-id": "secretClientId", "client-secret": "secretClientSecret"} - ) - secret_service = SecretManager() - secret_service.services = [fake_storage_service] - yield secret_service.setServiceParent(self._master) - self.githubAuth_secret.reconfigAuth(master, master.config) + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_gitlab_auth(self): + auth = oauth2.GitLabAuth("https://gitlab.test/", "glclientID", "clientSECRET") + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth + + @defer.inlineCallbacks + def setup_bitbucket_auth(self): + auth = oauth2.BitbucketAuth("bbclientID", "clientSECRET") + master = yield self.make_master(url='h:/a/b/', auth=auth) + auth.reconfigAuth(master, master.config) + return auth @defer.inlineCallbacks def tearDown(self): @@ -119,7 +153,8 @@ def tearDown(self): @defer.inlineCallbacks def test_getGoogleLoginURL(self): - res = yield self.googleAuth.getLoginURL('http://redir') + auth = yield self.setup_google_auth() + res = yield auth.getLoginURL('http://redir') exp = ( "https://accounts.google.com/o/oauth2/auth?client_id=ggclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -128,7 +163,7 @@ def test_getGoogleLoginURL(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.googleAuth.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://accounts.google.com/o/oauth2/auth?client_id=ggclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -140,7 +175,8 @@ def test_getGoogleLoginURL(self): @defer.inlineCallbacks def test_getGithubLoginURL(self): - res = yield self.githubAuth.getLoginURL('http://redir') + auth = yield self.setup_github_auth() + res = yield auth.getLoginURL('http://redir') exp = ( "https://github.com/login/oauth/authorize?client_id=ghclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -148,7 +184,7 @@ def test_getGithubLoginURL(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.githubAuth.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://github.com/login/oauth/authorize?client_id=ghclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -158,7 +194,8 @@ def test_getGithubLoginURL(self): @defer.inlineCallbacks def test_getGithubLoginURL_with_secret(self): - res = yield self.githubAuth_secret.getLoginURL('http://redir') + auth = yield self.setup_github_auth_v4_secret() + res = yield auth.getLoginURL('http://redir') exp = ( "https://github.com/login/oauth/authorize?client_id=secretClientId&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -166,7 +203,7 @@ def test_getGithubLoginURL_with_secret(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.githubAuth_secret.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://github.com/login/oauth/authorize?client_id=secretClientId&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -176,7 +213,9 @@ def test_getGithubLoginURL_with_secret(self): @defer.inlineCallbacks def test_getGithubELoginURL(self): - res = yield self.githubAuthEnt.getLoginURL('http://redir') + auth = yield self.setup_github_auth_enterprise() + + res = yield auth.getLoginURL('http://redir') exp = ( "https://git.corp.fakecorp.com/login/oauth/authorize?client_id=ghclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -184,7 +223,7 @@ def test_getGithubELoginURL(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.githubAuthEnt.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://git.corp.fakecorp.com/login/oauth/authorize?client_id=ghclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -194,7 +233,9 @@ def test_getGithubELoginURL(self): @defer.inlineCallbacks def test_getGithubLoginURL_v4(self): - res = yield self.githubAuthEnt_v4.getLoginURL('http://redir') + auth = yield self.setup_github_auth_enterprise_v4() + + res = yield auth.getLoginURL('http://redir') exp = ( "https://git.corp.fakecorp.com/login/oauth/authorize?client_id=ghclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -202,7 +243,7 @@ def test_getGithubLoginURL_v4(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.githubAuthEnt_v4.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://git.corp.fakecorp.com/login/oauth/authorize?client_id=ghclientID&" "redirect_uri=h%3A%2Fa%2Fb%2Fauth%2Flogin&response_type=code&" @@ -212,7 +253,9 @@ def test_getGithubLoginURL_v4(self): @defer.inlineCallbacks def test_getGitLabLoginURL(self): - res = yield self.gitlabAuth.getLoginURL('http://redir') + auth = yield self.setup_gitlab_auth() + + res = yield auth.getLoginURL('http://redir') exp = ( "https://gitlab.test/oauth/authorize" "?client_id=glclientID&" @@ -221,7 +264,7 @@ def test_getGitLabLoginURL(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.gitlabAuth.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://gitlab.test/oauth/authorize" "?client_id=glclientID&" @@ -232,7 +275,9 @@ def test_getGitLabLoginURL(self): @defer.inlineCallbacks def test_getBitbucketLoginURL(self): - res = yield self.bitbucketAuth.getLoginURL('http://redir') + auth = yield self.setup_bitbucket_auth() + + res = yield auth.getLoginURL('http://redir') exp = ( "https://bitbucket.org/site/oauth2/authorize?" "client_id=bbclientID&" @@ -241,7 +286,7 @@ def test_getBitbucketLoginURL(self): "state=redirect%3Dhttp%253A%252F%252Fredir" ) self.assertEqual(res, exp) - res = yield self.bitbucketAuth.getLoginURL(None) + res = yield auth.getLoginURL(None) exp = ( "https://bitbucket.org/site/oauth2/authorize?" "client_id=bbclientID&" @@ -252,12 +297,14 @@ def test_getBitbucketLoginURL(self): @defer.inlineCallbacks def test_GoogleVerifyCode(self): + auth = yield self.setup_google_auth() + requests.get.side_effect = [] requests.post.side_effect = [FakeResponse({"access_token": 'TOK3N'})] - self.googleAuth.get = mock.Mock( + auth.get = mock.Mock( side_effect=[{"name": 'foo bar', "email": 'bar@foo', "picture": 'http://pic'}] ) - res = yield self.googleAuth.verifyCode("code!") + res = yield auth.verifyCode("code!") self.assertEqual( { 'avatar_url': 'http://pic', @@ -270,6 +317,8 @@ def test_GoogleVerifyCode(self): @defer.inlineCallbacks def test_GithubVerifyCode(self): + auth = yield self.setup_github_auth() + test = self requests.get.side_effect = [] requests.post.side_effect = [FakeResponse({"access_token": 'TOK3N'})] @@ -296,9 +345,9 @@ def fake_get(self, ep, **kwargs): ] return None - self.githubAuth.get = fake_get + auth.get = fake_get - res = yield self.githubAuth.verifyCode("code!") + res = yield auth.verifyCode("code!") self.assertEqual( { 'email': 'bar@foo', @@ -311,9 +360,11 @@ def fake_get(self, ep, **kwargs): @defer.inlineCallbacks def test_GithubVerifyCode_v4(self): + auth = yield self.setup_github_auth_v4() + requests.get.side_effect = [] requests.post.side_effect = [FakeResponse({"access_token": 'TOK3N'})] - self.githubAuth_v4.post = mock.Mock( + auth.post = mock.Mock( side_effect=[ { 'data': { @@ -329,7 +380,7 @@ def test_GithubVerifyCode_v4(self): } ] ) - res = yield self.githubAuth_v4.verifyCode("code!") + res = yield auth.verifyCode("code!") self.assertEqual( { 'email': 'bar@foo', @@ -342,9 +393,11 @@ def test_GithubVerifyCode_v4(self): @defer.inlineCallbacks def test_GithubVerifyCode_v4_teams(self): + auth = yield self.setup_github_auth_v4_teams() + requests.get.side_effect = [] requests.post.side_effect = [FakeResponse({"access_token": 'TOK3N'})] - self.githubAuth_v4_teams.post = mock.Mock( + auth.post = mock.Mock( side_effect=[ { 'data': { @@ -387,7 +440,7 @@ def test_GithubVerifyCode_v4_teams(self): }, ] ) - res = yield self.githubAuth_v4_teams.verifyCode("code!") + res = yield auth.verifyCode("code!") self.assertEqual( { 'email': 'bar@foo', @@ -424,9 +477,11 @@ def test_GitHubAuthRaiseErrorWithApiV3AndGetTeamMembership(self): @defer.inlineCallbacks def test_GitlabVerifyCode(self): + auth = yield self.setup_gitlab_auth() + requests.get.side_effect = [] requests.post.side_effect = [FakeResponse({"access_token": 'TOK3N'})] - self.gitlabAuth.get = mock.Mock( + auth.get = mock.Mock( side_effect=[ { # /user "name": "Foo Bar", @@ -442,7 +497,7 @@ def test_GitlabVerifyCode(self): ], ] ) - res = yield self.gitlabAuth.verifyCode("code!") + res = yield auth.verifyCode("code!") self.assertEqual( { "full_name": "Foo Bar", @@ -456,9 +511,11 @@ def test_GitlabVerifyCode(self): @defer.inlineCallbacks def test_BitbucketVerifyCode(self): + auth = yield self.setup_bitbucket_auth() + requests.get.side_effect = [] requests.post.side_effect = [FakeResponse({"access_token": 'TOK3N'})] - self.bitbucketAuth.get = mock.Mock( + auth.get = mock.Mock( side_effect=[ {"username": 'bar', "display_name": 'foo bar'}, # /user { @@ -470,7 +527,7 @@ def test_BitbucketVerifyCode(self): {"values": [{'slug': 'hello'}, {'slug': 'grp'}]}, # /workspaces?role=member ] ) - res = yield self.bitbucketAuth.verifyCode("code!") + res = yield auth.verifyCode("code!") self.assertEqual( { 'email': 'bar@foo', @@ -483,6 +540,8 @@ def test_BitbucketVerifyCode(self): @defer.inlineCallbacks def test_loginResource(self): + auth = yield self.setup_github_auth() + class fakeAuth: homeUri = "://me" getLoginURL = mock.Mock(side_effect=lambda x: defer.succeed("://")) @@ -490,7 +549,7 @@ class fakeAuth: acceptToken = mock.Mock(side_effect=lambda token: defer.succeed({"username": "bar"})) userInfoProvider = None - rsrc = self.githubAuth.getLoginResource() + rsrc = auth.getLoginResource() rsrc.auth = fakeAuth() res = yield self.render_resource(rsrc, b'/') rsrc.auth.getLoginURL.assert_called_once_with(None) @@ -507,21 +566,35 @@ class fakeAuth: res = yield self.render_resource(rsrc, b'/?token=token!') rsrc.auth.getLoginURL.assert_called_once() - def test_getConfig(self): + @defer.inlineCallbacks + def test_getConfig_github(self): + auth = yield self.setup_github_auth() self.assertEqual( - self.githubAuth.getConfigDict(), + auth.getConfigDict(), {'fa_icon': 'fa-github', 'autologin': False, 'name': 'GitHub', 'oauth2': True}, ) + + @defer.inlineCallbacks + def test_getConfig_google(self): + auth = yield self.setup_google_auth() self.assertEqual( - self.googleAuth.getConfigDict(), + auth.getConfigDict(), {'fa_icon': 'fa-google-plus', 'autologin': False, 'name': 'Google', 'oauth2': True}, ) + + @defer.inlineCallbacks + def test_getConfig_gitlab(self): + auth = yield self.setup_gitlab_auth() self.assertEqual( - self.gitlabAuth.getConfigDict(), + auth.getConfigDict(), {'fa_icon': 'fa-git', 'autologin': False, 'name': 'GitLab', 'oauth2': True}, ) + + @defer.inlineCallbacks + def test_getConfig_bitbucket(self): + auth = yield self.setup_bitbucket_auth() self.assertEqual( - self.bitbucketAuth.getConfigDict(), + auth.getConfigDict(), {'fa_icon': 'fa-bitbucket', 'autologin': False, 'name': 'Bitbucket', 'oauth2': True}, )