From 3745dc4fe109e61c6c912a9c15889587e4bcfb39 Mon Sep 17 00:00:00 2001 From: Pieter Date: Wed, 10 Apr 2019 22:48:26 +0200 Subject: [PATCH 001/417] Update Dockerfile to get latest image (#162) --- Dockerfile | 2 +- sentry.conf.py | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e212aedc35..e5df0ded33 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1 @@ -FROM sentry:9.0-onbuild +FROM sentry:9.1-onbuild diff --git a/sentry.conf.py b/sentry.conf.py index e4e0baf184..78d96b1f4e 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -29,6 +29,9 @@ # SENTRY_MAILGUN_API_KEY # SENTRY_SINGLE_ORGANIZATION # SENTRY_SECRET_KEY +# SLACK_CLIENT_ID +# SLACK_CLIENT_SECRET +# SLACK_VERIFICATION_TOKEN # GITHUB_APP_ID # GITHUB_API_SECRET # BITBUCKET_CONSUMER_KEY @@ -279,6 +282,15 @@ if SENTRY_OPTIONS['mail.enable-replies']: SENTRY_OPTIONS['mail.reply-hostname'] = env('SENTRY_SMTP_HOSTNAME') or '' +##################### +# SLACK INTEGRATION # +##################### +slack = env('SLACK_CLIENT_ID') and env('SLACK_CLIENT_SECRET') +if slack: + SENTRY_OPTIONS['slack.client-id'] = env('SLACK_CLIENT_ID') + SENTRY_OPTIONS['slack.client-secret'] = env('SLACK_CLIENT_SECRET') + SENTRY_OPTIONS['slack.verification-token'] = env('SLACK_VERIFICATION_TOKEN') or '' + # If this value ever becomes compromised, it's important to regenerate your # SENTRY_SECRET_KEY. Changing this value will result in all current sessions # being invalidated. @@ -303,4 +315,4 @@ if 'BITBUCKET_CONSUMER_KEY' in os.environ: BITBUCKET_CONSUMER_KEY = env('BITBUCKET_CONSUMER_KEY') - BITBUCKET_CONSUMER_SECRET = env('BITBUCKET_CONSUMER_SECRET') + BITBUCKET_CONSUMER_SECRET = env('BITBUCKET_CONSUMER_SECRET') \ No newline at end of file From d876a6c9027cfffef0b6943060ff9786d04831ff Mon Sep 17 00:00:00 2001 From: Omid Raha Date: Sat, 13 Apr 2019 02:32:40 +0430 Subject: [PATCH 002/417] Add Minimum Hardware Requirements (#165) --- README.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/README.md b/README.md index 343ad1f22e..6d1e94b62f 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,10 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke * Docker 1.10.0+ * Compose 1.6.0+ _(optional)_ + + ## Minimum Hardware Requirements: + + * You need at least 3GB Ram ## Up and Running From 41200b79a6fe94f7a994d5f0bcd9562201687a47 Mon Sep 17 00:00:00 2001 From: David Cramer Date: Mon, 15 Apr 2019 10:01:10 -0700 Subject: [PATCH 003/417] feat: Improve configuration to be less Docker-specific This removes auto-binding of various values (specified in config.yaml) when they're not actually configured. It ensures that these values can then be configured from the web UI as Sentry has intended. --- sentry.conf.py | 89 +++++++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 37 deletions(-) diff --git a/sentry.conf.py b/sentry.conf.py index 78d96b1f4e..8572e33cc1 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -40,6 +40,7 @@ import os import os.path +import six CONF_ROOT = os.path.dirname(__file__) @@ -250,46 +251,54 @@ # 'workers': 3, # the number of web workers } -############### -# Mail Server # -############### - -email = env('SENTRY_EMAIL_HOST') or (env('SMTP_PORT_25_TCP_ADDR') and 'smtp') -if email: - SENTRY_OPTIONS['mail.backend'] = 'smtp' - SENTRY_OPTIONS['mail.host'] = email - SENTRY_OPTIONS['mail.password'] = env('SENTRY_EMAIL_PASSWORD') or '' - SENTRY_OPTIONS['mail.username'] = env('SENTRY_EMAIL_USER') or '' - SENTRY_OPTIONS['mail.port'] = int(env('SENTRY_EMAIL_PORT') or 25) - SENTRY_OPTIONS['mail.use-tls'] = env('SENTRY_EMAIL_USE_TLS', False) -else: - SENTRY_OPTIONS['mail.backend'] = 'dummy' - -# The email address to send on behalf of -SENTRY_OPTIONS['mail.from'] = env('SENTRY_SERVER_EMAIL') or 'root@localhost' -# If you're using mailgun for inbound mail, set your API key and configure a -# route to forward to /api/hooks/mailgun/inbound/ -SENTRY_OPTIONS['mail.mailgun-api-key'] = env('SENTRY_MAILGUN_API_KEY') or '' +########## +# Docker # +########## -# If you specify a MAILGUN_API_KEY, you definitely want EMAIL_REPLIES -if SENTRY_OPTIONS['mail.mailgun-api-key']: - SENTRY_OPTIONS['mail.enable-replies'] = True -else: - SENTRY_OPTIONS['mail.enable-replies'] = env('SENTRY_ENABLE_EMAIL_REPLIES', False) +# Docker's environment configuration needs to happen +# prior to anything that might rely on these values to +# enable more "smart" configuration. + +ENV_CONFIG_MAPPING = { + 'SENTRY_EMAIL_PASSWORD': 'mail.password', + 'SENTRY_EMAIL_USER': 'mail.username', + 'SENTRY_EMAIL_PORT': ('mail.port', int), + 'SENTRY_EMAIL_USE_TLS': ('mail.use-tls', bool), + 'SENTRY_EMAIL_HOST': 'mail.host', + 'SENTRY_SERVER_EMAIL': 'mail.from', + 'SENTRY_ENABLE_EMAIL_REPLIES': 'mail.enable-replies', + 'SENTRY_SMTP_HOSTNAME': 'mail.reply-hostname', + + # If you're using mailgun for inbound mail, set your API key and configure a + # route to forward to /api/hooks/mailgun/inbound/ + 'SENTRY_MAILGUN_API_KEY': 'mail.mailgun-api-key', + + 'SLACK_CLIENT_ID': 'slack.client-id', + 'SLACK_CLIENT_SECRET': 'slack.client-secret', + 'SLACK_VERIFICATION_TOKEN': 'slack.verification-token', + + 'SECRET_KEY': 'system.secret-key', +} -if SENTRY_OPTIONS['mail.enable-replies']: - SENTRY_OPTIONS['mail.reply-hostname'] = env('SENTRY_SMTP_HOSTNAME') or '' -##################### -# SLACK INTEGRATION # -##################### -slack = env('SLACK_CLIENT_ID') and env('SLACK_CLIENT_SECRET') -if slack: - SENTRY_OPTIONS['slack.client-id'] = env('SLACK_CLIENT_ID') - SENTRY_OPTIONS['slack.client-secret'] = env('SLACK_CLIENT_SECRET') - SENTRY_OPTIONS['slack.verification-token'] = env('SLACK_VERIFICATION_TOKEN') or '' +def bind_env_config(config=SENTRY_OPTIONS, mapping=ENV_CONFIG_MAPPING): + """ + Automatically bind SENTRY_OPTIONS from a set of environment variables. + """ + for env_var, item in six.iteritems(mapping): + value = env(env_var) + if value is None: + continue + if isinstance(item, tuple): + opt_key, type_ = item + # only coerce the value if its not falsey (e.g. '') + if value: + value = type_(value) + else: + opt_key = item + config[opt_key] = value # If this value ever becomes compromised, it's important to regenerate your # SENTRY_SECRET_KEY. Changing this value will result in all current sessions @@ -306,7 +315,13 @@ print('!! Regenerate with `generate-secret-key`. !!') print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') -SENTRY_OPTIONS['system.secret-key'] = secret_key +# Grab the easy configuration first - these are all fixed +# key=value with no logic behind them +bind_env_config() + +# If you specify a MAILGUN_API_KEY, you definitely want EMAIL_REPLIES +if SENTRY_OPTIONS.get('mail.mailgun-api-key'): + SENTRY_OPTIONS.setdefault('mail.enable-replies', True) if 'GITHUB_APP_ID' in os.environ: GITHUB_EXTENDED_PERMISSIONS = ['repo'] @@ -315,4 +330,4 @@ if 'BITBUCKET_CONSUMER_KEY' in os.environ: BITBUCKET_CONSUMER_KEY = env('BITBUCKET_CONSUMER_KEY') - BITBUCKET_CONSUMER_SECRET = env('BITBUCKET_CONSUMER_SECRET') \ No newline at end of file + BITBUCKET_CONSUMER_SECRET = env('BITBUCKET_CONSUMER_SECRET') From d469fb8135e6f78f01e1c8c23ebd5ac02b1d93da Mon Sep 17 00:00:00 2001 From: David Cramer Date: Wed, 17 Apr 2019 12:29:53 -0700 Subject: [PATCH 004/417] feat: Add VSTS and GitHub integration config from env vars --- sentry.conf.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/sentry.conf.py b/sentry.conf.py index 8572e33cc1..f52638c9c6 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -29,11 +29,23 @@ # SENTRY_MAILGUN_API_KEY # SENTRY_SINGLE_ORGANIZATION # SENTRY_SECRET_KEY +# (slack integration) # SLACK_CLIENT_ID # SLACK_CLIENT_SECRET # SLACK_VERIFICATION_TOKEN +# (github plugin, sso) # GITHUB_APP_ID # GITHUB_API_SECRET +# (github integration) +# GITHUB_APP_ID +# GITHUB_CLIENT_ID +# GITHUB_CLIENT_SECRET +# GITHUB_WEBHOOK_SECRET +# GITHUB_PRIVATE_KEY +# (azure devops integration) +# VSTS_CLIENT_ID +# VSTS_CLIENT_SECRET +# (bitbucket plugin) # BITBUCKET_CONSUMER_KEY # BITBUCKET_CONSUMER_SECRET from sentry.conf.server import * # NOQA @@ -279,6 +291,15 @@ 'SLACK_CLIENT_SECRET': 'slack.client-secret', 'SLACK_VERIFICATION_TOKEN': 'slack.verification-token', + 'GITHUB_APP_ID': 'github-app.id', + 'GITHUB_CLIENT_ID': 'github-app.client-id', + 'GITHUB_CLIENT_SECRET': 'github-app.client-secret', + 'GITHUB_WEBHOOK_SECRET': 'github-app.webhook-secret', + 'GITHUB_PRIVATE_KEY': 'github-app.private-key', + + 'VSTS_CLIENT_ID': 'vsts.client-id', + 'VSTS_CLIENT_SECRET': 'vsts.client-secret', + 'SECRET_KEY': 'system.secret-key', } From 82aa12b88655ffa846304318eede2e8921f792f1 Mon Sep 17 00:00:00 2001 From: Matt Robenolt Date: Fri, 19 Apr 2019 12:02:55 -0700 Subject: [PATCH 005/417] I think fixed stuff up --- sentry.conf.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/sentry.conf.py b/sentry.conf.py index f52638c9c6..0a0fde1888 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -49,6 +49,7 @@ # BITBUCKET_CONSUMER_KEY # BITBUCKET_CONSUMER_SECRET from sentry.conf.server import * # NOQA +from sentry.utils.types import Bool, Int import os import os.path @@ -276,12 +277,13 @@ ENV_CONFIG_MAPPING = { 'SENTRY_EMAIL_PASSWORD': 'mail.password', 'SENTRY_EMAIL_USER': 'mail.username', - 'SENTRY_EMAIL_PORT': ('mail.port', int), - 'SENTRY_EMAIL_USE_TLS': ('mail.use-tls', bool), + 'SENTRY_EMAIL_PORT': ('mail.port', Int), + 'SENTRY_EMAIL_USE_TLS': ('mail.use-tls', Bool), 'SENTRY_EMAIL_HOST': 'mail.host', 'SENTRY_SERVER_EMAIL': 'mail.from', 'SENTRY_ENABLE_EMAIL_REPLIES': 'mail.enable-replies', 'SENTRY_SMTP_HOSTNAME': 'mail.reply-hostname', + 'SENTRY_SECRET_KEY': 'system.secret-key', # If you're using mailgun for inbound mail, set your API key and configure a # route to forward to /api/hooks/mailgun/inbound/ @@ -299,8 +301,6 @@ 'VSTS_CLIENT_ID': 'vsts.client-id', 'VSTS_CLIENT_SECRET': 'vsts.client-secret', - - 'SECRET_KEY': 'system.secret-key', } @@ -309,14 +309,13 @@ def bind_env_config(config=SENTRY_OPTIONS, mapping=ENV_CONFIG_MAPPING): Automatically bind SENTRY_OPTIONS from a set of environment variables. """ for env_var, item in six.iteritems(mapping): - value = env(env_var) - if value is None: + try: + value = os.environ[env_var] + except KeyError: continue if isinstance(item, tuple): opt_key, type_ = item - # only coerce the value if its not falsey (e.g. '') - if value: - value = type_(value) + value = type_(value) else: opt_key = item config[opt_key] = value From 3eb2caf4cb2c7fa4e213039baa96b176e690509f Mon Sep 17 00:00:00 2001 From: Matt Robenolt Date: Mon, 6 May 2019 14:13:18 -0700 Subject: [PATCH 006/417] Fixes to default config Fixes #177, #180 --- sentry.conf.py | 58 ++++++++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 26 deletions(-) diff --git a/sentry.conf.py b/sentry.conf.py index 0a0fde1888..1fb457acb5 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -30,21 +30,21 @@ # SENTRY_SINGLE_ORGANIZATION # SENTRY_SECRET_KEY # (slack integration) -# SLACK_CLIENT_ID -# SLACK_CLIENT_SECRET -# SLACK_VERIFICATION_TOKEN +# SENTRY_SLACK_CLIENT_ID +# SENTRY_SLACK_CLIENT_SECRET +# SENTRY_SLACK_VERIFICATION_TOKEN # (github plugin, sso) # GITHUB_APP_ID # GITHUB_API_SECRET # (github integration) -# GITHUB_APP_ID -# GITHUB_CLIENT_ID -# GITHUB_CLIENT_SECRET -# GITHUB_WEBHOOK_SECRET -# GITHUB_PRIVATE_KEY +# SENTRY_GITHUB_APP_ID +# SENTRY_GITHUB_APP_CLIENT_ID +# SENTRY_GITHUB_APP_CLIENT_SECRET +# SENTRY_GITHUB_APP_WEBHOOK_SECRET +# SENTRY_GITHUB_APP_PRIVATE_KEY # (azure devops integration) -# VSTS_CLIENT_ID -# VSTS_CLIENT_SECRET +# SENTRY_VSTS_CLIENT_ID +# SENTRY_VSTS_CLIENT_SECRET # (bitbucket plugin) # BITBUCKET_CONSUMER_KEY # BITBUCKET_CONSUMER_SECRET @@ -289,18 +289,18 @@ # route to forward to /api/hooks/mailgun/inbound/ 'SENTRY_MAILGUN_API_KEY': 'mail.mailgun-api-key', - 'SLACK_CLIENT_ID': 'slack.client-id', - 'SLACK_CLIENT_SECRET': 'slack.client-secret', - 'SLACK_VERIFICATION_TOKEN': 'slack.verification-token', + 'SENTRY_SLACK_CLIENT_ID': 'slack.client-id', + 'SENTRY_SLACK_CLIENT_SECRET': 'slack.client-secret', + 'SENTRY_SLACK_VERIFICATION_TOKEN': 'slack.verification-token', - 'GITHUB_APP_ID': 'github-app.id', - 'GITHUB_CLIENT_ID': 'github-app.client-id', - 'GITHUB_CLIENT_SECRET': 'github-app.client-secret', - 'GITHUB_WEBHOOK_SECRET': 'github-app.webhook-secret', - 'GITHUB_PRIVATE_KEY': 'github-app.private-key', + 'SENTRY_GITHUB_APP_ID': ('github-app.id', Int), + 'SENTRY_GITHUB_APP_CLIENT_ID': 'github-app.client-id', + 'SENTRY_GITHUB_APP_CLIENT_SECRET': 'github-app.client-secret', + 'SENTRY_GITHUB_APP_WEBHOOK_SECRET': 'github-app.webhook-secret', + 'SENTRY_GITHUB_APP_PRIVATE_KEY': 'github-app.private-key', - 'VSTS_CLIENT_ID': 'vsts.client-id', - 'VSTS_CLIENT_SECRET': 'vsts.client-secret', + 'SENTRY_VSTS_CLIENT_ID': 'vsts.client-id', + 'SENTRY_VSTS_CLIENT_SECRET': 'vsts.client-secret', } @@ -309,16 +309,22 @@ def bind_env_config(config=SENTRY_OPTIONS, mapping=ENV_CONFIG_MAPPING): Automatically bind SENTRY_OPTIONS from a set of environment variables. """ for env_var, item in six.iteritems(mapping): - try: - value = os.environ[env_var] - except KeyError: + # HACK: we need to check both in `os.environ` and `env._cache`. + # This is very much an implementation detail leaking out + # due to assumptions about how `env` would be used previously. + # `env` will pop values out of `os.environ` when they are seen, + # so checking against `os.environ` only means it's likely + # they won't exist if `env()` has been called on the variable + # before at any point. So we're choosing to check both, but this + # behavior is different since we're trying to only conditionally + # apply variables, instead of setting them always. + if env_var not in os.environ and env_var not in env._cache: continue if isinstance(item, tuple): opt_key, type_ = item - value = type_(value) else: - opt_key = item - config[opt_key] = value + opt_key, type_ = item, None + config[opt_key] = env(env_var, type=type_) # If this value ever becomes compromised, it's important to regenerate your # SENTRY_SECRET_KEY. Changing this value will result in all current sessions From 89ec146ee137d3fd47df6154d68b8e92e9892467 Mon Sep 17 00:00:00 2001 From: Matt Robenolt Date: Mon, 6 May 2019 14:16:30 -0700 Subject: [PATCH 007/417] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6d1e94b62f..0e4bae8a82 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ Updating Sentry using Compose is relatively simple. Just use the following steps Use the following steps after updating this repository or your Dockerfile: ```sh -docker-compose build # Build the services again after updating +docker-compose build --pull # Build the services again after updating, and make sure we're up to date on patch version docker-compose run --rm web upgrade # Run new migrations docker-compose up -d # Recreate the services ``` From 259bc29e78b7bdb3173da004458a378e5e3fa8b4 Mon Sep 17 00:00:00 2001 From: Matt Robenolt Date: Mon, 6 May 2019 14:17:59 -0700 Subject: [PATCH 008/417] Add `--pull` into default `make build` target --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 57d2721e19..20a75218e0 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ NO_COLOR=\033[0m build: @echo "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" - @docker build --rm -t $(REPOSITORY):$(TAG) . + @docker build --pull --rm -t $(REPOSITORY):$(TAG) . $(REPOSITORY)_$(TAG).tar: build @echo "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" From ba82e9ffc4ae0fdef3bfd8cc8d571f9e720e89d7 Mon Sep 17 00:00:00 2001 From: Jens Willmer Date: Tue, 7 May 2019 09:48:38 +0200 Subject: [PATCH 009/417] Fix: 'mail.enable-replies' type Excpetion: 'mail.enable-replies': got , expected boolean --- sentry.conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry.conf.py b/sentry.conf.py index 1fb457acb5..9b199c5b95 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -281,7 +281,7 @@ 'SENTRY_EMAIL_USE_TLS': ('mail.use-tls', Bool), 'SENTRY_EMAIL_HOST': 'mail.host', 'SENTRY_SERVER_EMAIL': 'mail.from', - 'SENTRY_ENABLE_EMAIL_REPLIES': 'mail.enable-replies', + 'SENTRY_ENABLE_EMAIL_REPLIES': ('mail.enable-replies', Bool), 'SENTRY_SMTP_HOSTNAME': 'mail.reply-hostname', 'SENTRY_SECRET_KEY': 'system.secret-key', From 2589cbef43659151e70fd3d20eb8b34d7f1f574f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 10 Jun 2019 22:21:49 +0300 Subject: [PATCH 010/417] meta(license): Add Apache 2.0 license to the project Fixes #117, supersedes #120. --- LICENSE | 203 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 203 insertions(+) create mode 100644 LICENSE diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000000..9f039c21e6 --- /dev/null +++ b/LICENSE @@ -0,0 +1,203 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2017 Functional Software, Inc. + Copyright 2014 Dropbox, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. From a093daac1c51cec9bc5c227d3c45b566470262df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Yi=C4=9Fit=20Kaya?= Date: Mon, 10 Jun 2019 23:50:39 +0300 Subject: [PATCH 011/417] meta(gitignore): Add .vscode/tags to .gitignore (#190) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e10a1538ba..f72d456238 100644 --- a/.gitignore +++ b/.gitignore @@ -73,3 +73,4 @@ docker-compose.override.yml *.tar data/ +.vscode/tags \ No newline at end of file From 9f8c89a5f7d7718c6f3c1e48cee0f32d77340810 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Burak=20Yi=C4=9Fit=20Kaya?= Date: Tue, 11 Jun 2019 23:21:39 +0300 Subject: [PATCH 012/417] fix(email): Add `SENTRY_EMAIL_LIST_NAMESPACE` env setting (#192) Port of getsentry/docker-sentry#170 --- sentry.conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry.conf.py b/sentry.conf.py index 9b199c5b95..4f9311a7a1 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -24,6 +24,7 @@ # SENTRY_EMAIL_USER # SENTRY_EMAIL_PASSWORD # SENTRY_EMAIL_USE_TLS +# SENTRY_EMAIL_LIST_NAMESPACE # SENTRY_ENABLE_EMAIL_REPLIES # SENTRY_SMTP_HOSTNAME # SENTRY_MAILGUN_API_KEY @@ -282,6 +283,7 @@ 'SENTRY_EMAIL_HOST': 'mail.host', 'SENTRY_SERVER_EMAIL': 'mail.from', 'SENTRY_ENABLE_EMAIL_REPLIES': ('mail.enable-replies', Bool), + 'SENTRY_EMAIL_LIST_NAMESPACE': 'mail.list-namespace', 'SENTRY_SMTP_HOSTNAME': 'mail.reply-hostname', 'SENTRY_SECRET_KEY': 'system.secret-key', From ceed39c1d2088b131ad48a7c69b6ce57a750bda7 Mon Sep 17 00:00:00 2001 From: Kashyap Date: Wed, 12 Jun 2019 11:37:53 +0530 Subject: [PATCH 013/417] Fix possible typo in gitignore MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove a match-all glob pattern. It looks like some non-git tools read the `*,cover` as `*` and `cover` separately and basically ignore all files. Even though this is not really a problem with the repo itself, I think the change is still valid, considering `*,cover` looks like an invalid rule anyway. _____ Background: (__This part is vim-specific__) For a while now, my Neosnippet configuration has been going for a toss when working on our fork based off of this repo. I see a message similar to the one reported [here](https://github.com/Shougo/neocomplcache.vim/issues/468), but my vimrc configuration is as per the documentation available in the plugin docs. I finally spent enough time to debug this today, and it turns out this issue was happening *only* in this repo and nowhere else. So I went through the files that might've to do anything with folder-specific configuration (think dotenv, editorconfig etc.), and turned out `.gitignore` was the only such file; I use [this plugin](https://github.com/vim-scripts/gitignore) to load the gitignore rules into the `wildmenu` completion and the `*,` part basically (from what I understand) removes most of the load path configuration from vim 😄. I'll probably have to go make a change in that plugin as well though. --- .gitignore | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index f72d456238..d31ea68355 100644 --- a/.gitignore +++ b/.gitignore @@ -42,7 +42,7 @@ htmlcov/ .cache nosetests.xml coverage.xml -*,cover +*.cover .hypothesis/ # Translations @@ -73,4 +73,4 @@ docker-compose.override.yml *.tar data/ -.vscode/tags \ No newline at end of file +.vscode/tags From 7e48342757fdf64ca230f97330166017672728e8 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 13 Jun 2019 23:26:40 +0300 Subject: [PATCH 014/417] meta(readme): Update minimum docker-compose version (#194) Fixes #191 . --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e4bae8a82..25182987ee 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements * Docker 1.10.0+ - * Compose 1.6.0+ _(optional)_ + * Compose 1.17.0+ _(optional)_ ## Minimum Hardware Requirements: From 4e845995425f5a73ec11cd3d3f3e75dcba33d233 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 18 Jun 2019 22:26:47 +0300 Subject: [PATCH 015/417] fix(config): Remove autocommit option from DB config (#195) Django 1.7 already defaults to `True` for this and in Django 1.8, which is what the latest Sentry uses, the option is removed and causes a DB connection error so dropping it to fix git builds. --- sentry.conf.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sentry.conf.py b/sentry.conf.py index 4f9311a7a1..3a8dde422a 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -83,9 +83,6 @@ env('SENTRY_POSTGRES_PORT') or '' ), - 'OPTIONS': { - 'autocommit': True, - }, }, } From ae39a61d4d8a8ec8b9fd7af9c1d64e80c9bdd640 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 20 Jun 2019 01:10:22 +0300 Subject: [PATCH 016/417] fix(license): Fix license attribution terms (#197) --- LICENSE | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/LICENSE b/LICENSE index 9f039c21e6..c085307bd4 100644 --- a/LICENSE +++ b/LICENSE @@ -187,8 +187,7 @@ same "printed page" as the copyright notice for easier identification within third-party archives. - Copyright 2017 Functional Software, Inc. - Copyright 2014 Dropbox, Inc. + Copyright 2016 Functional Software, Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From c42fc264df5f34944ebd8e4e4967d99dfaaf5d40 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 1 Jul 2019 22:48:12 +0300 Subject: [PATCH 017/417] feat(install): Add fully automated install script This adds a fully automated `install.sh` that does the following: - Checks minimum Docker and `docker-compose` versions - Checks minimum RAM available to Docker containers - Removes potential user errors in install steps --- install.sh | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100755 install.sh diff --git a/install.sh b/install.sh new file mode 100755 index 0000000000..4d73730c42 --- /dev/null +++ b/install.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +set -e + +MIN_DOCKER_VERSION='1.10.0' +MIN_COMPOSE_VERSION='1.17.0' +MIN_RAM=3072 # MB +ENV_FILE='.env' + +DID_CLEAN_UP=0 +# the cleanup function will be the exit point +cleanup () { + if [ "$DID_CLEAN_UP" -eq 1 ]; then + return 0; + fi + echo "Cleaning up..." + docker-compose down &> /dev/null + DID_CLEAN_UP=1 +} +trap cleanup ERR INT TERM + +echo "Checking minimum requirements..." + +DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') +COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') +RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); + +# Function below is inspired by https://stackoverflow.com/a/29394504/90297 +function ver { printf "%03d%03d%03d%03d" $(echo "$1" | sed 's/^0*\([0-9]\+\)\.0*\([0-9]\+\)\.0*\([0-9]\+\).*/\1 \2 \3/' | head -n 3 ); } + +if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then + echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" + exit -1 +fi + +if [ $(ver $COMPOSE_VERSION) -lt $(ver $MIN_COMPOSE_VERSION) ]; then + echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" + exit -1 +fi + +if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then + echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" + exit -1 +fi + +echo "" +echo "Creating volumes for persistent storage..." +echo "Created $(docker volume create --name=sentry-data)." +echo "Created $(docker volume create --name=sentry-postgres)." +echo "" + +if [ -f "$ENV_FILE" ]; then + echo "$ENV_FILE already exists, skipped creation." +else + echo "Creating $ENV_FILE..." + cp -n .env.example "$ENV_FILE" +fi + +echo "" +echo "Building and tagging Docker images..." +echo "" +docker-compose build +echo "" +echo "Docker images built." + +echo "" +echo "Generating secret key..." +# This is to escape the secret key to be used in sed below +SECRET_KEY=$(docker-compose run --rm web config generate-secret-key 2> /dev/null | tail -n1 | sed -e 's/[\/&]/\\&/g') +sed -i -e 's/^SENTRY_SECRET_KEY=.*$/SENTRY_SECRET_KEY='"$SECRET_KEY"'/' $ENV_FILE +echo "Secret key written to $ENV_FILE" + +echo "" +echo "Setting up database..." +if [ $CI ]; then + docker-compose run --rm web upgrade --noinput + echo "" + echo "Did not prompt for user creation due to non-interactive shell." + echo "Run the following command to create one yourself (recommended):" + echo "" + echo " docker-compose run --rm web createuser" + echo "" +else + docker-compose run --rm web upgrade +fi + +cleanup + +echo "" +echo "----------------" +echo "You're all done! Run the following command get Sentry running:" +echo "" +echo " docker-compose up -d" +echo "" \ No newline at end of file From f015462a51ce999c3019c69cd689fe73918b79f6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 1 Jul 2019 22:50:21 +0300 Subject: [PATCH 018/417] ci(travis): Add e2e smoke testing with Travis CI --- .travis.yml | 9 +++++++++ test.sh | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+) create mode 100644 .travis.yml create mode 100755 test.sh diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..8ad3f038ee --- /dev/null +++ b/.travis.yml @@ -0,0 +1,9 @@ +language: bash +services: docker + +script: + - ./install.sh + - docker-compose run --rm web createuser --superuser --email test@sentry.io --password test123TEST + - docker-compose up -d + - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - ./test.sh diff --git a/test.sh b/test.sh new file mode 100755 index 0000000000..0465ae1b53 --- /dev/null +++ b/test.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +set -e + +TEST_USER='test@sentry.io' +TEST_PASS='test123TEST' +COOKIE_FILE=$(mktemp) +declare -a TEST_STRINGS=( + '"isAuthenticated":true,' + '"username":"test@sentry.io",' + '"isSuperuser":true,' +) + +INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null http://localhost:9000 -w %{url_effective}) +if [ "$INITIAL_AUTH_REDIRECT" != "http://localhost:9000/auth/login/sentry/" ]; then + echo "Initial /auth/login/ redirect failed, exiting..." + echo "$INITIAL_AUTH_REDIRECT" + exit -1 +fi + +CSRF_TOKEN=$(curl http://localhost:9000 -sL -c "$COOKIE_FILE" | awk -F "'" ' + /csrfmiddlewaretoken/ { + print $4 "=" $6; + exit; + }') +LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" http://localhost:9000/auth/login/ -H 'Referer: http://localhost/auth/login/' -b "$COOKIE_FILE" -c "$COOKIE_FILE") + +TEST_RESULT=0 +for i in "${TEST_STRINGS[@]}" +do + echo "Testing '$i'..." + echo "$LOGIN_RESPONSE" | grep "$i" >& /dev/null + echo "Pass." +done From 8b5b724d1ee1d88fdae8efefc7c532348a0adb8a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 1 Jul 2019 23:54:11 +0300 Subject: [PATCH 019/417] chore(dockerignore): Ignore new CI/install files in Docker --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index 6fd1d2510b..e45d0676ff 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,3 +7,6 @@ README.md *.tar docker-compose.yml data/ +.travis.yml +install.sh +test.sh From 792e2cc961a488048447658c5278f26890407721 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 2 Jul 2019 23:58:49 +0300 Subject: [PATCH 020/417] docs(readme): Update readme for the install script (#209) Follow up to #207. --- README.md | 52 ++++++++++++++++++++++++++-------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 25182987ee..71d6f2c315 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,35 @@ -# Sentry On-Premise +# Sentry On-Premise [![Build Status][build-status-image]][build-status-url] Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). ## Requirements * Docker 1.10.0+ - * Compose 1.17.0+ _(optional)_ - - ## Minimum Hardware Requirements: - - * You need at least 3GB Ram - -## Up and Running - -Assuming you've just cloned this repository, the following steps -will get you up and running in no time! - -There may need to be modifications to the included `docker-compose.yml` file to accommodate your needs or your environment. These instructions are a guideline for what you should generally do. - -1. `docker volume create --name=sentry-data && docker volume create --name=sentry-postgres` - Make our local database and sentry volumes - Docker volumes have to be created manually, as they are declared as external to be more durable. -2. `cp -n .env.example .env` - create env config file -3. `docker-compose build` - Build and tag the Docker services -4. `docker-compose run --rm web config generate-secret-key` - Generate a secret key. - Add it to `.env` as `SENTRY_SECRET_KEY`. -5. `docker-compose run --rm web upgrade` - Build the database. - Use the interactive prompts to create a user account. -6. `docker-compose up -d` - Lift all services (detached/background mode). -7. Access your instance at `localhost:9000`! + * Compose 1.17.0+ + +## Minimum Hardware Requirements: + + * You need at least 3GB RAM + +## Setup + +To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. + +There may need to be modifications to the included `docker-compose.yml` file to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script. + +The recommended way to customize your configuration is using the files below, in that order: + + * `config.yml` + * `sentry.conf.py` + * `.env` w/ environment variables + +If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! ## Securing Sentry with SSL/TLS If you'd like to protect your Sentry install with SSL/TLS, there are fantastic SSL/TLS proxies like [HAProxy](http://www.haproxy.org/) -and [Nginx](http://nginx.org/). +and [Nginx](http://nginx.org/). You'll likely to add this service to your `docker-compose.yml` file. ## Updating Sentry @@ -49,6 +45,10 @@ docker-compose up -d # Recreate the services ## Resources * [Documentation](https://docs.sentry.io/server/installation/docker/) - * [Bug Tracker](https://github.com/getsentry/onpremise) + * [Bug Tracker](https://github.com/getsentry/onpremise/issues) * [Forums](https://forum.sentry.io/c/on-premise) * [IRC](irc://chat.freenode.net/sentry) (chat.freenode.net, #sentry) + + +[build-status-image]: https://api.travis-ci.com/getsentry/onpremise.svg?branch=master +[build-status-url]: https://travis-ci.com/getsentry/onpremise \ No newline at end of file From 9177155c7906498754057aa76c144c6bfb8ac4ba Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 8 Jul 2019 18:56:51 +0300 Subject: [PATCH 021/417] feat(nightlies): Add ability to build from nightlies (#211) This patch adds the optional IMAGE env variable to set the base image and adds nightly builds to Travis CI. --- .env.example | 3 ++- .travis.yml | 4 ++++ Dockerfile | 3 ++- docker-compose.yml | 5 ++++- test.sh | 8 ++++---- 5 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.env.example b/.env.example index c560055d33..471e7d47f1 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,4 @@ +IMAGE=sentry:9.1 # Run `docker-compose run web config generate-secret-key` # to get the SENTRY_SECRET_KEY value. -SENTRY_SECRET_KEY= +SENTRY_SECRET_KEY= \ No newline at end of file diff --git a/.travis.yml b/.travis.yml index 8ad3f038ee..8d53fb70d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,10 @@ language: bash services: docker +env: + - IMAGE=sentry:9.1 + - IMAGE=getsentry/sentry:git + script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@sentry.io --password test123TEST diff --git a/Dockerfile b/Dockerfile index e5df0ded33..e42391e009 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1,2 @@ -FROM sentry:9.1-onbuild +ARG IMAGE +FROM ${IMAGE}-onbuild diff --git a/docker-compose.yml b/docker-compose.yml index 68348e20ee..bcc9cb4ce9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,10 @@ version: '3.4' x-defaults: &defaults restart: unless-stopped - build: . + build: + context: . + args: + IMAGE: ${IMAGE} depends_on: - redis - postgres diff --git a/test.sh b/test.sh index 0465ae1b53..688787ac89 100755 --- a/test.sh +++ b/test.sh @@ -5,9 +5,9 @@ TEST_USER='test@sentry.io' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) declare -a TEST_STRINGS=( - '"isAuthenticated":true,' - '"username":"test@sentry.io",' - '"isSuperuser":true,' + '"isAuthenticated":true' + '"username":"test@sentry.io"' + '"isSuperuser":true' ) INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null http://localhost:9000 -w %{url_effective}) @@ -28,6 +28,6 @@ TEST_RESULT=0 for i in "${TEST_STRINGS[@]}" do echo "Testing '$i'..." - echo "$LOGIN_RESPONSE" | grep "$i" >& /dev/null + echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done From af2d805f32860e42a2398a29d1037c56e67bcabe Mon Sep 17 00:00:00 2001 From: Jamin Collins Date: Mon, 15 Jul 2019 18:08:17 -0600 Subject: [PATCH 022/417] fix(Makefile): -e needed for color codes (#215) closes #214 Signed-off-by: Jamin W. Collins --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 20a75218e0..0ab7e4362e 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,15 @@ OK_COLOR=\033[32;01m NO_COLOR=\033[0m build: - @echo "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" + @echo -e "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" @docker build --pull --rm -t $(REPOSITORY):$(TAG) . $(REPOSITORY)_$(TAG).tar: build - @echo "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" + @echo -e "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" @docker save $(REPOSITORY):$(TAG) > $@ push: build - @echo "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)" + @echo -e "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)" @docker push $(REPOSITORY):$(TAG) all: build push From ce1d25983485463370d26c6b55b8178d76ce1d94 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 17 Jul 2019 22:13:23 +0300 Subject: [PATCH 023/417] fix: Update minimum Docker version (#217) Required after #211. Fixes #213. --- README.md | 4 ++-- install.sh | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 71d6f2c315..b82d16978e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements - * Docker 1.10.0+ + * Docker 17.05.0+ * Compose 1.17.0+ ## Minimum Hardware Requirements: @@ -51,4 +51,4 @@ docker-compose up -d # Recreate the services [build-status-image]: https://api.travis-ci.com/getsentry/onpremise.svg?branch=master -[build-status-url]: https://travis-ci.com/getsentry/onpremise \ No newline at end of file +[build-status-url]: https://travis-ci.com/getsentry/onpremise diff --git a/install.sh b/install.sh index 4d73730c42..c9fd5642a2 100755 --- a/install.sh +++ b/install.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -MIN_DOCKER_VERSION='1.10.0' +MIN_DOCKER_VERSION='17.05.0' MIN_COMPOSE_VERSION='1.17.0' MIN_RAM=3072 # MB ENV_FILE='.env' @@ -90,4 +90,4 @@ echo "----------------" echo "You're all done! Run the following command get Sentry running:" echo "" echo " docker-compose up -d" -echo "" \ No newline at end of file +echo "" From 0fd037985961048af664951d13816be1d56f8d75 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 17 Jul 2019 22:44:42 +0300 Subject: [PATCH 024/417] fix(Makefile): Fix invalid reference format error (#218) Required after #211. Fixes #216. Also fixes printing of `-e` after #215. --- Makefile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0ab7e4362e..9a0eaab62b 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,15 @@ OK_COLOR=\033[32;01m NO_COLOR=\033[0m build: - @echo -e "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" - @docker build --pull --rm -t $(REPOSITORY):$(TAG) . + @printf "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" + @docker build --pull --rm -t $(REPOSITORY):$(TAG) . --build-arg IMAGE=sentry:9.1 $(REPOSITORY)_$(TAG).tar: build - @echo -e "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" + @printf "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" @docker save $(REPOSITORY):$(TAG) > $@ push: build - @echo -e "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)" + @printf "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)" @docker push $(REPOSITORY):$(TAG) all: build push From 424cc20e8446c016d9e0c56d950c1534ef3a20e0 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 18 Jul 2019 11:42:33 +0300 Subject: [PATCH 025/417] fix(Makefile): Bring back the new lines from messages (#219) Follow up to #218. --- Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 9a0eaab62b..ac756825df 100644 --- a/Makefile +++ b/Makefile @@ -5,15 +5,15 @@ OK_COLOR=\033[32;01m NO_COLOR=\033[0m build: - @printf "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" + @printf "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)\n" @docker build --pull --rm -t $(REPOSITORY):$(TAG) . --build-arg IMAGE=sentry:9.1 $(REPOSITORY)_$(TAG).tar: build - @printf "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" + @printf "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@\n" @docker save $(REPOSITORY):$(TAG) > $@ push: build - @printf "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)" + @printf "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)\n" @docker push $(REPOSITORY):$(TAG) all: build push From af07ad9c55183b4341c4ad49fe1009cad9dd4135 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 24 Jul 2019 11:28:48 +0300 Subject: [PATCH 026/417] fix(Dockerfile): Fix default IMAGE missing (#223) --- .env.example | 3 +-- .travis.yml | 2 +- Dockerfile | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/.env.example b/.env.example index 471e7d47f1..c560055d33 100644 --- a/.env.example +++ b/.env.example @@ -1,4 +1,3 @@ -IMAGE=sentry:9.1 # Run `docker-compose run web config generate-secret-key` # to get the SENTRY_SECRET_KEY value. -SENTRY_SECRET_KEY= \ No newline at end of file +SENTRY_SECRET_KEY= diff --git a/.travis.yml b/.travis.yml index 8d53fb70d2..b079f4d10b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: bash services: docker env: - - IMAGE=sentry:9.1 + - IMAGE=sentry:9.1.2 - IMAGE=getsentry/sentry:git script: diff --git a/Dockerfile b/Dockerfile index e42391e009..bbd3044aea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,2 @@ -ARG IMAGE +ARG IMAGE=sentry:9.1.2 FROM ${IMAGE}-onbuild From 13b510fc14a0e74447a5533682f6b181872885a1 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 26 Jul 2019 09:50:46 +0300 Subject: [PATCH 027/417] fix(install): Fix invalid reference format again (#231) Fix #230. --- .travis.yml | 4 ++-- Dockerfile | 4 ++-- docker-compose.yml | 2 +- install.sh | 8 ++++++++ 4 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index b079f4d10b..243de20f87 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,8 +2,8 @@ language: bash services: docker env: - - IMAGE=sentry:9.1.2 - - IMAGE=getsentry/sentry:git + - SENTRY_IMAGE=sentry:9.1.2 + - SENTRY_IMAGE=getsentry/sentry:git script: - ./install.sh diff --git a/Dockerfile b/Dockerfile index bbd3044aea..50a59ea227 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1,2 @@ -ARG IMAGE=sentry:9.1.2 -FROM ${IMAGE}-onbuild +ARG SENTRY_IMAGE +FROM ${SENTRY_IMAGE}-onbuild diff --git a/docker-compose.yml b/docker-compose.yml index bcc9cb4ce9..20d688cef4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ x-defaults: &defaults build: context: . args: - IMAGE: ${IMAGE} + SENTRY_IMAGE: ${SENTRY_IMAGE} depends_on: - redis - postgres diff --git a/install.sh b/install.sh index c9fd5642a2..d3af603a66 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -e +LATEST_STABLE_SENTRY_IMAGE='sentry:9.1.2' + MIN_DOCKER_VERSION='17.05.0' MIN_COMPOSE_VERSION='1.17.0' MIN_RAM=3072 # MB @@ -55,6 +57,12 @@ else cp -n .env.example "$ENV_FILE" fi +if [ -z $SENTRY_IMAGE ]; then + echo "" + echo "\$SENTRY_IMAGE not set, using latest stable: $LATEST_STABLE_SENTRY_IMAGE"; + export SENTRY_IMAGE=$LATEST_STABLE_SENTRY_IMAGE +fi + echo "" echo "Building and tagging Docker images..." echo "" From 263acda90b4bf3368066b07240eb03d11500c509 Mon Sep 17 00:00:00 2001 From: Ashok Bommisetti Date: Mon, 29 Jul 2019 16:55:35 +0200 Subject: [PATCH 028/417] Dockerfile & Makefile argument name mismatch (#232) Dockerfile expects the argument to be `SENTRY_IMAGE`. So, modifying Makefile to enable `make build` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ac756825df..0ab11e2875 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ NO_COLOR=\033[0m build: @printf "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)\n" - @docker build --pull --rm -t $(REPOSITORY):$(TAG) . --build-arg IMAGE=sentry:9.1 + @docker build --pull --rm -t $(REPOSITORY):$(TAG) . --build-arg SENTRY_IMAGE=sentry:9.1 $(REPOSITORY)_$(TAG).tar: build @printf "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@\n" From 4fade2fb1f08eae008c435e4c0c5ef03acc9d9a7 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 13 Aug 2019 02:38:12 +0300 Subject: [PATCH 029/417] fix(uwsgi): Fix broken uWSGI config (#237) This patch introduces the following fixes to uWSGI config which was not set up correctly to be used publicly (not behind a router, load balancer, proxy like nginx etc.): - Use `http` option for a front-proxy - Set protocol back to `uwsgi` due to above - Prevent forced socket binding to `uwsgi` due to protocol change above (https://git.io/fj7Lw) - Turn on "keep-alive" support Should fix getsentry/sentry-cli#40. --- sentry.conf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/sentry.conf.py b/sentry.conf.py index 3a8dde422a..d02e0d8343 100644 --- a/sentry.conf.py +++ b/sentry.conf.py @@ -259,11 +259,16 @@ SENTRY_WEB_HOST = '0.0.0.0' SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { + 'http': '%s:%s' % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), + 'protocol': 'uwsgi', + # This is need to prevent https://git.io/fj7Lw + 'uwsgi-socket': None, + 'http-keepalive': True, + 'memory-report': False, # 'workers': 3, # the number of web workers } - ########## # Docker # ########## From ea78661e360aafd436a8248e14365fe0db5e179d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 14 Oct 2019 21:31:38 +0300 Subject: [PATCH 030/417] build(travis): Use :latest tag for getsentry/sentry (#252) --- .travis.yml | 2 +- Dockerfile | 3 +-- docker-compose.yml | 2 -- install.sh | 8 -------- 4 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 243de20f87..07a67bc7e8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,7 +3,7 @@ services: docker env: - SENTRY_IMAGE=sentry:9.1.2 - - SENTRY_IMAGE=getsentry/sentry:git + - SENTRY_IMAGE=getsentry/sentry:latest script: - ./install.sh diff --git a/Dockerfile b/Dockerfile index 50a59ea227..0a3dae537c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,2 +1 @@ -ARG SENTRY_IMAGE -FROM ${SENTRY_IMAGE}-onbuild +FROM ${SENTRY_IMAGE:-sentry:9.1.2}-onbuild diff --git a/docker-compose.yml b/docker-compose.yml index 20d688cef4..18cee6a316 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,6 @@ x-defaults: &defaults restart: unless-stopped build: context: . - args: - SENTRY_IMAGE: ${SENTRY_IMAGE} depends_on: - redis - postgres diff --git a/install.sh b/install.sh index d3af603a66..c9fd5642a2 100755 --- a/install.sh +++ b/install.sh @@ -1,8 +1,6 @@ #!/usr/bin/env bash set -e -LATEST_STABLE_SENTRY_IMAGE='sentry:9.1.2' - MIN_DOCKER_VERSION='17.05.0' MIN_COMPOSE_VERSION='1.17.0' MIN_RAM=3072 # MB @@ -57,12 +55,6 @@ else cp -n .env.example "$ENV_FILE" fi -if [ -z $SENTRY_IMAGE ]; then - echo "" - echo "\$SENTRY_IMAGE not set, using latest stable: $LATEST_STABLE_SENTRY_IMAGE"; - export SENTRY_IMAGE=$LATEST_STABLE_SENTRY_IMAGE -fi - echo "" echo "Building and tagging Docker images..." echo "" From ac33e08b47025d20222c3d57ed82cb24bd3bdd16 Mon Sep 17 00:00:00 2001 From: Ben Johnson Date: Wed, 23 Oct 2019 18:10:14 -0400 Subject: [PATCH 031/417] Fix typo (#263) There's a word missing here, although, I'm not sure if it should be "want" or "need"; either one seems to work. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b82d16978e..bac6c436a7 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ If you have any issues or questions, our [Community Forum](https://forum.sentry. If you'd like to protect your Sentry install with SSL/TLS, there are fantastic SSL/TLS proxies like [HAProxy](http://www.haproxy.org/) -and [Nginx](http://nginx.org/). You'll likely to add this service to your `docker-compose.yml` file. +and [Nginx](http://nginx.org/). You'll likely want to add this service to your `docker-compose.yml` file. ## Updating Sentry From fc24bd4b82807d845342d7a494414674b587108e Mon Sep 17 00:00:00 2001 From: Thomas Lutz Date: Thu, 24 Oct 2019 00:14:53 +0200 Subject: [PATCH 032/417] fix(build): make version checker more robust in install (#260) Resolves #246. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index c9fd5642a2..afc7b57d02 100755 --- a/install.sh +++ b/install.sh @@ -24,8 +24,8 @@ DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); -# Function below is inspired by https://stackoverflow.com/a/29394504/90297 -function ver { printf "%03d%03d%03d%03d" $(echo "$1" | sed 's/^0*\([0-9]\+\)\.0*\([0-9]\+\)\.0*\([0-9]\+\).*/\1 \2 \3/' | head -n 3 ); } +# Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 +function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" From 2018fb50d083fbbbd0faafb78d40fe0bca6a2cb7 Mon Sep 17 00:00:00 2001 From: mark burdett Date: Mon, 28 Oct 2019 12:33:06 -0700 Subject: [PATCH 033/417] Declare ARG in Dockerfile. (#265) Resolves [Warning] One or more build-args [SENTRY_IMAGE] were not consumed. --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 0a3dae537c..08d8252a34 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1 +1,2 @@ +ARG SENTRY_IMAGE FROM ${SENTRY_IMAGE:-sentry:9.1.2}-onbuild From e2b7c743af70e192588d843512412e653eddab17 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 6 Nov 2019 23:39:55 +0300 Subject: [PATCH 034/417] meta(LICENSE): Switch to Business Source License 1.1 (#267) See the public announcement about this on: https://blog.sentry.io/2019/11/06/relicensing-sentry/ --- LICENSE | 304 +++++++++++++++++++------------------------------------- 1 file changed, 103 insertions(+), 201 deletions(-) diff --git a/LICENSE b/LICENSE index c085307bd4..7adada950a 100644 --- a/LICENSE +++ b/LICENSE @@ -1,202 +1,104 @@ +Business Source License 1.1 - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2016 Functional Software, Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. +Parameters + +Licensor: Functional Software, Inc. +Licensed Work: Sentry + The Licensed Work is (c) 2019 Functional Software, Inc. +Additional Use Grant: You may make use of the Licensed Work, provided that you do + not use the Licensed Work for an Application Monitoring + Service. + + An "Application Monitoring Service" is a commercial offering + that allows third parties (other than your employees and + contractors) to access the functionality of the Licensed + Work so that such third parties directly benefit from the + error-reporting or application monitoring features of the + Licensed Work. + +Change Date: 2022-09-15 + +Change License: Apache License, Version 2.0 + +For information about alternative licensing arrangements for the Software, +please visit: https://sentry.io/pricing/ + +Notice + +The Business Source License (this document, or the "License") is not an Open +Source license. However, the Licensed Work will eventually be made available +under an Open Source License, as stated in this License. + +License text copyright (c) 2017 MariaDB Corporation Ab, All Rights Reserved. +"Business Source License" is a trademark of MariaDB Corporation Ab. + +----------------------------------------------------------------------------- + +Business Source License 1.1 + +Terms + +The Licensor hereby grants you the right to copy, modify, create derivative +works, redistribute, and make non-production use of the Licensed Work. The +Licensor may make an Additional Use Grant, above, permitting limited +production use. + +Effective on the Change Date, or the fourth anniversary of the first publicly +available distribution of a specific version of the Licensed Work under this +License, whichever comes first, the Licensor hereby grants you rights under +the terms of the Change License, and the rights granted in the paragraph +above terminate. + +If your use of the Licensed Work does not comply with the requirements +currently in effect as described in this License, you must purchase a +commercial license from the Licensor, its affiliated entities, or authorized +resellers, or you must refrain from using the Licensed Work. + +All copies of the original and modified Licensed Work, and derivative works +of the Licensed Work, are subject to this License. This License applies +separately for each version of the Licensed Work and the Change Date may vary +for each version of the Licensed Work released by Licensor. + +You must conspicuously display this License on each original or modified copy +of the Licensed Work. If you receive the Licensed Work in original or +modified form from a third party, the terms and conditions set forth in this +License apply to your use of that work. + +Any use of the Licensed Work in violation of this License will automatically +terminate your rights under this License for the current and all other +versions of the Licensed Work. + +This License does not grant you any right in any trademark or logo of +Licensor or its affiliates (provided that you may use a trademark or logo of +Licensor as expressly required by this License). + +TO THE EXTENT PERMITTED BY APPLICABLE LAW, THE LICENSED WORK IS PROVIDED ON +AN "AS IS" BASIS. LICENSOR HEREBY DISCLAIMS ALL WARRANTIES AND CONDITIONS, +EXPRESS OR IMPLIED, INCLUDING (WITHOUT LIMITATION) WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE, NON-INFRINGEMENT, AND +TITLE. + +MariaDB hereby grants you permission to use this License’s text to license +your works, and to refer to it using the trademark "Business Source License", +as long as you comply with the Covenants of Licensor below. + +Covenants of Licensor + +In consideration of the right to use this License’s text and the "Business +Source License" name and trademark, Licensor covenants to MariaDB, and to all +other recipients of the licensed work to be provided by Licensor: + +1. To specify as the Change License the GPL Version 2.0 or any later version, + or a license that is compatible with GPL Version 2.0 or a later version, + where "compatible" means that software provided under the Change License can + be included in a program with software provided under GPL Version 2.0 or a + later version. Licensor may specify additional Change Licenses without + limitation. + +2. To either: (a) specify an additional grant of rights to use that does not + impose any additional restriction on the right granted in this License, as + the Additional Use Grant; or (b) insert the text "None". + +3. To specify a Change Date. + +4. Not to modify this License in any other way. From 5d064c2224aa73a45ee0e29063be85c8810f4707 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 12 Nov 2019 02:18:59 +0300 Subject: [PATCH 035/417] feat(sentry10): Make on-premise work for Sentry 10 (#220) --- .dockerignore | 12 - .env | 1 + .env.example | 3 - .gitignore | 8 +- .travis.yml | 11 +- Dockerfile | 2 - README.md | 16 +- cron/Dockerfile | 6 + cron/entrypoint.sh | 15 + docker-compose.yml | 178 ++++++--- install.sh | 62 ++- sentry.conf.py | 363 ------------------ sentry/Dockerfile | 18 + config.yml => sentry/config.example.yml | 18 +- .../requirements.example.txt | 0 sentry/sentry.conf.example.py | 223 +++++++++++ test.sh | 13 +- 17 files changed, 485 insertions(+), 464 deletions(-) delete mode 100644 .dockerignore create mode 100644 .env delete mode 100644 .env.example delete mode 100644 Dockerfile create mode 100644 cron/Dockerfile create mode 100755 cron/entrypoint.sh delete mode 100644 sentry.conf.py create mode 100644 sentry/Dockerfile rename config.yml => sentry/config.example.yml (83%) rename requirements.txt => sentry/requirements.example.txt (100%) create mode 100644 sentry/sentry.conf.example.py diff --git a/.dockerignore b/.dockerignore deleted file mode 100644 index e45d0676ff..0000000000 --- a/.dockerignore +++ /dev/null @@ -1,12 +0,0 @@ -.git -.gitignore -.dockerignore -Makefile -README.md -*.pyc -*.tar -docker-compose.yml -data/ -.travis.yml -install.sh -test.sh diff --git a/.env b/.env new file mode 100644 index 0000000000..2eeb2440a4 --- /dev/null +++ b/.env @@ -0,0 +1 @@ +SENTRY_EVENT_RETENTION_DAYS=90 diff --git a/.env.example b/.env.example deleted file mode 100644 index c560055d33..0000000000 --- a/.env.example +++ /dev/null @@ -1,3 +0,0 @@ -# Run `docker-compose run web config generate-secret-key` -# to get the SENTRY_SECRET_KEY value. -SENTRY_SECRET_KEY= diff --git a/.gitignore b/.gitignore index d31ea68355..802aab9241 100644 --- a/.gitignore +++ b/.gitignore @@ -68,9 +68,11 @@ target/ # https://docs.docker.com/compose/extends/ docker-compose.override.yml -# env config -.env - *.tar data/ .vscode/tags + +# custom Sentry config +sentry/sentry.conf.py +sentry/config.yml +sentry/requirements.txt diff --git a/.travis.yml b/.travis.yml index 07a67bc7e8..8ec590d44b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,12 +2,17 @@ language: bash services: docker env: - - SENTRY_IMAGE=sentry:9.1.2 - - SENTRY_IMAGE=getsentry/sentry:latest + - DOCKER_COMPOSE_VERSION=1.24.1 + +before_install: + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin script: - ./install.sh - - docker-compose run --rm web createuser --superuser --email test@sentry.io --password test123TEST + - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index 08d8252a34..0000000000 --- a/Dockerfile +++ /dev/null @@ -1,2 +0,0 @@ -ARG SENTRY_IMAGE -FROM ${SENTRY_IMAGE:-sentry:9.1.2}-onbuild diff --git a/README.md b/README.md index bac6c436a7..486cad4622 100644 --- a/README.md +++ b/README.md @@ -1,21 +1,23 @@ -# Sentry On-Premise [![Build Status][build-status-image]][build-status-url] +# Sentry 10 On-Premise BETA [![Build Status][build-status-image]][build-status-url] Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). +**NOTE:** If you are not installing Sentry from scratch, visit [On-Premise Stable for Sentry 9.1.2](https://github.com/getsentry/onpremise/tree/stable) as this version is not fully backward compatible. + ## Requirements * Docker 17.05.0+ - * Compose 1.17.0+ + * Compose 1.19.0+ ## Minimum Hardware Requirements: - * You need at least 3GB RAM + * You need at least 2400MB RAM ## Setup To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. -There may need to be modifications to the included `docker-compose.yml` file to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script. +There may need to be modifications to the included example config files (`sentry/config.example.yml` and `sentry/sentry.conf.example.py`) to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script and copy them without the `.example` extensions in the name (such as `sentry/sentry.conf.py`) before running the `install.sh` script. The recommended way to customize your configuration is using the files below, in that order: @@ -23,8 +25,14 @@ The recommended way to customize your configuration is using the files below, in * `sentry.conf.py` * `.env` w/ environment variables +We currently support a very minimal set of environment variables to promote other means of configuration. + If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! +## Event Retention + +Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. + ## Securing Sentry with SSL/TLS If you'd like to protect your Sentry install with SSL/TLS, there are diff --git a/cron/Dockerfile b/cron/Dockerfile new file mode 100644 index 0000000000..0e6e76e9e8 --- /dev/null +++ b/cron/Dockerfile @@ -0,0 +1,6 @@ +ARG BASE_IMAGE +FROM ${BASE_IMAGE} +RUN apt-get update && apt-get install -y --no-install-recommends cron && \ + rm -r /var/lib/apt/lists/* +COPY entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh new file mode 100755 index 0000000000..b0f4d5b75e --- /dev/null +++ b/cron/entrypoint.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash + +# Prior art: +# - https://git.io/fjNOg +# - https://blog.knoldus.com/running-a-cron-job-in-docker-container/ + +declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /container.env + +{ for cron_job in "$@"; do echo -e "SHELL=/bin/bash +BASH_ENV=/container.env +${cron_job} > /proc/1/fd/1 2>/proc/1/fd/2"; done } \ + | sed --regexp-extended 's/\\(.)/\1/g' \ + | crontab - +crontab -l +exec cron -f -l -L 15 diff --git a/docker-compose.yml b/docker-compose.yml index 18cee6a316..12f21cb67a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,67 +1,159 @@ -# NOTE: This docker-compose.yml is meant to be just an example of how -# you could accomplish this on your own. It is not intended to work in -# all use-cases and must be adapted to fit your needs. This is merely -# a guideline. - -# See docs.getsentry.com/on-premise/server/ for full -# instructions - version: '3.4' - -x-defaults: &defaults +x-restart-policy: &restart_policy restart: unless-stopped +x-sentry-defaults: &sentry_defaults + << : *restart_policy build: - context: . + context: ./sentry + args: + - SENTRY_IMAGE + image: sentry-onpremise-local depends_on: - redis - postgres - memcached - smtp - env_file: .env + - snuba-api + - snuba-consumer + - snuba-replacer + - symbolicator + - kafka environment: - SENTRY_MEMCACHED_HOST: memcached - SENTRY_REDIS_HOST: redis - SENTRY_POSTGRES_HOST: postgres - SENTRY_EMAIL_HOST: smtp + SNUBA: 'http://snuba-api:1218' volumes: - - sentry-data:/var/lib/sentry/files - - + - 'sentry-data:/var/lib/sentry/files' +x-snuba-defaults: &snuba_defaults + << : *restart_policy + depends_on: + - redis + - clickhouse + - kafka + image: 'getsentry/snuba:latest' + environment: + SNUBA_SETTINGS: docker + CLICKHOUSE_HOST: clickhouse + DEFAULT_BROKERS: 'kafka:9093' + REDIS_HOST: redis + # TODO: Remove these after getsentry/snuba#353 + UWSGI_MAX_REQUESTS: '10000' + UWSGI_DISABLE_LOGGING: 'true' + UWSGI_ENABLE_THREADS: 'true' + UWSGI_DIE_ON_TERM: 'true' + UWSGI_NEED_APP: 'true' + UWSGI_IGNORE_SIGPIPE: 'true' + UWSGI_IGNORE_WRITE_ERRORS: 'true' + UWSGI_DISABLE_WRITE_EXCEPTION: 'true' services: smtp: - restart: unless-stopped + << : *restart_policy image: tianon/exim4 - memcached: - restart: unless-stopped - image: memcached:1.5-alpine - + << : *restart_policy + image: 'memcached:1.5-alpine' redis: - restart: unless-stopped - image: redis:3.2-alpine - + << : *restart_policy + image: 'redis:5.0-alpine' + volumes: + - 'sentry-redis:/data' postgres: - restart: unless-stopped - image: postgres:9.5 + << : *restart_policy + image: 'postgres:9.6' + volumes: + - 'sentry-postgres:/var/lib/postgresql/data' + zookeeper: + << : *restart_policy + image: 'confluentinc/cp-zookeeper:5.1.2' + environment: + ZOOKEEPER_CLIENT_PORT: '2181' + ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'ERROR' + ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'ERROR' + volumes: + - 'sentry-zookeeper:/var/lib/zookeeper' + kafka: + << : *restart_policy + depends_on: + - zookeeper + image: 'confluentinc/cp-kafka:5.1.2' + environment: + KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' + KAFKA_LISTENERS: 'INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092' + KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka:9093,EXTERNAL://kafka:9092' + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT' + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN' + KAFKA_LOG4J_ROOT_LOGLEVEL: 'ERROR' + KAFKA_TOOLS_LOG4J_LOGLEVEL: 'ERROR' + volumes: + - 'sentry-kafka:/var/lib/kafka' + clickhouse: + << : *restart_policy + image: 'yandex/clickhouse-server:19.4' + ulimits: + nofile: + soft: 262144 + hard: 262144 + volumes: + - 'sentry-clickhouse:/var/lib/clickhouse' + snuba-api: + << : *snuba_defaults + snuba-consumer: + << : *snuba_defaults + command: consumer --auto-offset-reset=latest --max-batch-time-ms 750 + snuba-replacer: + << : *snuba_defaults + command: replacer --auto-offset-reset=latest --max-batch-size 3 + snuba-cleanup: + << : *snuba_defaults + image: snuba-cleanup-onpremise-local + build: + context: ./cron + args: + BASE_IMAGE: 'getsentry/snuba:latest' + command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' + symbolicator: + << : *restart_policy + image: us.gcr.io/sentryio/symbolicator:latest volumes: - - sentry-postgres:/var/lib/postgresql/data - + - 'sentry-symbolicator:/data' + command: run + symbolicator-cleanup: + image: symbolicator-cleanup-onpremise-local + build: + context: ./cron + args: + BASE_IMAGE: 'us.gcr.io/sentryio/symbolicator:latest' + command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' web: - <<: *defaults + << : *sentry_defaults ports: - - '9000:9000' - + - '9000:9000/tcp' cron: - <<: *defaults + << : *sentry_defaults command: run cron - worker: - <<: *defaults + << : *sentry_defaults command: run worker - - + sentry-cleanup: + << : *sentry_defaults + image: sentry-cleanup-onpremise-local + build: + context: ./cron + args: + BASE_IMAGE: 'sentry-onpremise-local' + command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' volumes: - sentry-data: - external: true - sentry-postgres: - external: true + sentry-data: + external: true + sentry-postgres: + external: true + sentry-redis: + external: true + sentry-zookeeper: + external: true + sentry-kafka: + external: true + sentry-clickhouse: + external: true + sentry-symbolicator: + external: true diff --git a/install.sh b/install.sh index afc7b57d02..4cc93c87d4 100755 --- a/install.sh +++ b/install.sh @@ -2,9 +2,12 @@ set -e MIN_DOCKER_VERSION='17.05.0' -MIN_COMPOSE_VERSION='1.17.0' -MIN_RAM=3072 # MB -ENV_FILE='.env' +MIN_COMPOSE_VERSION='1.19.0' +MIN_RAM=2400 # MB + +SENTRY_CONFIG_PY='sentry/sentry.conf.py' +SENTRY_CONFIG_YML='sentry/config.yml' +SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' DID_CLEAN_UP=0 # the cleanup function will be the exit point @@ -27,6 +30,16 @@ RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Me # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } +# Thanks to https://stackoverflow.com/a/25123013/90297 for the quick `sed` pattern +function ensure_file_from_example { + if [ -f "$1" ]; then + echo "$1 already exists, skipped creation." + else + echo "Creating $1..." + cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" + fi +} + if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit -1 @@ -42,33 +55,37 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then exit -1 fi +echo "" +ensure_file_from_example $SENTRY_CONFIG_PY +ensure_file_from_example $SENTRY_CONFIG_YML +ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS + echo "" echo "Creating volumes for persistent storage..." echo "Created $(docker volume create --name=sentry-data)." echo "Created $(docker volume create --name=sentry-postgres)." -echo "" +echo "Created $(docker volume create --name=sentry-redis)." +echo "Created $(docker volume create --name=sentry-zookeeper)." +echo "Created $(docker volume create --name=sentry-kafka)." +echo "Created $(docker volume create --name=sentry-clickhouse)." +echo "Created $(docker volume create --name=sentry-symbolicator)." -if [ -f "$ENV_FILE" ]; then - echo "$ENV_FILE already exists, skipped creation." -else - echo "Creating $ENV_FILE..." - cp -n .env.example "$ENV_FILE" -fi +echo "" +echo "Generating secret key..." +# This is to escape the secret key to be used in sed below +SECRET_KEY=$(head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') +sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML +echo "Secret key written to $SENTRY_CONFIG_YML" echo "" echo "Building and tagging Docker images..." echo "" -docker-compose build +# Build the sentry onpremise image first as it is needed for the cron image +docker-compose build --force-rm web +docker-compose build --force-rm echo "" echo "Docker images built." -echo "" -echo "Generating secret key..." -# This is to escape the secret key to be used in sed below -SECRET_KEY=$(docker-compose run --rm web config generate-secret-key 2> /dev/null | tail -n1 | sed -e 's/[\/&]/\\&/g') -sed -i -e 's/^SENTRY_SECRET_KEY=.*$/SENTRY_SECRET_KEY='"$SECRET_KEY"'/' $ENV_FILE -echo "Secret key written to $ENV_FILE" - echo "" echo "Setting up database..." if [ $CI ]; then @@ -83,11 +100,18 @@ else docker-compose run --rm web upgrade fi +echo "Boostrapping Snuba..." +docker-compose up -d kafka redis clickhouse +until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do + docker-compose run --rm snuba-api bootstrap --force || true; +done; +echo "" + cleanup echo "" echo "----------------" -echo "You're all done! Run the following command get Sentry running:" +echo "You're all done! Run the following command to get Sentry running:" echo "" echo " docker-compose up -d" echo "" diff --git a/sentry.conf.py b/sentry.conf.py deleted file mode 100644 index d02e0d8343..0000000000 --- a/sentry.conf.py +++ /dev/null @@ -1,363 +0,0 @@ -# This file is just Python, with a touch of Django which means -# you can inherit and tweak settings to your hearts content. - -# For Docker, the following environment variables are supported: -# SENTRY_POSTGRES_HOST -# SENTRY_POSTGRES_PORT -# SENTRY_DB_NAME -# SENTRY_DB_USER -# SENTRY_DB_PASSWORD -# SENTRY_RABBITMQ_HOST -# SENTRY_RABBITMQ_USERNAME -# SENTRY_RABBITMQ_PASSWORD -# SENTRY_RABBITMQ_VHOST -# SENTRY_REDIS_HOST -# SENTRY_REDIS_PASSWORD -# SENTRY_REDIS_PORT -# SENTRY_REDIS_DB -# SENTRY_MEMCACHED_HOST -# SENTRY_MEMCACHED_PORT -# SENTRY_FILESTORE_DIR -# SENTRY_SERVER_EMAIL -# SENTRY_EMAIL_HOST -# SENTRY_EMAIL_PORT -# SENTRY_EMAIL_USER -# SENTRY_EMAIL_PASSWORD -# SENTRY_EMAIL_USE_TLS -# SENTRY_EMAIL_LIST_NAMESPACE -# SENTRY_ENABLE_EMAIL_REPLIES -# SENTRY_SMTP_HOSTNAME -# SENTRY_MAILGUN_API_KEY -# SENTRY_SINGLE_ORGANIZATION -# SENTRY_SECRET_KEY -# (slack integration) -# SENTRY_SLACK_CLIENT_ID -# SENTRY_SLACK_CLIENT_SECRET -# SENTRY_SLACK_VERIFICATION_TOKEN -# (github plugin, sso) -# GITHUB_APP_ID -# GITHUB_API_SECRET -# (github integration) -# SENTRY_GITHUB_APP_ID -# SENTRY_GITHUB_APP_CLIENT_ID -# SENTRY_GITHUB_APP_CLIENT_SECRET -# SENTRY_GITHUB_APP_WEBHOOK_SECRET -# SENTRY_GITHUB_APP_PRIVATE_KEY -# (azure devops integration) -# SENTRY_VSTS_CLIENT_ID -# SENTRY_VSTS_CLIENT_SECRET -# (bitbucket plugin) -# BITBUCKET_CONSUMER_KEY -# BITBUCKET_CONSUMER_SECRET -from sentry.conf.server import * # NOQA -from sentry.utils.types import Bool, Int - -import os -import os.path -import six - -CONF_ROOT = os.path.dirname(__file__) - -postgres = env('SENTRY_POSTGRES_HOST') or (env('POSTGRES_PORT_5432_TCP_ADDR') and 'postgres') -if postgres: - DATABASES = { - 'default': { - 'ENGINE': 'sentry.db.postgres', - 'NAME': ( - env('SENTRY_DB_NAME') - or env('POSTGRES_ENV_POSTGRES_USER') - or 'postgres' - ), - 'USER': ( - env('SENTRY_DB_USER') - or env('POSTGRES_ENV_POSTGRES_USER') - or 'postgres' - ), - 'PASSWORD': ( - env('SENTRY_DB_PASSWORD') - or env('POSTGRES_ENV_POSTGRES_PASSWORD') - or '' - ), - 'HOST': postgres, - 'PORT': ( - env('SENTRY_POSTGRES_PORT') - or '' - ), - }, - } - -# You should not change this setting after your database has been created -# unless you have altered all schemas first -SENTRY_USE_BIG_INTS = True - -# If you're expecting any kind of real traffic on Sentry, we highly recommend -# configuring the CACHES and Redis settings - -########### -# General # -########### - -# Instruct Sentry that this install intends to be run by a single organization -# and thus various UI optimizations should be enabled. -SENTRY_SINGLE_ORGANIZATION = env('SENTRY_SINGLE_ORGANIZATION', True) - -######### -# Redis # -######### - -# Generic Redis configuration used as defaults for various things including: -# Buffers, Quotas, TSDB - -redis = env('SENTRY_REDIS_HOST') or (env('REDIS_PORT_6379_TCP_ADDR') and 'redis') -if not redis: - raise Exception('Error: REDIS_PORT_6379_TCP_ADDR (or SENTRY_REDIS_HOST) is undefined, did you forget to `--link` a redis container?') - -redis_password = env('SENTRY_REDIS_PASSWORD') or '' -redis_port = env('SENTRY_REDIS_PORT') or '6379' -redis_db = env('SENTRY_REDIS_DB') or '0' - -SENTRY_OPTIONS.update({ - 'redis.clusters': { - 'default': { - 'hosts': { - 0: { - 'host': redis, - 'password': redis_password, - 'port': redis_port, - 'db': redis_db, - }, - }, - }, - }, -}) - -######### -# Cache # -######### - -# Sentry currently utilizes two separate mechanisms. While CACHES is not a -# requirement, it will optimize several high throughput patterns. - -memcached = env('SENTRY_MEMCACHED_HOST') or (env('MEMCACHED_PORT_11211_TCP_ADDR') and 'memcached') -if memcached: - memcached_port = ( - env('SENTRY_MEMCACHED_PORT') - or '11211' - ) - CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': [memcached + ':' + memcached_port], - 'TIMEOUT': 3600, - } - } - -# A primary cache is required for things such as processing events -SENTRY_CACHE = 'sentry.cache.redis.RedisCache' - -######### -# Queue # -######### - -# See https://docs.getsentry.com/on-premise/server/queue/ for more -# information on configuring your queue broker and workers. Sentry relies -# on a Python framework called Celery to manage queues. - -rabbitmq = env('SENTRY_RABBITMQ_HOST') or (env('RABBITMQ_PORT_5672_TCP_ADDR') and 'rabbitmq') - -if rabbitmq: - BROKER_URL = ( - 'amqp://' + ( - env('SENTRY_RABBITMQ_USERNAME') - or env('RABBITMQ_ENV_RABBITMQ_DEFAULT_USER') - or 'guest' - ) + ':' + ( - env('SENTRY_RABBITMQ_PASSWORD') - or env('RABBITMQ_ENV_RABBITMQ_DEFAULT_PASS') - or 'guest' - ) + '@' + rabbitmq + '/' + ( - env('SENTRY_RABBITMQ_VHOST') - or env('RABBITMQ_ENV_RABBITMQ_DEFAULT_VHOST') - or '/' - ) - ) -else: - BROKER_URL = 'redis://:' + redis_password + '@' + redis + ':' + redis_port + '/' + redis_db - - -############### -# Rate Limits # -############### - -# Rate limits apply to notification handlers and are enforced per-project -# automatically. - -SENTRY_RATELIMITER = 'sentry.ratelimits.redis.RedisRateLimiter' - -################## -# Update Buffers # -################## - -# Buffers (combined with queueing) act as an intermediate layer between the -# database and the storage API. They will greatly improve efficiency on large -# numbers of the same events being sent to the API in a short amount of time. -# (read: if you send any kind of real data to Sentry, you should enable buffers) - -SENTRY_BUFFER = 'sentry.buffer.redis.RedisBuffer' - -########## -# Quotas # -########## - -# Quotas allow you to rate limit individual projects or the Sentry install as -# a whole. - -SENTRY_QUOTAS = 'sentry.quotas.redis.RedisQuota' - -######## -# TSDB # -######## - -# The TSDB is used for building charts as well as making things like per-rate -# alerts possible. - -SENTRY_TSDB = 'sentry.tsdb.redis.RedisTSDB' - -########### -# Digests # -########### - -# The digest backend powers notification summaries. - -SENTRY_DIGESTS = 'sentry.digests.backends.redis.RedisBackend' - -################ -# File storage # -################ - -# Uploaded media uses these `filestore` settings. The available -# backends are either `filesystem` or `s3`. - -SENTRY_OPTIONS['filestore.backend'] = 'filesystem' -SENTRY_OPTIONS['filestore.options'] = { - 'location': env('SENTRY_FILESTORE_DIR'), -} - -############## -# Web Server # -############## - -# If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto -# header and set `SENTRY_USE_SSL=1` - -if env('SENTRY_USE_SSL', False): - SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') - SESSION_COOKIE_SECURE = True - CSRF_COOKIE_SECURE = True - SOCIAL_AUTH_REDIRECT_IS_HTTPS = True - -SENTRY_WEB_HOST = '0.0.0.0' -SENTRY_WEB_PORT = 9000 -SENTRY_WEB_OPTIONS = { - 'http': '%s:%s' % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), - 'protocol': 'uwsgi', - # This is need to prevent https://git.io/fj7Lw - 'uwsgi-socket': None, - 'http-keepalive': True, - 'memory-report': False, - # 'workers': 3, # the number of web workers -} - - -########## -# Docker # -########## - -# Docker's environment configuration needs to happen -# prior to anything that might rely on these values to -# enable more "smart" configuration. - -ENV_CONFIG_MAPPING = { - 'SENTRY_EMAIL_PASSWORD': 'mail.password', - 'SENTRY_EMAIL_USER': 'mail.username', - 'SENTRY_EMAIL_PORT': ('mail.port', Int), - 'SENTRY_EMAIL_USE_TLS': ('mail.use-tls', Bool), - 'SENTRY_EMAIL_HOST': 'mail.host', - 'SENTRY_SERVER_EMAIL': 'mail.from', - 'SENTRY_ENABLE_EMAIL_REPLIES': ('mail.enable-replies', Bool), - 'SENTRY_EMAIL_LIST_NAMESPACE': 'mail.list-namespace', - 'SENTRY_SMTP_HOSTNAME': 'mail.reply-hostname', - 'SENTRY_SECRET_KEY': 'system.secret-key', - - # If you're using mailgun for inbound mail, set your API key and configure a - # route to forward to /api/hooks/mailgun/inbound/ - 'SENTRY_MAILGUN_API_KEY': 'mail.mailgun-api-key', - - 'SENTRY_SLACK_CLIENT_ID': 'slack.client-id', - 'SENTRY_SLACK_CLIENT_SECRET': 'slack.client-secret', - 'SENTRY_SLACK_VERIFICATION_TOKEN': 'slack.verification-token', - - 'SENTRY_GITHUB_APP_ID': ('github-app.id', Int), - 'SENTRY_GITHUB_APP_CLIENT_ID': 'github-app.client-id', - 'SENTRY_GITHUB_APP_CLIENT_SECRET': 'github-app.client-secret', - 'SENTRY_GITHUB_APP_WEBHOOK_SECRET': 'github-app.webhook-secret', - 'SENTRY_GITHUB_APP_PRIVATE_KEY': 'github-app.private-key', - - 'SENTRY_VSTS_CLIENT_ID': 'vsts.client-id', - 'SENTRY_VSTS_CLIENT_SECRET': 'vsts.client-secret', -} - - -def bind_env_config(config=SENTRY_OPTIONS, mapping=ENV_CONFIG_MAPPING): - """ - Automatically bind SENTRY_OPTIONS from a set of environment variables. - """ - for env_var, item in six.iteritems(mapping): - # HACK: we need to check both in `os.environ` and `env._cache`. - # This is very much an implementation detail leaking out - # due to assumptions about how `env` would be used previously. - # `env` will pop values out of `os.environ` when they are seen, - # so checking against `os.environ` only means it's likely - # they won't exist if `env()` has been called on the variable - # before at any point. So we're choosing to check both, but this - # behavior is different since we're trying to only conditionally - # apply variables, instead of setting them always. - if env_var not in os.environ and env_var not in env._cache: - continue - if isinstance(item, tuple): - opt_key, type_ = item - else: - opt_key, type_ = item, None - config[opt_key] = env(env_var, type=type_) - -# If this value ever becomes compromised, it's important to regenerate your -# SENTRY_SECRET_KEY. Changing this value will result in all current sessions -# being invalidated. -secret_key = env('SENTRY_SECRET_KEY') -if not secret_key: - raise Exception('Error: SENTRY_SECRET_KEY is undefined, run `generate-secret-key` and set to -e SENTRY_SECRET_KEY') - -if 'SENTRY_RUNNING_UWSGI' not in os.environ and len(secret_key) < 32: - print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') - print('!! CAUTION !!') - print('!! Your SENTRY_SECRET_KEY is potentially insecure. !!') - print('!! We recommend at least 32 characters long. !!') - print('!! Regenerate with `generate-secret-key`. !!') - print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') - -# Grab the easy configuration first - these are all fixed -# key=value with no logic behind them -bind_env_config() - -# If you specify a MAILGUN_API_KEY, you definitely want EMAIL_REPLIES -if SENTRY_OPTIONS.get('mail.mailgun-api-key'): - SENTRY_OPTIONS.setdefault('mail.enable-replies', True) - -if 'GITHUB_APP_ID' in os.environ: - GITHUB_EXTENDED_PERMISSIONS = ['repo'] - GITHUB_APP_ID = env('GITHUB_APP_ID') - GITHUB_API_SECRET = env('GITHUB_API_SECRET') - -if 'BITBUCKET_CONSUMER_KEY' in os.environ: - BITBUCKET_CONSUMER_KEY = env('BITBUCKET_CONSUMER_KEY') - BITBUCKET_CONSUMER_SECRET = env('BITBUCKET_CONSUMER_SECRET') diff --git a/sentry/Dockerfile b/sentry/Dockerfile new file mode 100644 index 0000000000..032f340847 --- /dev/null +++ b/sentry/Dockerfile @@ -0,0 +1,18 @@ +ARG SENTRY_IMAGE +FROM ${SENTRY_IMAGE:-getsentry/sentry:latest} + +WORKDIR /usr/src/sentry + +# Add WORKDIR to PYTHONPATH so local python files don't need to be installed +ENV PYTHONPATH /usr/src/sentry +COPY . /usr/src/sentry + +# Hook for installing additional plugins +RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi + +# Hook for installing a local app as an addon +RUN if [ -s setup.py ]; then pip install -e .; fi + +# Hook for staging in custom configs +RUN if [ -s sentry.conf.py ]; then cp sentry.conf.py $SENTRY_CONF/; fi \ + && if [ -s config.yml ]; then cp config.yml $SENTRY_CONF/; fi diff --git a/config.yml b/sentry/config.example.yml similarity index 83% rename from config.yml rename to sentry/config.example.yml index 8a0844595c..5547720a1a 100644 --- a/config.yml +++ b/sentry/config.example.yml @@ -8,7 +8,7 @@ ############### # mail.backend: 'smtp' # Use dummy if you want to disable email entirely -# mail.host: 'localhost' +mail.host: 'smtp' # mail.port: 25 # mail.username: '' # mail.password: '' @@ -17,13 +17,14 @@ # mail.from: 'root@localhost' # If you'd like to configure email replies, enable this. -# mail.enable-replies: false +# mail.enable-replies: true # When email-replies are enabled, this value is used in the Reply-To header # mail.reply-hostname: '' # If you're using mailgun for inbound mail, set your API key and configure a # route to forward to /api/hooks/mailgun/inbound/ +# Also don't forget to set `mail.enable-replies: true` above. # mail.mailgun-api-key: '' ################### @@ -33,7 +34,7 @@ # If this file ever becomes compromised, it's important to regenerate your a new key # Changing this value will result in all current sessions being invalidated. # A new key can be generated with `$ sentry config generate-secret-key` -# system.secret-key: 'changeme' +system.secret-key: '!!changeme!!' # The ``redis.clusters`` setting is used, unsurprisingly, to configure Redis # clusters. These clusters can be then referred to by name when configuring @@ -52,12 +53,17 @@ # Uploaded media uses these `filestore` settings. The available # backends are either `filesystem` or `s3`. -# filestore.backend: 'filesystem' -# filestore.options: -# location: '/tmp/sentry-files' +filestore.backend: 'filesystem' +filestore.options: + location: '/var/lib/sentry/files' # filestore.backend: 's3' # filestore.options: # access_key: 'AKIXXXXXX' # secret_key: 'XXXXXXX' # bucket_name: 's3-bucket-name' + +system.internal-url-prefix: 'http://web:9000' +symbolicator.enabled: true +symbolicator.options: + url: "http://symbolicator:3021" diff --git a/requirements.txt b/sentry/requirements.example.txt similarity index 100% rename from requirements.txt rename to sentry/requirements.example.txt diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py new file mode 100644 index 0000000000..a0e989ce12 --- /dev/null +++ b/sentry/sentry.conf.example.py @@ -0,0 +1,223 @@ +# This file is just Python, with a touch of Django which means +# you can inherit and tweak settings to your hearts content. + +from sentry.conf.server import * # NOQA + +DATABASES = { + "default": { + "ENGINE": "sentry.db.postgres", + "NAME": "postgres", + "USER": "postgres", + "PASSWORD": "", + "HOST": "postgres", + "PORT": "", + } +} + +# You should not change this setting after your database has been created +# unless you have altered all schemas first +SENTRY_USE_BIG_INTS = True + +# If you're expecting any kind of real traffic on Sentry, we highly recommend +# configuring the CACHES and Redis settings + +########### +# General # +########### + +# Instruct Sentry that this install intends to be run by a single organization +# and thus various UI optimizations should be enabled. +SENTRY_SINGLE_ORGANIZATION = True + +######### +# Redis # +######### + +# Generic Redis configuration used as defaults for various things including: +# Buffers, Quotas, TSDB + +SENTRY_OPTIONS["redis.clusters"] = { + "default": { + "hosts": {0: {"host": "redis", "password": "", "port": "6379", "db": "0"}} + } +} + +######### +# Queue # +######### + +# See https://docs.getsentry.com/on-premise/server/queue/ for more +# information on configuring your queue broker and workers. Sentry relies +# on a Python framework called Celery to manage queues. + +rabbitmq_host = None +if rabbitmq_host: + BROKER_URL = "amqp://{username}:{password}@{host}/{vhost}".format( + username="guest", password="guest", host=rabbitmq_host, vhost="/" + ) +else: + BROKER_URL = "redis://:{password}@{host}:{port}/{db}".format( + **SENTRY_OPTIONS["redis.clusters"]["default"]["hosts"][0] + ) + + +######### +# Cache # +######### + +# Sentry currently utilizes two separate mechanisms. While CACHES is not a +# requirement, it will optimize several high throughput patterns. + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.memcached.MemcachedCache", + "LOCATION": ["memcached:11211"], + "TIMEOUT": 3600, + } +} + +# A primary cache is required for things such as processing events +SENTRY_CACHE = "sentry.cache.redis.RedisCache" + +DEFAULT_KAFKA_OPTIONS = { + "bootstrap.servers": "kafka:9092", + "message.max.bytes": 50000000, + "socket.timeout.ms": 1000, +} + +SENTRY_EVENTSTREAM = "sentry.eventstream.kafka.KafkaEventStream" +SENTRY_EVENTSTREAM_OPTIONS = {"producer_configuration": DEFAULT_KAFKA_OPTIONS} + +KAFKA_CLUSTERS["default"] = DEFAULT_KAFKA_OPTIONS + +############### +# Rate Limits # +############### + +# Rate limits apply to notification handlers and are enforced per-project +# automatically. + +SENTRY_RATELIMITER = "sentry.ratelimits.redis.RedisRateLimiter" + +################## +# Update Buffers # +################## + +# Buffers (combined with queueing) act as an intermediate layer between the +# database and the storage API. They will greatly improve efficiency on large +# numbers of the same events being sent to the API in a short amount of time. +# (read: if you send any kind of real data to Sentry, you should enable buffers) + +SENTRY_BUFFER = "sentry.buffer.redis.RedisBuffer" + +########## +# Quotas # +########## + +# Quotas allow you to rate limit individual projects or the Sentry install as +# a whole. + +SENTRY_QUOTAS = "sentry.quotas.redis.RedisQuota" + +######## +# TSDB # +######## + +# The TSDB is used for building charts as well as making things like per-rate +# alerts possible. + +SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB" + +######### +# SNUBA # +######### + +SENTRY_SEARCH = "sentry.search.snuba.SnubaSearchBackend" +SENTRY_SEARCH_OPTIONS = {} +SENTRY_TAGSTORE_OPTIONS = {} + +########### +# Digests # +########### + +# The digest backend powers notification summaries. + +SENTRY_DIGESTS = "sentry.digests.backends.redis.RedisBackend" + +############## +# Web Server # +############## + +SENTRY_WEB_HOST = "0.0.0.0" +SENTRY_WEB_PORT = 9000 +SENTRY_WEB_OPTIONS = { + "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), + "protocol": "uwsgi", + # This is needed to prevent https://git.io/fj7Lw + "uwsgi-socket": None, + "http-keepalive": True, + "memory-report": False, + # 'workers': 3, # the number of web workers +} + +########### +# SSL/TLS # +########### + +# If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto +# header and enable the settings below + +# SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') +# SESSION_COOKIE_SECURE = True +# CSRF_COOKIE_SECURE = True +# SOCIAL_AUTH_REDIRECT_IS_HTTPS = True + +# End of SSL/TLS settings + +############ +# Features # +############ + +SENTRY_FEATURES["projects:sample-events"] = False +SENTRY_FEATURES.update( + { + feature: True + for feature in ( + "organizations:discover", + "organizations:events", + "organizations:global-views", + "organizations:integrations-issue-basic", + "organizations:integrations-issue-sync", + "organizations:invite-members", + "organizations:new-issue-ui", + "organizations:repos", + "organizations:require-2fa", + "organizations:sentry10", + "organizations:sso-basic", + "organizations:sso-rippling", + "organizations:sso-saml2", + "organizations:suggested-commits", + "projects:custom-inbound-filters", + "projects:data-forwarding", + "projects:discard-groups", + "projects:plugins", + "projects:rate-limits", + "projects:servicehooks", + ) + } +) + +###################### +# GitHub Integration # +##################### + +# GITHUB_APP_ID = 'YOUR_GITHUB_APP_ID' +# GITHUB_API_SECRET = 'YOUR_GITHUB_API_SECRET' +# GITHUB_EXTENDED_PERMISSIONS = ['repo'] + +######################### +# Bitbucket Integration # +######################## + +# BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' +# BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET' diff --git a/test.sh b/test.sh index 688787ac89..41800796f4 100755 --- a/test.sh +++ b/test.sh @@ -1,28 +1,29 @@ #!/usr/bin/env bash set -e -TEST_USER='test@sentry.io' +SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" +TEST_USER='test@example.com' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) declare -a TEST_STRINGS=( '"isAuthenticated":true' - '"username":"test@sentry.io"' + '"username":"test@example.com"' '"isSuperuser":true' ) -INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null http://localhost:9000 -w %{url_effective}) -if [ "$INITIAL_AUTH_REDIRECT" != "http://localhost:9000/auth/login/sentry/" ]; then +INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) +if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then echo "Initial /auth/login/ redirect failed, exiting..." echo "$INITIAL_AUTH_REDIRECT" exit -1 fi -CSRF_TOKEN=$(curl http://localhost:9000 -sL -c "$COOKIE_FILE" | awk -F "'" ' +CSRF_TOKEN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' /csrfmiddlewaretoken/ { print $4 "=" $6; exit; }') -LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" http://localhost:9000/auth/login/ -H 'Referer: http://localhost/auth/login/' -b "$COOKIE_FILE" -c "$COOKIE_FILE") +LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" "$SENTRY_TEST_HOST/auth/login/" -H "Referer: $SENTRY_TEST_HOST/auth/login/" -b "$COOKIE_FILE" -c "$COOKIE_FILE") TEST_RESULT=0 for i in "${TEST_STRINGS[@]}" From 9d218472183c15296bbdd17ba968c4a8db888a1f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 25 Nov 2019 23:01:14 +0300 Subject: [PATCH 036/417] fix(snuba): Add migrate call on setup (#281) We need to run `snuba migrate` to be able to safely upgrade between Snuba versions --- install.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/install.sh b/install.sh index 4cc93c87d4..56d3b95c42 100755 --- a/install.sh +++ b/install.sh @@ -103,7 +103,10 @@ fi echo "Boostrapping Snuba..." docker-compose up -d kafka redis clickhouse until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do + # `bootstrap` is for fresh installs, and `migrate` is for existing installs + # Running them both for both cases is harmless so we blindly run them docker-compose run --rm snuba-api bootstrap --force || true; + docker-compose run --rm snuba-api migrate || true; done; echo "" From b461a7591d58e84f68642c8a44b3d8dfc0a4a966 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 26 Nov 2019 02:33:04 +0300 Subject: [PATCH 037/417] =?UTF-8?q?feat(symbolicator):=20Use=20the=20image?= =?UTF-8?q?=20from=20Docker=20Hub=20for=20symbolicat=E2=80=A6=20(#282)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 12f21cb67a..a1a65c46ff 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -113,7 +113,7 @@ services: command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: << : *restart_policy - image: us.gcr.io/sentryio/symbolicator:latest + image: 'getsentry/symbolicator:latest' volumes: - 'sentry-symbolicator:/data' command: run @@ -122,7 +122,7 @@ services: build: context: ./cron args: - BASE_IMAGE: 'us.gcr.io/sentryio/symbolicator:latest' + BASE_IMAGE: 'getsentry/symbolicator:latest' command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' web: << : *sentry_defaults From ddabbe96c650a133952d2781f5844af26943fef0 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 27 Nov 2019 03:14:09 +0300 Subject: [PATCH 038/417] feat(postgres): Add auto upgrade from pg 9.5 to 9.6 (#283) --- install.sh | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/install.sh b/install.sh index 56d3b95c42..b602ded661 100755 --- a/install.sh +++ b/install.sh @@ -55,6 +55,26 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then exit -1 fi +# Very naively check whether there's an existing sentry-postgres volume and the PG version in it +if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then + # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume + docker run --rm \ + -v sentry-postgres:/var/lib/postgresql/9.5/data \ + -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ + tianon/postgres-upgrade:9.5-to-9.6 + + # Get rid of the old volume as we'll rename the new one to that + docker volume rm sentry-postgres + docker volume create --name sentry-postgres + # There's no rename volume in Docker so copy the contents from old to new name + # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` + # doesn't do that automatically. + docker run --rm -it -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ + "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" + # Finally, remove the new old volume as we are all in sentry-postgres now + docker volume rm sentry-postgres-new +fi + echo "" ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML From 83f52d1fe54afb5093ac5324a051a004afb75bb9 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 28 Nov 2019 01:19:08 +0300 Subject: [PATCH 039/417] fix(tracing): Force-disable self-tracing on on-premise (#285) --- sentry/config.example.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 5547720a1a..5cff314677 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -67,3 +67,5 @@ system.internal-url-prefix: 'http://web:9000' symbolicator.enabled: true symbolicator.options: url: "http://symbolicator:3021" + +transaction-events.force-disable-internal-project: true From 62c0cb16bfa33635bd0fd768b386522ccd2a6941 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 4 Dec 2019 00:53:38 +0300 Subject: [PATCH 040/417] ref(kafka): Simpler kafka listener setup (#288) --- docker-compose.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index a1a65c46ff..17c94856d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -32,7 +32,7 @@ x-snuba-defaults: &snuba_defaults environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse - DEFAULT_BROKERS: 'kafka:9093' + DEFAULT_BROKERS: 'kafka:9092' REDIS_HOST: redis # TODO: Remove these after getsentry/snuba#353 UWSGI_MAX_REQUESTS: '10000' @@ -76,10 +76,7 @@ services: image: 'confluentinc/cp-kafka:5.1.2' environment: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' - KAFKA_LISTENERS: 'INTERNAL://0.0.0.0:9093,EXTERNAL://0.0.0.0:9092' - KAFKA_ADVERTISED_LISTENERS: 'INTERNAL://kafka:9093,EXTERNAL://kafka:9092' - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: 'INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT' - KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN' KAFKA_LOG4J_ROOT_LOGLEVEL: 'ERROR' From 6c0d7e48f072397c23d60743259101980461fb09 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 4 Dec 2019 01:30:24 +0300 Subject: [PATCH 041/417] fix(kafka): Disable support metrics in Kafka & ZK (#290) Disables the annoying automatic support metrics for Kafka and ZK while defaulting all logs to WARN level, instead of some being WARN and some ERROR. --- docker-compose.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 17c94856d2..fd68260667 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -65,8 +65,9 @@ services: image: 'confluentinc/cp-zookeeper:5.1.2' environment: ZOOKEEPER_CLIENT_PORT: '2181' - ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'ERROR' - ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'ERROR' + CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' + ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'WARN' + ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'WARN' volumes: - 'sentry-zookeeper:/var/lib/zookeeper' kafka: @@ -78,9 +79,10 @@ services: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN' - KAFKA_LOG4J_ROOT_LOGLEVEL: 'ERROR' - KAFKA_TOOLS_LOG4J_LOGLEVEL: 'ERROR' + KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN' + KAFKA_TOOLS_LOG4J_LOGLEVEL: 'WARN' volumes: - 'sentry-kafka:/var/lib/kafka' clickhouse: From 7b10fea284a520ca86f754c36ea4ed12ecc57858 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 4 Dec 2019 02:29:01 +0300 Subject: [PATCH 042/417] fix(volumes): Fix creation of random volumes at every run (#291) Plugs all the holes (VOLUME defs) in the images we use so they at least persist across runs/restarts for things like random logs or SMTP spool. This also fixes symbolicator_cleanup service not having access to symbolicator data. --- docker-compose.yml | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index fd68260667..b3e15a722c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -47,6 +47,9 @@ services: smtp: << : *restart_policy image: tianon/exim4 + volumes: + - 'sentry-smtp:/var/spool/exim4' + - 'sentry-smtp-log:/var/log/exim4' memcached: << : *restart_policy image: 'memcached:1.5-alpine' @@ -69,7 +72,9 @@ services: ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'WARN' ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'WARN' volumes: - - 'sentry-zookeeper:/var/lib/zookeeper' + - 'sentry-zookeeper:/var/lib/zookeeper/data' + - 'sentry-zookeeper-log:/var/lib/zookeeper/log' + - 'sentry-secrets:/etc/zookeeper/secrets' kafka: << : *restart_policy depends_on: @@ -84,7 +89,9 @@ services: KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN' KAFKA_TOOLS_LOG4J_LOGLEVEL: 'WARN' volumes: - - 'sentry-kafka:/var/lib/kafka' + - 'sentry-kafka:/var/lib/kafka/data' + - 'sentry-kafka-log:/var/lib/kafka/log' + - 'sentry-secrets:/etc/kafka/secrets' clickhouse: << : *restart_policy image: 'yandex/clickhouse-server:19.4' @@ -123,6 +130,8 @@ services: args: BASE_IMAGE: 'getsentry/symbolicator:latest' command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' + volumes: + - 'sentry-symbolicator:/data' web: << : *sentry_defaults ports: @@ -156,3 +165,8 @@ volumes: external: true sentry-symbolicator: external: true + sentry-secrets: + sentry-smtp: + sentry-zookeeper-log: + sentry-kafka-log: + sentry-smtp-log: From 89e8053c4089f2bdc8b8ae5b7c20792fbe824828 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Dec 2019 03:32:31 +0300 Subject: [PATCH 043/417] fix(data): Use `/data` for persisting files and file caches (#295) Moves sentry-data volume to /data mount point and sets all file-based storage settings to /data/files, /data/dsym-cache etc. accordingly. See https://github.com/getsentry/sentry/blob/50ac5506669043dd939dd0e44fe4cfb00377ff1d/src/sentry/options/defaults.py#L45-L54 --- docker-compose.yml | 2 +- install.sh | 4 ++++ sentry/config.example.yml | 4 +++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index b3e15a722c..9cbb331dba 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -21,7 +21,7 @@ x-sentry-defaults: &sentry_defaults environment: SNUBA: 'http://snuba-api:1218' volumes: - - 'sentry-data:/var/lib/sentry/files' + - 'sentry-data:/data' x-snuba-defaults: &snuba_defaults << : *restart_policy depends_on: diff --git a/install.sh b/install.sh index b602ded661..6700db30da 100755 --- a/install.sh +++ b/install.sh @@ -130,6 +130,10 @@ until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --que done; echo "" +echo "Migrating file storage..." +docker run --rm -it -v sentry-data:/data alpine ash -c \ + "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" + cleanup echo "" diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 5cff314677..d31cf2e5d5 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -55,7 +55,9 @@ system.secret-key: '!!changeme!!' filestore.backend: 'filesystem' filestore.options: - location: '/var/lib/sentry/files' + location: '/data/files' +dsym.cache-path: '/data/dsym-cache' +releasefile.cache-path: '/data/releasefile-cache' # filestore.backend: 's3' # filestore.options: From 4af4ecfd0e3704652cfb64667e6752fc7f4306b2 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Dec 2019 23:21:54 +0300 Subject: [PATCH 044/417] fix(install): Don't run migrations in interactive mode (#296) This should fix the GCB failures. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 6700db30da..1e41aac04e 100755 --- a/install.sh +++ b/install.sh @@ -69,7 +69,7 @@ if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm # There's no rename volume in Docker so copy the contents from old to new name # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` # doesn't do that automatically. - docker run --rm -it -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ + docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" # Finally, remove the new old volume as we are all in sentry-postgres now docker volume rm sentry-postgres-new @@ -131,7 +131,7 @@ done; echo "" echo "Migrating file storage..." -docker run --rm -it -v sentry-data:/data alpine ash -c \ +docker run --rm -v sentry-data:/data alpine ash -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" cleanup From 3c9e8c28ff52d1a0dce1603de5a514915f545adc Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 9 Dec 2019 21:18:43 +0300 Subject: [PATCH 045/417] fix(sentry-data): Don't migrate when not needed (#297) With this patch, `./install.sh` becomes safer as it won't recursively move things under `/data/file`. It also won't try to migrate an empty volume. --- install.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index 1e41aac04e..1512c11eb6 100755 --- a/install.sh +++ b/install.sh @@ -130,9 +130,12 @@ until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --que done; echo "" -echo "Migrating file storage..." -docker run --rm -v sentry-data:/data alpine ash -c \ - "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" +SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l") +if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then + echo "Migrating file storage..." + docker run --rm -v sentry-data:/data alpine ash -c \ + "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" +fi cleanup From ba67058070049e4e74de6fc1d0f4725c4ac0d339 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 9 Dec 2019 21:33:02 +0300 Subject: [PATCH 046/417] feat(upgrade): Add automatic Snuba migration for upgrades (#292) Upgrades existing events from last `$SENTRY_EVENT_RETENTION_DAYS` to Snuba automatically. Relies on getsentry/sentry#15934. --- install.sh | 72 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 41 insertions(+), 31 deletions(-) diff --git a/install.sh b/install.sh index 1512c11eb6..39349bc083 100755 --- a/install.sh +++ b/install.sh @@ -16,7 +16,7 @@ cleanup () { return 0; fi echo "Cleaning up..." - docker-compose down &> /dev/null + docker-compose stop &> /dev/null DID_CLEAN_UP=1 } trap cleanup ERR INT TERM @@ -55,30 +55,8 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then exit -1 fi -# Very naively check whether there's an existing sentry-postgres volume and the PG version in it -if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then - # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume - docker run --rm \ - -v sentry-postgres:/var/lib/postgresql/9.5/data \ - -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ - tianon/postgres-upgrade:9.5-to-9.6 - - # Get rid of the old volume as we'll rename the new one to that - docker volume rm sentry-postgres - docker volume create --name sentry-postgres - # There's no rename volume in Docker so copy the contents from old to new name - # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` - # doesn't do that automatically. - docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ - "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" - # Finally, remove the new old volume as we are all in sentry-postgres now - docker volume rm sentry-postgres-new -fi - -echo "" -ensure_file_from_example $SENTRY_CONFIG_PY -ensure_file_from_example $SENTRY_CONFIG_YML -ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS +# Ensure nothing is working while we install/update +docker-compose stop echo "" echo "Creating volumes for persistent storage..." @@ -90,6 +68,11 @@ echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." +echo "" +ensure_file_from_example $SENTRY_CONFIG_PY +ensure_file_from_example $SENTRY_CONFIG_YML +ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS + echo "" echo "Generating secret key..." # This is to escape the secret key to be used in sed below @@ -106,6 +89,26 @@ docker-compose build --force-rm echo "" echo "Docker images built." +# Very naively check whether there's an existing sentry-postgres volume and the PG version in it +if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then + # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume + docker run --rm \ + -v sentry-postgres:/var/lib/postgresql/9.5/data \ + -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ + tianon/postgres-upgrade:9.5-to-9.6 + + # Get rid of the old volume as we'll rename the new one to that + docker volume rm sentry-postgres + docker volume create --name sentry-postgres + # There's no rename volume in Docker so copy the contents from old to new name + # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` + # doesn't do that automatically. + docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ + "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" + # Finally, remove the new old volume as we are all in sentry-postgres now + docker volume rm sentry-postgres-new +fi + echo "" echo "Setting up database..." if [ $CI ]; then @@ -120,6 +123,13 @@ else docker-compose run --rm web upgrade fi +SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l") +if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then + echo "Migrating file storage..." + docker run --rm -v sentry-data:/data alpine ash -c \ + "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" +fi + echo "Boostrapping Snuba..." docker-compose up -d kafka redis clickhouse until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do @@ -130,12 +140,12 @@ until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --que done; echo "" -SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l") -if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then - echo "Migrating file storage..." - docker run --rm -v sentry-data:/data alpine ash -c \ - "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" -fi +set -o allexport +source .env +set +o allexport +echo "Migrating old events for the last $SENTRY_EVENT_RETENTION_DAYS days..." +docker-compose run --rm web django backfill_eventstream --no-input --last-days $SENTRY_EVENT_RETENTION_DAYS +echo "" cleanup From 9ec53c33502199b7dd3e28ccad5676ca539a6cc6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 9 Dec 2019 23:40:18 +0300 Subject: [PATCH 047/417] meta(readme): Update upgrade notice (#299) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 486cad4622..e73ce6d536 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). -**NOTE:** If you are not installing Sentry from scratch, visit [On-Premise Stable for Sentry 9.1.2](https://github.com/getsentry/onpremise/tree/stable) as this version is not fully backward compatible. +**NOTE:** If you are not installing Sentry from scratch, our recommendation is to visit [On-Premise Stable for Sentry 9.1.2](https://github.com/getsentry/onpremise/tree/stable) as this version may not be fully backward compatible. If you still want to try it out make sure you are on 9.1.2 first, back up your old Docker volumes just in case, and remember that if you haven't set up Redis persistency yourself some of your data (like your stats) may be lost during the upgrade. ## Requirements From a465b170be104c56f9b1ccedbfba6804021da0f2 Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Mon, 23 Dec 2019 21:20:05 +0000 Subject: [PATCH 048/417] license: Update BSL change date --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 7adada950a..ea68f54318 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2022-09-15 +Change Date: 2022-12-22 Change License: Apache License, Version 2.0 From 547bea23e6d7d088b9864498694efd0983dd6790 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 26 Dec 2019 08:26:28 +0300 Subject: [PATCH 049/417] ref(env): Set docker-compose project name (#305) --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index 2eeb2440a4..008cb10483 100644 --- a/.env +++ b/.env @@ -1 +1,2 @@ +COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 From 3af61c8f55250daec51228811ca70ce6d553e963 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 30 Dec 2019 23:07:17 +0300 Subject: [PATCH 050/417] fix(retention): Sync system.event-retention-days w/ cleanup (#308) --- sentry/sentry.conf.example.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index a0e989ce12..c6132cc978 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -29,6 +29,8 @@ # and thus various UI optimizations should be enabled. SENTRY_SINGLE_ORGANIZATION = True +SENTRY_OPTIONS["system.event-retention-days"] = env('SENTRY_EVENT_RETENTION_DAYS') or 90 + ######### # Redis # ######### From 3dfc01ec2dba1bc4945b44262c2910a1c8a24bd8 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 30 Dec 2019 23:07:42 +0300 Subject: [PATCH 051/417] fix(upgrade): Fix upgrade instructions and script (#304) --- README.md | 9 +-------- install.sh | 12 ++++++++---- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index e73ce6d536..2a0267c24a 100644 --- a/README.md +++ b/README.md @@ -41,14 +41,7 @@ and [Nginx](http://nginx.org/). You'll likely want to add this service to your ` ## Updating Sentry -Updating Sentry using Compose is relatively simple. Just use the following steps to update. Make sure that you have the latest version set in your Dockerfile. Or use the latest version of this repository. - -Use the following steps after updating this repository or your Dockerfile: -```sh -docker-compose build --pull # Build the services again after updating, and make sure we're up to date on patch version -docker-compose run --rm web upgrade # Run new migrations -docker-compose up -d # Recreate the services -``` +The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. ## Resources diff --git a/install.sh b/install.sh index 39349bc083..742d131cce 100755 --- a/install.sh +++ b/install.sh @@ -55,8 +55,8 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then exit -1 fi -# Ensure nothing is working while we install/update -docker-compose stop +# Clean up old stuff and ensure nothing is working while we install/update +docker-compose down --rmi local --remove-orphans echo "" echo "Creating volumes for persistent storage..." @@ -84,6 +84,8 @@ echo "" echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image +docker-compose pull --ignore-pull-failures +docker pull ${SENTRY_IMAGE:-getsentry/sentry:latest} docker-compose build --force-rm web docker-compose build --force-rm echo "" @@ -91,6 +93,7 @@ echo "Docker images built." # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then + docker volume rm sentry-postgres-new || true # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume docker run --rm \ -v sentry-postgres:/var/lib/postgresql/9.5/data \ @@ -123,10 +126,11 @@ else docker-compose run --rm web upgrade fi -SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l") +SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then echo "Migrating file storage..." - docker run --rm -v sentry-data:/data alpine ash -c \ + # Use the web (Sentry) image so the file owners are kept as sentry:sentry + docker-compose run --rm --entrypoint /bin/bash web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" fi From aa79f8baf9478be1340e6ef27cbb716969f303ff Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 30 Dec 2019 23:27:02 +0300 Subject: [PATCH 052/417] fix(post-process): Add missing post-process-forwarder (#309) We were not running the post-process forwarder, causing post-process to not run which covers all plugin and rule/alert work. Fixes #287. --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9cbb331dba..c8ada85cb9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -142,6 +142,10 @@ services: worker: << : *sentry_defaults command: run worker + post-process-forwarder: + << : *sentry_defaults + # Increase `--commit-batch-size 1` below to deal with high-load environments. + command: run post-process-forwarder --commit-batch-size 1 sentry-cleanup: << : *sentry_defaults image: sentry-cleanup-onpremise-local From 629ee07c79e544e67db60f8086a8109b96d90b67 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 3 Jan 2020 13:17:35 +0300 Subject: [PATCH 053/417] feat(install): Add automatic logs to install script (#312) This change makes the install script create a new install log file, `sentry_install_log-.txt`, for each run and records all the output there for future reference. --- .gitignore | 1 + README.md | 4 ++-- install.sh | 4 ++++ 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.gitignore b/.gitignore index 802aab9241..77b24eb843 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ var/ # Installer logs pip-log.txt pip-delete-this-directory.txt +sentry_install_log*.txt # Unit test / coverage reports htmlcov/ diff --git a/README.md b/README.md index 2a0267c24a..f74fbaad1c 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ The recommended way to customize your configuration is using the files below, in We currently support a very minimal set of environment variables to promote other means of configuration. -If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! +If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! Everytime you run the install script, it will generate a log file, `sentry_install_log-.txt` with the output. Sharing these logs would help people diagnose any issues you might be having. ## Event Retention @@ -41,7 +41,7 @@ and [Nginx](http://nginx.org/). You'll likely want to add this service to your ` ## Updating Sentry -The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. +The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. ## Resources diff --git a/install.sh b/install.sh index 742d131cce..9d6e32ff94 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,10 @@ #!/usr/bin/env bash set -e +# Thanks to https://unix.stackexchange.com/a/145654/108960 +log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" +exec &> >(tee -a "$log_file") + MIN_DOCKER_VERSION='17.05.0' MIN_COMPOSE_VERSION='1.19.0' MIN_RAM=2400 # MB From 4a82e9fa1d273c6cccae27de3134bb38b7069aa3 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 3 Jan 2020 22:06:33 +0300 Subject: [PATCH 054/417] feat(logs): Cleaner logs w/o ANSI sequences (#316) --- install.sh | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/install.sh b/install.sh index 9d6e32ff94..2caaa4ccfc 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,9 @@ #!/usr/bin/env bash set -e +dc="docker-compose --no-ansi" +dcr="$dc run --rm" + # Thanks to https://unix.stackexchange.com/a/145654/108960 log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") @@ -20,7 +23,7 @@ cleanup () { return 0; fi echo "Cleaning up..." - docker-compose stop &> /dev/null + $dc stop &> /dev/null DID_CLEAN_UP=1 } trap cleanup ERR INT TERM @@ -28,7 +31,7 @@ trap cleanup ERR INT TERM echo "Checking minimum requirements..." DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') -COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') +COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 @@ -60,7 +63,10 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then fi # Clean up old stuff and ensure nothing is working while we install/update -docker-compose down --rmi local --remove-orphans +# This is for older versions of on-premise: +$dc -p onpremise down --rmi local --remove-orphans +# This is for newer versions +$dc down --rmi local --remove-orphans echo "" echo "Creating volumes for persistent storage..." @@ -88,10 +94,10 @@ echo "" echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image -docker-compose pull --ignore-pull-failures +$dc pull --ignore-pull-failures docker pull ${SENTRY_IMAGE:-getsentry/sentry:latest} -docker-compose build --force-rm web -docker-compose build --force-rm +$dc build --force-rm web +$dc build --force-rm echo "" echo "Docker images built." @@ -119,7 +125,7 @@ fi echo "" echo "Setting up database..." if [ $CI ]; then - docker-compose run --rm web upgrade --noinput + $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." echo "Run the following command to create one yourself (recommended):" @@ -127,24 +133,25 @@ if [ $CI ]; then echo " docker-compose run --rm web createuser" echo "" else - docker-compose run --rm web upgrade + $dcr web upgrade fi + SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then echo "Migrating file storage..." # Use the web (Sentry) image so the file owners are kept as sentry:sentry - docker-compose run --rm --entrypoint /bin/bash web -c \ + $dcr --entrypoint /bin/bash web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" fi echo "Boostrapping Snuba..." -docker-compose up -d kafka redis clickhouse -until $(docker-compose run --rm clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do +$dc up -d kafka redis clickhouse +until $($dcr clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do # `bootstrap` is for fresh installs, and `migrate` is for existing installs # Running them both for both cases is harmless so we blindly run them - docker-compose run --rm snuba-api bootstrap --force || true; - docker-compose run --rm snuba-api migrate || true; + $dcr snuba-api bootstrap --force || true; + $dcr snuba-api migrate || true; done; echo "" @@ -152,7 +159,7 @@ set -o allexport source .env set +o allexport echo "Migrating old events for the last $SENTRY_EVENT_RETENTION_DAYS days..." -docker-compose run --rm web django backfill_eventstream --no-input --last-days $SENTRY_EVENT_RETENTION_DAYS +$dcr web django backfill_eventstream --no-input --last-days $SENTRY_EVENT_RETENTION_DAYS echo "" cleanup From 1d33ae6b04a6270d41e59cdc05163c2caca70794 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 3 Jan 2020 22:52:22 +0300 Subject: [PATCH 055/417] fix(migration): Move Snuba bootstrapping before DB upgrade (#317) Since we are [moving the eventstream into a proper db migration](https://github.com/getsentry/sentry/pull/16226), we need Snuba and its friends ready at the time of `sentry upgrade` command. This patch does exactly that. Co-authored-by: Lyn Nagara --- install.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/install.sh b/install.sh index 2caaa4ccfc..250ef22a2f 100755 --- a/install.sh +++ b/install.sh @@ -101,6 +101,16 @@ $dc build --force-rm echo "" echo "Docker images built." +echo "Bootstrapping Snuba..." +$dc up -d kafka redis clickhouse +until $($dcr clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do + # `bootstrap` is for fresh installs, and `migrate` is for existing installs + # Running them both for both cases is harmless so we blindly run them + $dcr snuba-api bootstrap --force || true; + $dcr snuba-api migrate || true; +done; +echo "" + # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then docker volume rm sentry-postgres-new || true @@ -145,16 +155,6 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" fi -echo "Boostrapping Snuba..." -$dc up -d kafka redis clickhouse -until $($dcr clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do - # `bootstrap` is for fresh installs, and `migrate` is for existing installs - # Running them both for both cases is harmless so we blindly run them - $dcr snuba-api bootstrap --force || true; - $dcr snuba-api migrate || true; -done; -echo "" - set -o allexport source .env set +o allexport From 753683d3d64c8b0dd2b2c78573023e9c1b726171 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 3 Jan 2020 23:28:43 +0300 Subject: [PATCH 056/417] ref(install): Remove manual eventstream backfill (#318) This became obsolete thanks to getsentry/sentry#16226 --- install.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/install.sh b/install.sh index 250ef22a2f..a3cbfd9eef 100755 --- a/install.sh +++ b/install.sh @@ -155,13 +155,6 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" fi -set -o allexport -source .env -set +o allexport -echo "Migrating old events for the last $SENTRY_EVENT_RETENTION_DAYS days..." -$dcr web django backfill_eventstream --no-input --last-days $SENTRY_EVENT_RETENTION_DAYS -echo "" - cleanup echo "" From 6587df32694169aa0a7df4b605452556374c19f6 Mon Sep 17 00:00:00 2001 From: Chris Fuller Date: Tue, 7 Jan 2020 11:51:30 -0500 Subject: [PATCH 057/417] Changing search class name reference (#319) Currently in sentry, both SnubaSearchBackend and EventsDatasetSnubaSearchBackend exist, and they are the same. This just points to the new (renamed) class so we can delete SnubaSearchBackend from sentry, as we only left it in to transition getsentry & onpremise. --- sentry/sentry.conf.example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index c6132cc978..b2aedcf4f1 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -134,7 +134,7 @@ # SNUBA # ######### -SENTRY_SEARCH = "sentry.search.snuba.SnubaSearchBackend" +SENTRY_SEARCH = "sentry.search.snuba.EventsDatasetSnubaSearchBackend" SENTRY_SEARCH_OPTIONS = {} SENTRY_TAGSTORE_OPTIONS = {} From 6d80e5a68d14a330a124ad90b3e6fd199548e225 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 7 Jan 2020 20:21:45 +0300 Subject: [PATCH 058/417] license(date): Update license change date to 2023-01-07 (#320) --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ea68f54318..955263a0bc 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2022-12-22 +Change Date: 2023-01-07 Change License: Apache License, Version 2.0 From b2080c14467fd44d10c3faffc7efc80a9d98c96d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 7 Jan 2020 20:44:20 +0300 Subject: [PATCH 059/417] meta(readme): Remove stable 9.1.2 notice (#321) --- README.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/README.md b/README.md index f74fbaad1c..76c0e18efe 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,7 @@ -# Sentry 10 On-Premise BETA [![Build Status][build-status-image]][build-status-url] +# Sentry 10 On-Premise [![Build Status][build-status-image]][build-status-url] Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). -**NOTE:** If you are not installing Sentry from scratch, our recommendation is to visit [On-Premise Stable for Sentry 9.1.2](https://github.com/getsentry/onpremise/tree/stable) as this version may not be fully backward compatible. If you still want to try it out make sure you are on 9.1.2 first, back up your old Docker volumes just in case, and remember that if you haven't set up Redis persistency yourself some of your data (like your stats) may be lost during the upgrade. - ## Requirements * Docker 17.05.0+ From d54c1a8d6c0315e96e57fac10622fbef747bc739 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 9 Jan 2020 16:13:18 +0300 Subject: [PATCH 060/417] fix(Makefile): Remove the obsolete Makefile (#333) rip --- Makefile | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 Makefile diff --git a/Makefile b/Makefile deleted file mode 100644 index 0ab11e2875..0000000000 --- a/Makefile +++ /dev/null @@ -1,21 +0,0 @@ -REPOSITORY?=sentry-onpremise -TAG?=latest - -OK_COLOR=\033[32;01m -NO_COLOR=\033[0m - -build: - @printf "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)\n" - @docker build --pull --rm -t $(REPOSITORY):$(TAG) . --build-arg SENTRY_IMAGE=sentry:9.1 - -$(REPOSITORY)_$(TAG).tar: build - @printf "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@\n" - @docker save $(REPOSITORY):$(TAG) > $@ - -push: build - @printf "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)\n" - @docker push $(REPOSITORY):$(TAG) - -all: build push - -.PHONY: all build push From bc234481414062294f1fbdb9c0573555bf1b34b0 Mon Sep 17 00:00:00 2001 From: Simon Golms Date: Thu, 9 Jan 2020 20:55:20 +0100 Subject: [PATCH 061/417] fix(install): support for Windows and Git Bash (#335) Fixes #329. --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index a3cbfd9eef..95fd72897c 100755 --- a/install.sh +++ b/install.sh @@ -151,7 +151,8 @@ SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then echo "Migrating file storage..." # Use the web (Sentry) image so the file owners are kept as sentry:sentry - $dcr --entrypoint /bin/bash web -c \ + # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. + $dcr --entrypoint \"/bin/bash\" web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" fi From d6951a2c325583bbf45861ad059534ee1d679796 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 10 Jan 2020 12:34:36 +0300 Subject: [PATCH 062/417] fix(install): Fix secret key gen on BSD (#338) Fixes #330. --- install.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 95fd72897c..dd040aab18 100755 --- a/install.sh +++ b/install.sh @@ -86,7 +86,9 @@ ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS echo "" echo "Generating secret key..." # This is to escape the secret key to be used in sed below -SECRET_KEY=$(head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') +# Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode +# whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 +SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML echo "Secret key written to $SENTRY_CONFIG_YML" From a655b4799c935a7a3a7c9a7299cebb72a0fcb4cb Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jan 2020 18:53:54 +0300 Subject: [PATCH 063/417] feat(build): Enable parallel builds for docker-compose (#343) --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index dd040aab18..eb05ca2a1a 100755 --- a/install.sh +++ b/install.sh @@ -99,7 +99,7 @@ echo "" $dc pull --ignore-pull-failures docker pull ${SENTRY_IMAGE:-getsentry/sentry:latest} $dc build --force-rm web -$dc build --force-rm +$dc build --force-rm --parallel echo "" echo "Docker images built." From 70674e252acc867d125e8f7fbdccde287fb361a2 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 13 Jan 2020 22:25:27 +0300 Subject: [PATCH 064/417] fix(data-migration): Make sure to chown the /data folder on sentry (#336) Fixes #334. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index eb05ca2a1a..0b789a5771 100755 --- a/install.sh +++ b/install.sh @@ -155,7 +155,7 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then # Use the web (Sentry) image so the file owners are kept as sentry:sentry # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. $dcr --entrypoint \"/bin/bash\" web -c \ - "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files" + "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi cleanup From dffd59723c05eb7a8a13f441a6cbb8b4e85fc437 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 13 Jan 2020 22:28:19 +0300 Subject: [PATCH 065/417] meta(readme): Update docs and chat links in readme (#346) Fixes #345. --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 76c0e18efe..e2421cd4c5 100644 --- a/README.md +++ b/README.md @@ -43,10 +43,10 @@ The included `install.sh` script is meant to be idempotent and to bring you to t ## Resources - * [Documentation](https://docs.sentry.io/server/installation/docker/) + * [Documentation](https://docs.sentry.io/server/) * [Bug Tracker](https://github.com/getsentry/onpremise/issues) * [Forums](https://forum.sentry.io/c/on-premise) - * [IRC](irc://chat.freenode.net/sentry) (chat.freenode.net, #sentry) + * [Discord](https://discord.gg/mg5V76F) (Sentry Community, #sentry-server) [build-status-image]: https://api.travis-ci.com/getsentry/onpremise.svg?branch=master From b8405fca760e183c81a5cdfe5429a76a4328cc2b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 21 Jan 2020 22:56:04 +0300 Subject: [PATCH 066/417] build(compose): Bump required compose version (#353) With #343, we added the `--parallel` flag which only got introduced in `docker-compose` `1.23.0` (source https://medium.com/schkn/parallelize-your-docker-compose-build-8ac653e3e596 as Docker docs themselves don't really mention these) so bumping the minimum required version. Fixes #351. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 0b789a5771..01d8a34f79 100755 --- a/install.sh +++ b/install.sh @@ -9,7 +9,7 @@ log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") MIN_DOCKER_VERSION='17.05.0' -MIN_COMPOSE_VERSION='1.19.0' +MIN_COMPOSE_VERSION='1.23.0' MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' From 1292a125b867e38c5f662ed1224c8731934c9240 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 21 Jan 2020 22:56:45 +0300 Subject: [PATCH 067/417] =?UTF-8?q?fix(install):=20Remove=20unnecessary=20?= =?UTF-8?q?infinite=20loop=20from=20snuba=20bootst=E2=80=A6=20(#354)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With getsentry/snuba#709 merged, we no longer need to randomly try forever creating Kafka topics, which were a prerequisite for creating Clickhouse tables. This patch also removes the preceding (and obsolete) `docker-compose up` statement which was used to "speed up" the `snuba bootstrap` command in the hopes to fix it. --- install.sh | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/install.sh b/install.sh index 01d8a34f79..de1f6adb91 100755 --- a/install.sh +++ b/install.sh @@ -104,13 +104,10 @@ echo "" echo "Docker images built." echo "Bootstrapping Snuba..." -$dc up -d kafka redis clickhouse -until $($dcr clickhouse clickhouse-client -h clickhouse --query="SHOW TABLES;" | grep -q sentry_local); do - # `bootstrap` is for fresh installs, and `migrate` is for existing installs - # Running them both for both cases is harmless so we blindly run them - $dcr snuba-api bootstrap --force || true; - $dcr snuba-api migrate || true; -done; +# `bootstrap` is for fresh installs, and `migrate` is for existing installs +# Running them both for both cases is harmless so we blindly run them +$dcr snuba-api bootstrap --force +$dcr snuba-api migrate echo "" # Very naively check whether there's an existing sentry-postgres volume and the PG version in it From dbcf4cc5288ffd64b986371717d1b4090bea6800 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 23 Jan 2020 22:13:28 +0300 Subject: [PATCH 068/417] license: Update BSL change date * license: Update BSL change date * try being too protective in the future * set the date properly --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 955263a0bc..97c0e8f080 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-01-07 +Change Date: 2023-01-15 Change License: Apache License, Version 2.0 From 1748073324f770ebef498a7174067981332c0b59 Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Thu, 23 Jan 2020 19:36:19 +0000 Subject: [PATCH 069/417] license: Update BSL change date --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 97c0e8f080..784bd05879 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-01-15 +Change Date: 2023-01-23 Change License: Apache License, Version 2.0 From 73bf3f5ab440a450f7a18dce45c803521ca4eaff Mon Sep 17 00:00:00 2001 From: Simon Golms Date: Fri, 24 Jan 2020 19:04:39 +0100 Subject: [PATCH 070/417] fix(config): parse env variable as int (#359) - parse SENTRY_EVENT_RETENTION_DAYS as integer in case it is defined as a string like in kubernetes --- sentry/sentry.conf.example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index b2aedcf4f1..a11cb41985 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -29,7 +29,7 @@ # and thus various UI optimizations should be enabled. SENTRY_SINGLE_ORGANIZATION = True -SENTRY_OPTIONS["system.event-retention-days"] = env('SENTRY_EVENT_RETENTION_DAYS') or 90 +SENTRY_OPTIONS["system.event-retention-days"] = int(env('SENTRY_EVENT_RETENTION_DAYS', '90')) ######### # Redis # From bc6d3b47e257057587e29153947c1ba223160416 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 24 Jan 2020 21:08:41 +0300 Subject: [PATCH 071/417] feat(config): Have better GitHub config examples (#327) Better defaults and guidance regarding GitHub app settings after getsentry/sentry#16550. --- sentry/config.example.yml | 18 ++++++++++++++++++ sentry/sentry.conf.example.py | 6 ++---- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index d31cf2e5d5..ad237d9c04 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -71,3 +71,21 @@ symbolicator.options: url: "http://symbolicator:3021" transaction-events.force-disable-internal-project: true + +###################### +# GitHub Integration # +###################### + +# github-app.id: GITHUB_APP_ID +# github-app.name: 'GITHUB_APP_NAME' +# github-app.webhook-secret: 'GITHUB_WEBHOOK_SECRET' # Use only if configured in GitHub +# github-app.client-id: 'GITHUB_CLIENT_ID' +# github-app.client-secret: 'GITHUB_CLIENT_SECRET' +# github-app.private-key: | +# -----BEGIN RSA PRIVATE KEY----- +# privatekeyprivatekeyprivatekeyprivatekey +# privatekeyprivatekeyprivatekeyprivatekey +# privatekeyprivatekeyprivatekeyprivatekey +# privatekeyprivatekeyprivatekeyprivatekey +# privatekeyprivatekeyprivatekeyprivatekey +# -----END RSA PRIVATE KEY----- diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index a11cb41985..81dd29a53e 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -211,11 +211,9 @@ ###################### # GitHub Integration # -##################### +###################### -# GITHUB_APP_ID = 'YOUR_GITHUB_APP_ID' -# GITHUB_API_SECRET = 'YOUR_GITHUB_API_SECRET' -# GITHUB_EXTENDED_PERMISSIONS = ['repo'] +GITHUB_EXTENDED_PERMISSIONS = ['repo'] ######################### # Bitbucket Integration # From cd1f9e811eee5304a15c5612e1ea30ca2178394f Mon Sep 17 00:00:00 2001 From: Amphaal Date: Wed, 29 Jan 2020 14:29:38 +0100 Subject: [PATCH 072/417] Check SSE4.2 compatibility before installation (#361) Fixes #358. --- install.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/install.sh b/install.sh index de1f6adb91..3b04f45417 100755 --- a/install.sh +++ b/install.sh @@ -62,6 +62,13 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then exit -1 fi +#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) +SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo); +if (($SUPPORTS_SSE42 == 0)); then + echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." + exit -1 +fi + # Clean up old stuff and ensure nothing is working while we install/update # This is for older versions of on-premise: $dc -p onpremise down --rmi local --remove-orphans From 37d6166032c6acb1601b9ee4f7b2b3ddff0db7eb Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 30 Jan 2020 22:54:12 +0300 Subject: [PATCH 073/417] build(clickhouse): Bump clickhouse version to match Snuba (#371) --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c8ada85cb9..5888f34132 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -94,7 +94,7 @@ services: - 'sentry-secrets:/etc/kafka/secrets' clickhouse: << : *restart_policy - image: 'yandex/clickhouse-server:19.4' + image: 'yandex/clickhouse-server:19.11' ulimits: nofile: soft: 262144 From fd9bd107df57637f6901e14716ea62a63ba4cd3b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 3 Feb 2020 19:15:40 +0300 Subject: [PATCH 074/417] fix(config): Enable http-chunked-input for UWSGI (#376) Fixes getsentry/sentry#16596 --- sentry/sentry.conf.example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 81dd29a53e..2ea883cebc 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -158,6 +158,7 @@ # This is needed to prevent https://git.io/fj7Lw "uwsgi-socket": None, "http-keepalive": True, + "http-chunked-input": True, "memory-report": False, # 'workers': 3, # the number of web workers } From ce5834b8cf6e17930717f86d7526a90674c543e8 Mon Sep 17 00:00:00 2001 From: Vernon Hockney <30329026+fvhockney@users.noreply.github.com> Date: Mon, 3 Feb 2020 19:01:12 +0100 Subject: [PATCH 075/417] fix(install): Update exit status and fix premature exit (#375) Update exit -1 to exit 1 in line with best practices Fixes premature exit of the script when checking for sse42 support. The script would exit before displaying the reason for failure to the user. --- install.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/install.sh b/install.sh index 3b04f45417..9b22969f26 100755 --- a/install.sh +++ b/install.sh @@ -49,24 +49,24 @@ function ensure_file_from_example { if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" - exit -1 + exit 1 fi if [ $(ver $COMPOSE_VERSION) -lt $(ver $MIN_COMPOSE_VERSION) ]; then echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" - exit -1 + exit 1 fi if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" - exit -1 + exit 1 fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) -SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo); +SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :); if (($SUPPORTS_SSE42 == 0)); then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." - exit -1 + exit 1 fi # Clean up old stuff and ensure nothing is working while we install/update From 91cce7afa6bb14e5816f1615b5d39e4864572ed9 Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Mon, 3 Feb 2020 18:39:08 +0000 Subject: [PATCH 076/417] license: Update BSL change date Co-authored-by: Burak Yigit Kaya --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 784bd05879..6e927783d2 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-01-23 +Change Date: 2023-02-01 Change License: Apache License, Version 2.0 From 6538f472888bf0f8796faea649a9b32ca80ba959 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 5 Feb 2020 23:59:15 +0300 Subject: [PATCH 077/417] docs(upgrade): Add note regarding min Sentry 9.1.2 requirement (#380) Fixes #325. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index e2421cd4c5..a6dfff8cb2 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,8 @@ and [Nginx](http://nginx.org/). You'll likely want to add this service to your ` ## Updating Sentry +_You need to be on at least Sentry 9.1.2 to be able to upgrade automatically to the latest version. If you are not, upgrade to 9.1.2 first by checking out the [9.1.2 tag](https://github.com/getsentry/onpremise/tree/9.1.2) on this repo._ + The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. ## Resources From 6979959a710e74038472b4380bb5cc447e853690 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 15 Feb 2020 01:39:49 +0300 Subject: [PATCH 078/417] fix(postgres): Fix postgres suddenly failing after their 'patch' release See https://github.com/docker-library/postgres/pull/658#pullrequestreview-336007842 --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5888f34132..2b691419ee 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -61,6 +61,8 @@ services: postgres: << : *restart_policy image: 'postgres:9.6' + environment: + POSTGRES_HOST_AUTH_METHOD: 'trust' volumes: - 'sentry-postgres:/var/lib/postgresql/data' zookeeper: From 9dfc5c99defd01b7978a5988657581e7c668461b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 15 Feb 2020 01:59:25 +0300 Subject: [PATCH 079/417] docs(versioning): Add section about versioning and SENTRY_IMAGE (#381) This addreses multiple questions raised about how to install a specific version of Sentry or using specific versions for images such as #378, #355, #348, and #326. --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index a6dfff8cb2..227c95f516 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,24 @@ We currently support a very minimal set of environment variables to promote othe If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! Everytime you run the install script, it will generate a log file, `sentry_install_log-.txt` with the output. Sharing these logs would help people diagnose any issues you might be having. +## Versioning + +We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:lastest`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: + +```shell +SENTRY_IMAGE=getsentry/sentry:10 ./install.sh +``` + +or + +```shell +SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh +``` + +If you want to use different or specific images for other services, you may create a `docker-compose.overrides.yaml` file in the repo and override the `image` field for the corresponding services. + +We strongly recommend keeping the `latest` tags for all, if you are using this repository directly. We also recommend using specific commit tags if you are consuming any of our Docker images in an environment that needs consistent deploys such as [a Helm chart](https://github.com/helm/charts/tree/master/stable/sentry). + ## Event Retention Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. From 3e0ca482c21596d2667e72f6131a097e41ec7252 Mon Sep 17 00:00:00 2001 From: Rob Nieuwenhuizen Date: Tue, 25 Feb 2020 14:08:13 +0100 Subject: [PATCH 080/417] Make secret key generation idempotent (#390) Only generate secret key if not set --- install.sh | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/install.sh b/install.sh index 9b22969f26..56de70354a 100755 --- a/install.sh +++ b/install.sh @@ -90,14 +90,16 @@ ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS -echo "" -echo "Generating secret key..." -# This is to escape the secret key to be used in sed below -# Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode -# whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 -SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') -sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML -echo "Secret key written to $SENTRY_CONFIG_YML" +if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then + echo "" + echo "Generating secret key..." + # This is to escape the secret key to be used in sed below + # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode + # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 + SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') + sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML + echo "Secret key written to $SENTRY_CONFIG_YML" +fi echo "" echo "Building and tagging Docker images..." From 1ee602110aa4ecf48014240a46c7e68b662fafa8 Mon Sep 17 00:00:00 2001 From: AlexanderLevchenkoTechs Date: Wed, 26 Feb 2020 18:43:33 +0200 Subject: [PATCH 081/417] docs(versioning): Fix Docker image tag name (#393) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 227c95f516..87cde9fcdd 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ If you have any issues or questions, our [Community Forum](https://forum.sentry. ## Versioning -We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:lastest`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: +We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:latest`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:10 ./install.sh From f2876131fcff4c9b5550419d7e348b698f077a92 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 28 Feb 2020 17:49:31 +0300 Subject: [PATCH 082/417] feat(discover): Enable Discover v2 (#398) Also removes some obsolete feature switches. --- sentry/sentry.conf.example.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 2ea883cebc..a9fe034dd8 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -188,18 +188,16 @@ for feature in ( "organizations:discover", "organizations:events", + "organizations:discover-basic", + "organizations:discover-query", + "organizations:events-v2", "organizations:global-views", "organizations:integrations-issue-basic", "organizations:integrations-issue-sync", "organizations:invite-members", - "organizations:new-issue-ui", - "organizations:repos", - "organizations:require-2fa", - "organizations:sentry10", "organizations:sso-basic", "organizations:sso-rippling", "organizations:sso-saml2", - "organizations:suggested-commits", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups", From 2ff2f2ec12e214a741e8df1c395b96f1501c5e7b Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Mon, 2 Mar 2020 20:43:29 +0300 Subject: [PATCH 083/417] license: Update BSL change date (#399) --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 6e927783d2..274ed3ea6a 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-02-01 +Change Date: 2023-03-01 Change License: Apache License, Version 2.0 From 335016ce2493b9bb8de0bf1d65e127f7c1dfead7 Mon Sep 17 00:00:00 2001 From: xaver Date: Wed, 11 Mar 2020 15:24:57 +0100 Subject: [PATCH 084/417] Require docker compose version in README (#405) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 87cde9fcdd..9b2112dfe4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements * Docker 17.05.0+ - * Compose 1.19.0+ + * Compose 1.23.0+ ## Minimum Hardware Requirements: From 3e8ed1a6805eef6b2730cb8ee3b7ebe2dc388d24 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 12 Mar 2020 17:40:02 +0300 Subject: [PATCH 085/417] feat(snuba): Remove redundant UWSGI settings (#408) These became obsolete after getsentry/snuba#825 --- docker-compose.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2b691419ee..4a87fdcac2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -34,15 +34,8 @@ x-snuba-defaults: &snuba_defaults CLICKHOUSE_HOST: clickhouse DEFAULT_BROKERS: 'kafka:9092' REDIS_HOST: redis - # TODO: Remove these after getsentry/snuba#353 UWSGI_MAX_REQUESTS: '10000' UWSGI_DISABLE_LOGGING: 'true' - UWSGI_ENABLE_THREADS: 'true' - UWSGI_DIE_ON_TERM: 'true' - UWSGI_NEED_APP: 'true' - UWSGI_IGNORE_SIGPIPE: 'true' - UWSGI_IGNORE_WRITE_ERRORS: 'true' - UWSGI_DISABLE_WRITE_EXCEPTION: 'true' services: smtp: << : *restart_policy From c95eb5646778a789ebfc960470dcdcd33ab3ccdc Mon Sep 17 00:00:00 2001 From: ffauvel Date: Thu, 12 Mar 2020 10:41:27 -0400 Subject: [PATCH 086/417] Add missing restart policy unless-stopped (#409) Add missing restart policy unless-stopped for symbolicator-cleanup service --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4a87fdcac2..19f25ebf19 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,6 +119,7 @@ services: - 'sentry-symbolicator:/data' command: run symbolicator-cleanup: + << : *restart_policy image: symbolicator-cleanup-onpremise-local build: context: ./cron From 5f7c18bd187c649048de53c75a2fac56a4c6a5da Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 12 Mar 2020 20:14:46 +0300 Subject: [PATCH 087/417] feat(config): Mount config as a volume to Sentry (#407) This follows the best-practice of mounting the config folder as a volume and removes the need to rebuild sentry images for config changes. Partially addresses #314. --- docker-compose.yml | 2 ++ sentry/.dockerignore | 5 +++++ sentry/Dockerfile | 11 ----------- 3 files changed, 7 insertions(+), 11 deletions(-) create mode 100644 sentry/.dockerignore diff --git a/docker-compose.yml b/docker-compose.yml index 19f25ebf19..ccf134600a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,9 +19,11 @@ x-sentry-defaults: &sentry_defaults - symbolicator - kafka environment: + SENTRY_CONF: '/etc/sentry' SNUBA: 'http://snuba-api:1218' volumes: - 'sentry-data:/data' + - './sentry:/etc/sentry' x-snuba-defaults: &snuba_defaults << : *restart_policy depends_on: diff --git a/sentry/.dockerignore b/sentry/.dockerignore new file mode 100644 index 0000000000..693a7e0716 --- /dev/null +++ b/sentry/.dockerignore @@ -0,0 +1,5 @@ +# Ignore everything +* + +# Only allow requirements.txt +!/requirements.txt diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 032f340847..bf8f198ec6 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,18 +1,7 @@ ARG SENTRY_IMAGE FROM ${SENTRY_IMAGE:-getsentry/sentry:latest} -WORKDIR /usr/src/sentry - -# Add WORKDIR to PYTHONPATH so local python files don't need to be installed -ENV PYTHONPATH /usr/src/sentry COPY . /usr/src/sentry # Hook for installing additional plugins RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi - -# Hook for installing a local app as an addon -RUN if [ -s setup.py ]; then pip install -e .; fi - -# Hook for staging in custom configs -RUN if [ -s sentry.conf.py ]; then cp sentry.conf.py $SENTRY_CONF/; fi \ - && if [ -s config.yml ]; then cp config.yml $SENTRY_CONF/; fi From a7c5a32e36282f513e6dd5f9461f50114cb0597d Mon Sep 17 00:00:00 2001 From: Joe Adams Date: Mon, 16 Mar 2020 14:08:51 -0400 Subject: [PATCH 088/417] Fix regression from #407 for requirements install (#411) Changing the sentry/Dockerfile to remove the WORKDIR caused the RUN install requirements to use the wrong directory as context. --- sentry/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/Dockerfile b/sentry/Dockerfile index bf8f198ec6..88bb0761eb 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -4,4 +4,4 @@ FROM ${SENTRY_IMAGE:-getsentry/sentry:latest} COPY . /usr/src/sentry # Hook for installing additional plugins -RUN if [ -s requirements.txt ]; then pip install -r requirements.txt; fi +RUN if [ -s /usr/src/sentry/requirements.txt ]; then pip install -r /usr/src/sentry/requirements.txt; fi From d0ba529401ec3b533a82e5471126a42334d24998 Mon Sep 17 00:00:00 2001 From: ahmadali shafiee Date: Mon, 30 Mar 2020 22:12:41 +0430 Subject: [PATCH 089/417] add volume for clickhouse logs (#414) --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ccf134600a..a5aa2a8647 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -98,6 +98,7 @@ services: hard: 262144 volumes: - 'sentry-clickhouse:/var/lib/clickhouse' + - 'sentry-clickhouse-log:/var/log/clickhouse-server' snuba-api: << : *snuba_defaults snuba-consumer: @@ -172,3 +173,4 @@ volumes: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: + sentry-clickhouse-log: From 680fe082feaa3ce3a9100a298924424b9d6b971d Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Wed, 1 Apr 2020 16:59:13 +0300 Subject: [PATCH 090/417] license: Update BSL change date (#425) Co-authored-by: sentry-update-license-date[bot] <57668832+sentry-update-license-date[bot]@users.noreply.github.com> --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 274ed3ea6a..ea21db55cc 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-03-01 +Change Date: 2023-04-01 Change License: Apache License, Version 2.0 From 88991582c5a4d9ed9210ac5dcdd7efe03d4455ba Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Fri, 3 Apr 2020 15:16:26 +0200 Subject: [PATCH 091/417] fix: Add snuba outcomes consumers to setup (#426) * feat: Add snuba outcomes consumers to setup * fix: Rename all references of snuba-consumer * ref: Rename back to snuba-consumer * fix: Change auto-offset-reset * fix: Attempt to fix CI --- docker-compose.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a5aa2a8647..7f5a9f4584 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ x-sentry-defaults: &sentry_defaults - smtp - snuba-api - snuba-consumer + - snuba-outcomes-consumer - snuba-replacer - symbolicator - kafka @@ -101,9 +102,16 @@ services: - 'sentry-clickhouse-log:/var/log/clickhouse-server' snuba-api: << : *snuba_defaults + # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: << : *snuba_defaults - command: consumer --auto-offset-reset=latest --max-batch-time-ms 750 + command: consumer --dataset events --auto-offset-reset=latest --max-batch-time-ms 750 + # Kafka consumer responsible for feeding outcomes into Clickhouse + # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data + # since we did not do a proper migration + snuba-outcomes-consumer: + << : *snuba_defaults + command: consumer --dataset outcomes --auto-offset-reset=earliest --max-batch-time-ms 750 snuba-replacer: << : *snuba_defaults command: replacer --auto-offset-reset=latest --max-batch-size 3 From 3c5ab100c2b3e196434a000d8345240b50621c2b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 22 Apr 2020 18:45:10 +0300 Subject: [PATCH 092/417] docs(readme): Remove link to Discord (#438) We don't provide great support for Sentry Server over at Discord so remove the reference to it from the README --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index 9b2112dfe4..d3ad4a65d9 100644 --- a/README.md +++ b/README.md @@ -65,8 +65,7 @@ The included `install.sh` script is meant to be idempotent and to bring you to t * [Documentation](https://docs.sentry.io/server/) * [Bug Tracker](https://github.com/getsentry/onpremise/issues) - * [Forums](https://forum.sentry.io/c/on-premise) - * [Discord](https://discord.gg/mg5V76F) (Sentry Community, #sentry-server) + * [Community Forums](https://forum.sentry.io/c/on-premise) [build-status-image]: https://api.travis-ci.com/getsentry/onpremise.svg?branch=master From c20956527c5560625bb8b83fd605ee0b742a4ccf Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Wed, 22 Apr 2020 12:04:57 -0700 Subject: [PATCH 093/417] build: Update Snuba commands (#434) Update the consumer and replacer commands. These now take a storage instead of a dataset name as per https://github.com/getsentry/snuba/pull/875 and https://github.com/getsentry/snuba/pull/861. This would require `getsentry/snuba:ab2e49cc1f475e59a037d882eb1cecddd23596b9` or more recent to work. --- docker-compose.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7f5a9f4584..baaa744b20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -105,16 +105,16 @@ services: # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: << : *snuba_defaults - command: consumer --dataset events --auto-offset-reset=latest --max-batch-time-ms 750 + command: consumer --storage events --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: << : *snuba_defaults - command: consumer --dataset outcomes --auto-offset-reset=earliest --max-batch-time-ms 750 + command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 snuba-replacer: << : *snuba_defaults - command: replacer --auto-offset-reset=latest --max-batch-size 3 + command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 snuba-cleanup: << : *snuba_defaults image: snuba-cleanup-onpremise-local From b2076eaeeda51a4555314ada0d47717a449673c8 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 24 Apr 2020 15:13:38 +0300 Subject: [PATCH 094/417] ref(install): Reduce noise on docker-compose pull (#442) --- install.sh | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 56de70354a..3accea6e4e 100755 --- a/install.sh +++ b/install.sh @@ -101,12 +101,19 @@ if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then echo "Secret key written to $SENTRY_CONFIG_YML" fi +echo "" +echo "Fetching and updating Docker images..." +echo "" +# We tag locally built images with an '-onpremise-local' suffix. docker-compose pull tries to pull these too and +# shows a 404 error on the console which is confusing and unnecessary. To overcome this, we add the stderr>stdout +# redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. +$dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true +docker pull ${SENTRY_IMAGE:-getsentry/sentry:latest} + echo "" echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image -$dc pull --ignore-pull-failures -docker pull ${SENTRY_IMAGE:-getsentry/sentry:latest} $dc build --force-rm web $dc build --force-rm --parallel echo "" From e97da7c56f28532a29864876ec2fc58921691d6b Mon Sep 17 00:00:00 2001 From: Radu Woinaroski <5281987+RaduW@users.noreply.github.com> Date: Fri, 24 Apr 2020 14:31:59 +0200 Subject: [PATCH 095/417] feat(relay): Add Relay to onpremise installation (#421) Co-Authored-By: Burak Yigit Kaya --- .gitignore | 1 + docker-compose.yml | 27 ++++++++++++++++++-- install.sh | 24 ++++++++++++++++++ nginx/nginx.conf | 47 +++++++++++++++++++++++++++++++++++ relay/config.yml | 13 ++++++++++ sentry/sentry.conf.example.py | 1 + 6 files changed, 111 insertions(+), 2 deletions(-) create mode 100644 nginx/nginx.conf create mode 100644 relay/config.yml diff --git a/.gitignore b/.gitignore index 77b24eb843..3a74861bae 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,4 @@ data/ sentry/sentry.conf.py sentry/config.yml sentry/requirements.txt +relay/credentials.json diff --git a/docker-compose.yml b/docker-compose.yml index baaa744b20..ae12c1ba28 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,14 +141,15 @@ services: - 'sentry-symbolicator:/data' web: << : *sentry_defaults - ports: - - '9000:9000/tcp' cron: << : *sentry_defaults command: run cron worker: << : *sentry_defaults command: run worker + ingest-consumer: + << : *sentry_defaults + command: run ingest-consumer --all-consumer-types post-process-forwarder: << : *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. @@ -161,6 +162,28 @@ services: args: BASE_IMAGE: 'sentry-onpremise-local' command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' + nginx: + ports: + - '9000:80/tcp' + image: "nginx:1.16" + volumes: + - type: bind + read_only: true + source: ./nginx + target: /etc/nginx + depends_on: + - web + - relay + relay: + image: "us.gcr.io/sentryio/relay:latest" + command: 'run --config /etc/relay' + volumes: + - type: bind + source: ./relay + target: /etc/relay + depends_on: + - kafka + - redis volumes: sentry-data: external: true diff --git a/install.sh b/install.sh index 3accea6e4e..179a4e94b0 100755 --- a/install.sh +++ b/install.sh @@ -14,6 +14,8 @@ MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' +RELAY_CONFIG_YML='relay/config.yml' +RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' DID_CLEAN_UP=0 @@ -171,6 +173,28 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi + +if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then + echo "" + echo "Generating Relay credentials..." + + $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite + chmod a+r $RELAY_CREDENTIALS_JSON + CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") + CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" + + if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then + >&2 echo "FAIL: SENTRY_RELAY_WHITELIST_PK already exists in $SENTRY_CONFIG_PY, please replace with:" + >&2 echo "" + >&2 echo " $CREDENTIALS" + >&2 echo "" + exit 1 + fi + + echo "" >> "$SENTRY_CONFIG_PY" + echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" +fi + cleanup echo "" diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000000..20ca9a0c9b --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,47 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + sendfile on; + + keepalive_timeout 65; + + upstream relay { + server relay:3000; + } + + upstream sentry { + server web:9000; + } + + server { + listen 80; + # use the docker DNS server to resolve ips for relay and sentry containers + resolver 127.0.0.11 ipv6=off; + client_max_body_size 100M; + location /api/store/ { + proxy_pass http://relay; + } + location ~ ^/api/[1-9]\d*/ { + proxy_pass http://relay; + } + location / { + proxy_pass http://sentry; + } + } +} diff --git a/relay/config.yml b/relay/config.yml new file mode 100644 index 0000000000..9daca523c0 --- /dev/null +++ b/relay/config.yml @@ -0,0 +1,13 @@ +--- +relay: + upstream: "http://web:9000/" + host: 0.0.0.0 + port: 3000 +#logging: +# # Available logging levels: TRACE, DEBUG, INFO, WARN, ERROR +# level: WARN +processing: + enabled: true + kafka_config: + - {name: "bootstrap.servers", value: "kafka:9092"} + redis: redis://redis:6379 diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index a9fe034dd8..b640fa8310 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -220,3 +220,4 @@ # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET' + From 67ef528168918e5037ec29a587219c6a101c910b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 27 Apr 2020 17:10:22 +0300 Subject: [PATCH 096/417] ci(logs): Show docker-compose ps and logs on fail (#443) --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 8ec590d44b..761d9986d3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,3 +16,7 @@ script: - docker-compose up -d - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh + +after_failure: + - docker-compose ps + - docker-compose logs From d31f46831e5138655c942d46dc6e4efae84dcb87 Mon Sep 17 00:00:00 2001 From: Joshua Gigg Date: Mon, 27 Apr 2020 16:15:17 +0100 Subject: [PATCH 097/417] Use restart-policy for nginx & relay (#448) --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index ae12c1ba28..61aa2fdd53 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -163,6 +163,7 @@ services: BASE_IMAGE: 'sentry-onpremise-local' command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: + << : *restart_policy ports: - '9000:80/tcp' image: "nginx:1.16" @@ -175,6 +176,7 @@ services: - web - relay relay: + << : *restart_policy image: "us.gcr.io/sentryio/relay:latest" command: 'run --config /etc/relay' volumes: From ecccb211aad9702541a227a9d97bde54d5841a9d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 27 Apr 2020 20:54:59 +0300 Subject: [PATCH 098/417] fix(relay): Fix failed to write credentials (#450) --- docker-compose.yml | 4 ++-- install.sh | 23 ++++++++++++++++++----- relay/config.yml | 5 ++--- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 61aa2fdd53..0b5cf0fb6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -178,11 +178,11 @@ services: relay: << : *restart_policy image: "us.gcr.io/sentryio/relay:latest" - command: 'run --config /etc/relay' volumes: - type: bind + read_only: true source: ./relay - target: /etc/relay + target: /work/.relay depends_on: - kafka - redis diff --git a/install.sh b/install.sh index 179a4e94b0..33328c02f8 100755 --- a/install.sh +++ b/install.sh @@ -64,7 +64,7 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then exit 1 fi -#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) +#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :); if (($SUPPORTS_SSE42 == 0)); then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." @@ -178,9 +178,20 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "" echo "Generating Relay credentials..." - $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite - chmod a+r $RELAY_CREDENTIALS_JSON + # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials + # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but + # not valid JSON. + $dcr --no-deps --entrypoint /bin/bash relay -c "cp /work/.relay/config.yml /tmp/config.yml && /bin/relay --config /tmp credentials generate > /dev/null && cat /tmp/credentials.json" > "$RELAY_CREDENTIALS_JSON" CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") + if [ -z "$CREDENTIALS" ]; then + >&2 echo "FAIL: Cannot read credentials back from $RELAY_CREDENTIALS_JSON." + >&2 echo " Please ensure this file is readable and contains valid credentials." + >&2 echo "" + exit 1 + else + echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" + fi + CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then @@ -191,8 +202,10 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then exit 1 fi - echo "" >> "$SENTRY_CONFIG_PY" - echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" + echo "" >> "$SENTRY_CONFIG_PY" + echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" + echo "Relay public key written to $SENTRY_CONFIG_PY" + echo "" fi cleanup diff --git a/relay/config.yml b/relay/config.yml index 9daca523c0..8700336189 100644 --- a/relay/config.yml +++ b/relay/config.yml @@ -3,9 +3,8 @@ relay: upstream: "http://web:9000/" host: 0.0.0.0 port: 3000 -#logging: -# # Available logging levels: TRACE, DEBUG, INFO, WARN, ERROR -# level: WARN +logging: + level: WARN processing: enabled: true kafka_config: From 8c053b661af67c1616092d3d9a02f55c029eac07 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 27 Apr 2020 21:35:59 +0300 Subject: [PATCH 099/417] feat(relay): Use a simpler hack for credentials (#452) --- install.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 33328c02f8..e944da3404 100755 --- a/install.sh +++ b/install.sh @@ -180,8 +180,9 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but - # not valid JSON. - $dcr --no-deps --entrypoint /bin/bash relay -c "cp /work/.relay/config.yml /tmp/config.yml && /bin/relay --config /tmp credentials generate > /dev/null && cat /tmp/credentials.json" > "$RELAY_CREDENTIALS_JSON" + # not valid JSON. We hit this case as we redirect output to the same config folder, creating an empty + # credentials file before relay runs. + $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") if [ -z "$CREDENTIALS" ]; then >&2 echo "FAIL: Cannot read credentials back from $RELAY_CREDENTIALS_JSON." From 3244a966ec996c26f9615a8f2526b62d3afc8250 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 1 May 2020 16:23:50 +0300 Subject: [PATCH 100/417] fix(relay): Use Docker Hub as relay image registry (#462) We regularly prune old Google Cloud Build images and also GCB registry is not accessible to everyone all the time (firewall settings, being in China, etc.) Fixes #445. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0b5cf0fb6f..c728854539 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -177,7 +177,7 @@ services: - relay relay: << : *restart_policy - image: "us.gcr.io/sentryio/relay:latest" + image: "getsentry/relay:latest" volumes: - type: bind read_only: true From 6308970ea7d2fa1bff84bf3330e2385d5fe29a31 Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Fri, 1 May 2020 16:24:30 +0300 Subject: [PATCH 101/417] license: Update BSL change date (#461) --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index ea21db55cc..a6bad0ab83 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-04-01 +Change Date: 2023-05-01 Change License: Apache License, Version 2.0 From 4040e682670fec64fc4a6e862dc72e7be489392a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 2 May 2020 08:07:12 +0300 Subject: [PATCH 102/417] upgrade(clickhouse): Use 19.17 as it seems to be fixing some issues (#464) Fixes #433. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index c728854539..6584f6b059 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -92,7 +92,7 @@ services: - 'sentry-secrets:/etc/kafka/secrets' clickhouse: << : *restart_policy - image: 'yandex/clickhouse-server:19.11' + image: 'yandex/clickhouse-server:19.17' ulimits: nofile: soft: 262144 From 74c0d4c257d66dc33198deefbfb84f1fb8eccf01 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 4 May 2020 13:36:05 +0300 Subject: [PATCH 103/417] fix(nginx): Fix proxy settings for Sentry (#463) Fixes #447. This patch should also fix issuer regarding large uploads such as minidumps by turning off any buffering. --- nginx/nginx.conf | 5 ++++- sentry/sentry.conf.example.py | 11 +++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 20ca9a0c9b..c3bb9e58d7 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -18,7 +18,6 @@ http { '"$http_user_agent" "$http_x_forwarded_for"'; sendfile on; - keepalive_timeout 65; upstream relay { @@ -34,6 +33,10 @@ http { # use the docker DNS server to resolve ips for relay and sentry containers resolver 127.0.0.11 ipv6=off; client_max_body_size 100M; + + proxy_redirect off; + proxy_set_header Host $host; + location /api/store/ { proxy_pass http://relay; } diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index b640fa8310..09eb81ff94 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -29,7 +29,9 @@ # and thus various UI optimizations should be enabled. SENTRY_SINGLE_ORGANIZATION = True -SENTRY_OPTIONS["system.event-retention-days"] = int(env('SENTRY_EVENT_RETENTION_DAYS', '90')) +SENTRY_OPTIONS["system.event-retention-days"] = int( + env('SENTRY_EVENT_RETENTION_DAYS', '90') +) ######### # Redis # @@ -153,12 +155,6 @@ SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { - "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), - "protocol": "uwsgi", - # This is needed to prevent https://git.io/fj7Lw - "uwsgi-socket": None, - "http-keepalive": True, - "http-chunked-input": True, "memory-report": False, # 'workers': 3, # the number of web workers } @@ -220,4 +216,3 @@ # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET' - From 3c190eb13808da5c9ae7ddf91c11d9588c0d6e51 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 4 May 2020 21:36:04 +0300 Subject: [PATCH 104/417] upgrade(confluent): Upgrade cp-stack to 5.5.0 (#465) Suggested [on the forum](https://forum.sentry.io/t/connection-to-kafka-failed-when-installing/9162/10?u=byk) and the [upgrade docs](https://kafka.apache.org/25/documentation.html#upgrade) suggest upgrading from `5.1.x` without a rolling upgrade should be fine by just upgrading the code. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 6584f6b059..a59aa480d7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,7 +63,7 @@ services: - 'sentry-postgres:/var/lib/postgresql/data' zookeeper: << : *restart_policy - image: 'confluentinc/cp-zookeeper:5.1.2' + image: 'confluentinc/cp-zookeeper:5.5.0' environment: ZOOKEEPER_CLIENT_PORT: '2181' CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' @@ -77,7 +77,7 @@ services: << : *restart_policy depends_on: - zookeeper - image: 'confluentinc/cp-kafka:5.1.2' + image: 'confluentinc/cp-kafka:5.5.0' environment: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' From 024024b1983cf9eac44f82c3f78fba3537779850 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 4 May 2020 21:44:34 +0300 Subject: [PATCH 105/417] ref(relay): More robust Relay credentials setting (#470) This patch does two things: 1. Separate creating of Relay credentials from syncing them to Sentry config 2. Has a more flexible Relay credentials check and sync: look for the quoted public key in the config file, if it is there, assume this must be correctly set as it is very unlikely to have that random key in a different context with quotes around. The second one is to allow having other whitelisted relay keys by using an append method when adding the new key. --- install.sh | 55 +++++++++++++++++++++++------------------------------- 1 file changed, 23 insertions(+), 32 deletions(-) diff --git a/install.sh b/install.sh index e944da3404..eee22096ea 100755 --- a/install.sh +++ b/install.sh @@ -175,38 +175,29 @@ fi if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then - echo "" - echo "Generating Relay credentials..." - - # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials - # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but - # not valid JSON. We hit this case as we redirect output to the same config folder, creating an empty - # credentials file before relay runs. - $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" - CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") - if [ -z "$CREDENTIALS" ]; then - >&2 echo "FAIL: Cannot read credentials back from $RELAY_CREDENTIALS_JSON." - >&2 echo " Please ensure this file is readable and contains valid credentials." - >&2 echo "" - exit 1 - else - echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" - fi - - CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" - - if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then - >&2 echo "FAIL: SENTRY_RELAY_WHITELIST_PK already exists in $SENTRY_CONFIG_PY, please replace with:" - >&2 echo "" - >&2 echo " $CREDENTIALS" - >&2 echo "" - exit 1 - fi - - echo "" >> "$SENTRY_CONFIG_PY" - echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" - echo "Relay public key written to $SENTRY_CONFIG_PY" - echo "" + echo "" + echo "Generating Relay credentials..." + + # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials + # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but + # not valid JSON. We hit this case as we redirect output to the same config folder, creating an empty + # credentials file before relay runs. + $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" + echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" +fi + +RELAY_CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") +if [ -z "$RELAY_CREDENTIALS" ]; then + >&2 echo "FAIL: Cannot read credentials back from $RELAY_CREDENTIALS_JSON." + >&2 echo " Please ensure this file is readable and contains valid credentials." + >&2 echo "" + exit 1 +fi + +if ! grep -q "\"$RELAY_CREDENTIALS\"" "$SENTRY_CONFIG_PY"; then + echo "SENTRY_RELAY_WHITELIST_PK = (SENTRY_RELAY_WHITELIST_PK or []) + ([\"$RELAY_CREDENTIALS\"])" >> "$SENTRY_CONFIG_PY" + echo "Relay public key written to $SENTRY_CONFIG_PY" + echo "" fi cleanup From b467e1c5b5c302b2f7cc6f6cc9c2a4bc7591d68a Mon Sep 17 00:00:00 2001 From: wodry Date: Wed, 6 May 2020 16:54:00 +0200 Subject: [PATCH 106/417] docs(sentry): Fix typo in example Sentry config (#471) --- sentry/config.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index ad237d9c04..5c1c43a0fe 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -31,7 +31,7 @@ mail.host: 'smtp' # System Settings # ################### -# If this file ever becomes compromised, it's important to regenerate your a new key +# If this file ever becomes compromised, it's important to generate a new key. # Changing this value will result in all current sessions being invalidated. # A new key can be generated with `$ sentry config generate-secret-key` system.secret-key: '!!changeme!!' From ad5b76fd29f598dc728bc39b0fc5916a47ce7290 Mon Sep 17 00:00:00 2001 From: Eric Feng Date: Wed, 6 May 2020 08:11:31 -0700 Subject: [PATCH 107/417] Updating documentation link (#467) Fixes #466. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d3ad4a65d9..81525c89ba 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The included `install.sh` script is meant to be idempotent and to bring you to t ## Resources - * [Documentation](https://docs.sentry.io/server/) + * [Documentation](https://docs.sentry.io/development/server/) * [Bug Tracker](https://github.com/getsentry/onpremise/issues) * [Community Forums](https://forum.sentry.io/c/on-premise) From 45320c6e863d2a045d5d45c6c57f02de355de71b Mon Sep 17 00:00:00 2001 From: Mohamed Laradji Date: Sat, 9 May 2020 13:10:08 -0700 Subject: [PATCH 108/417] fix(install.sh): skip checking for sse4 2 flag if kvm was detected (#485) This PR disables checking for the SSE4.2 flag, which may not be present if the server is running in a VM even if SSE4.2 is in fact available to use (ClickHouse/ClickHouse#20). The KVM check was obtained from a [comment](https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297) in that issue. If SSE4.2 is not actually available in the VM, then the installation script may fail in an unpredictable way. Perhaps we can add a keyword argument to the script to let the user decide if they want to skip the check. --- install.sh | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index eee22096ea..71852202ef 100755 --- a/install.sh +++ b/install.sh @@ -65,10 +65,14 @@ if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) -SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :); -if (($SUPPORTS_SSE42 == 0)); then +# On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 +IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) +if (($IS_KVM == 0)); then + SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) + if (($SUPPORTS_SSE42 == 0)); then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." exit 1 + fi fi # Clean up old stuff and ensure nothing is working while we install/update From a74a0cb221a281bcf8ce4ee279e0f367092ae377 Mon Sep 17 00:00:00 2001 From: Mohamed Laradji Date: Sat, 9 May 2020 13:11:11 -0700 Subject: [PATCH 109/417] feat(slack-integration): add reference to guide (#484) This PR simply adds a reference to the guide for setting up the Slack integration. The url was obtained from [#249](https://github.com/getsentry/onpremise/issues/249#issuecomment-547117033). --- sentry/config.example.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 5c1c43a0fe..b75e911c54 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -89,3 +89,13 @@ transaction-events.force-disable-internal-project: true # privatekeyprivatekeyprivatekeyprivatekey # privatekeyprivatekeyprivatekeyprivatekey # -----END RSA PRIVATE KEY----- + +##################### +# Slack Integration # +##################### + +# Refer to https://forum.sentry.io/t/how-to-configure-slack-in-your-on-prem-sentry/3463 for setup instructions. + +# slack.client-id: <'client id'> +# slack.client-secret: +# slack.verification-token: \ No newline at end of file From adda25ee23b65314cc691d3a11548dbbbe64eef5 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Tue, 12 May 2020 12:02:40 +0200 Subject: [PATCH 110/417] feat: Instruct users to migrate TSDB (#430) Tested this in a Ubuntu VM. The output of `date` is not too pretty but at least localized (so D/M vs M/D is not confusing) ## What is the TSDB migration? We're effectively deprecating all TSDB backends but `sentry.tsdb.redissnuba.RedisSnubaTSDB`. We cannot reasonably support any other backend due to the fact that we would have to reimplement each of the backends in Relay, which is written in a different language. Also, like with deprecating mysql support, we don't really have the capacity to support things we do not use ourselves. ## Migration `install.sh` should rewrite your configuration automatically and define a cutover date such that no data is lost. Before the cutover date, data is written to two backends at once, Redis and Snuba, and read from one, Redis. After the cutover date, event-related metrics will be read from Snuba which matches what we have on sentry.io. ## Manual migration guide for TSDB In case `install.sh` is unable to migrate your files you will be given basic instructions on the console that essentially tell you to completely delete all TSDB config and paste the new, standard one. If for some reason you cannot say goodbye to your existing TSDB config, please create a new issue in this repo and cc @untitaker on it. --- .gitignore | 1 + install.sh | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/.gitignore b/.gitignore index 3a74861bae..5931ebe3e3 100644 --- a/.gitignore +++ b/.gitignore @@ -76,5 +76,6 @@ data/ # custom Sentry config sentry/sentry.conf.py sentry/config.yml +sentry/*.bak sentry/requirements.txt relay/credentials.json diff --git a/install.sh b/install.sh index 71852202ef..458782ea09 100755 --- a/install.sh +++ b/install.sh @@ -107,6 +107,45 @@ if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then echo "Secret key written to $SENTRY_CONFIG_YML" fi +replace_tsdb() { + if ( + [ -f "$SENTRY_CONFIG_PY" ] && + ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" + ); then + tsdb_settings="SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" + +# Automatic switchover 90 days after $(date). Can be removed afterwards. +SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}" + + if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then + echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" + else + echo "Attempting to automatically migrate to new TSDB" + # Escape newlines for sed + tsdb_settings="${tsdb_settings//$'\n'/\\n}" + cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" + sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true + + if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then + echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" + return + fi + + echo "Failed to automatically migrate TSDB. Reverting..." + mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" + echo "$SENTRY_CONFIG_PY restored from backup." + fi + + echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" + echo "" + echo "$tsdb_settings" + echo "" + echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" + fi +} + +replace_tsdb + echo "" echo "Fetching and updating Docker images..." echo "" From 29c6ef58ad96bf6a9ef41f517b60601e1c422718 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Sun, 17 May 2020 23:16:30 -0700 Subject: [PATCH 111/417] feat(snuba): Skip snuba migrate task (#495) --- install.sh | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 458782ea09..3aa3b8921b 100755 --- a/install.sh +++ b/install.sh @@ -164,11 +164,8 @@ $dc build --force-rm --parallel echo "" echo "Docker images built." -echo "Bootstrapping Snuba..." -# `bootstrap` is for fresh installs, and `migrate` is for existing installs -# Running them both for both cases is harmless so we blindly run them +echo "Bootstrapping and migrating Snuba..." $dcr snuba-api bootstrap --force -$dcr snuba-api migrate echo "" # Very naively check whether there's an existing sentry-postgres volume and the PG version in it From c34484ddbfce524bb0d6b14afa127d49d6a9ea2c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 18 May 2020 19:22:54 +0300 Subject: [PATCH 112/417] feat(kafka): Increase max message size (#497) Fixes #402. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index a59aa480d7..aa226737b9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -82,6 +82,8 @@ services: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_MESSAGE_MAX_BYTES: '50000000' #50MB or bust + KAFKA_MAX_REQUEST_SIZE: '50000000' #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN' KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN' From 9f39e3cea1a6f2cae2b38b8aa68319ceb5de9a75 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 22 May 2020 16:12:20 +0300 Subject: [PATCH 113/417] fix(uwsgi): Make sure uWSGI talks proper HTTP/1.1 (#499) This patch brings back the HTTP/1.1 related settings for uWSGI to fix #486 as apparently Relay tries to talk to Sentry Web with keep alives where uWSGI terminates the connection unexpectedly. It also ports some configs for uWSGI and nginx from single-tenant. --- nginx/nginx.conf | 105 +++++++++++++++++++++------------- sentry/sentry.conf.example.py | 24 +++++++- 2 files changed, 87 insertions(+), 42 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c3bb9e58d7..31b68e0333 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -1,50 +1,73 @@ -user nginx; -worker_processes 1; +user nginx; +worker_processes 1; -error_log /var/log/nginx/error.log warn; -pid /var/run/nginx.pid; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; events { - worker_connections 1024; + worker_connections 1024; } http { - default_type application/octet-stream; - - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent" "$http_x_forwarded_for"'; - - sendfile on; - keepalive_timeout 65; - - upstream relay { - server relay:3000; - } - - upstream sentry { - server web:9000; - } - - server { - listen 80; - # use the docker DNS server to resolve ips for relay and sentry containers - resolver 127.0.0.11 ipv6=off; - client_max_body_size 100M; - - proxy_redirect off; - proxy_set_header Host $host; - - location /api/store/ { - proxy_pass http://relay; - } - location ~ ^/api/[1-9]\d*/ { - proxy_pass http://relay; - } - location / { - proxy_pass http://sentry; - } - } + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + sendfile on; + tcp_nopush on; + tcp_nodelay on; + reset_timedout_connection on; + + keepalive_timeout 75s; + + gzip off; + server_tokens off; + + server_names_hash_bucket_size 64; + types_hash_max_size 2048; + types_hash_bucket_size 64; + client_max_body_size 5m; + + proxy_http_version 1.1; + proxy_redirect off; + proxy_buffering off; + proxy_next_upstream error timeout invalid_header http_502 http_503 non_idempotent; + proxy_next_upstream_tries 2; + + # Remove the Connection header if the client sends it, + # it could be "close" to close a keepalive connection + proxy_set_header Connection ''; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-Request-Id $request_id; + proxy_read_timeout 30s; + proxy_send_timeout 5s; + + upstream relay { + server relay:3000; + } + + upstream sentry { + server web:9000; + } + + server { + listen 80; + + location /api/store/ { + proxy_pass http://relay; + } + location ~ ^/api/[1-9]\d*/ { + proxy_pass http://relay; + } + location / { + proxy_pass http://sentry; + } + } } diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 09eb81ff94..32682aa884 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -155,8 +155,30 @@ SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { + # These ase for proper HTTP/1.1 support from uWSGI + # Without these it doesn't do keep-alives causing + # issues with Relay's direct requests. + "http-keepalive": True, + "http-chunked-input": True, + # the number of web workers + 'workers': 3, + # Turn off memory reporting "memory-report": False, - # 'workers': 3, # the number of web workers + # Some stuff so uwsgi will cycle workers sensibly + 'max-requests': 100000, + 'max-requests-delta': 500, + 'max-worker-lifetime': 86400, + # Duplicate options from sentry default just so we don't get + # bit by sentry changing a default value that we depend on. + 'thunder-lock': True, + 'log-x-forwarded-for': False, + 'buffer-size': 32768, + 'limit-post': 209715200, + 'disable-logging': True, + 'reload-on-rss': 600, + 'ignore-sigpipe': True, + 'ignore-write-errors': True, + 'disable-write-exception': True, } ########### From e8d8cda6882356f4ac30dfc5323675789497f79e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 23 May 2020 06:24:44 +0300 Subject: [PATCH 114/417] feat(versioning): Add SENTRY_VERSION env var for sentry, snuba, relay (#509) This is in preparation for the upcoming CalVer transition. Introduces a general `$SENTRY_VERSION` env variable, defaulting to `latest`. --- .env | 1 + docker-compose.yml | 5 +++-- sentry/Dockerfile | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/.env b/.env index 008cb10483..b4f5273538 100644 --- a/.env +++ b/.env @@ -1,2 +1,3 @@ COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 +SENTRY_VERSION=latest diff --git a/docker-compose.yml b/docker-compose.yml index aa226737b9..f0fce4ec68 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ x-sentry-defaults: &sentry_defaults context: ./sentry args: - SENTRY_IMAGE + - SENTRY_VERSION image: sentry-onpremise-local depends_on: - redis @@ -31,7 +32,7 @@ x-snuba-defaults: &snuba_defaults - redis - clickhouse - kafka - image: 'getsentry/snuba:latest' + image: 'getsentry/snuba:$SENTRY_VERSION' environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse @@ -179,7 +180,7 @@ services: - relay relay: << : *restart_policy - image: "getsentry/relay:latest" + image: "getsentry/relay:$SENTRY_VERSION" volumes: - type: bind read_only: true diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 88bb0761eb..406830edeb 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,5 +1,6 @@ +ARG SENTRY_VERSION=latest ARG SENTRY_IMAGE -FROM ${SENTRY_IMAGE:-getsentry/sentry:latest} +FROM ${SENTRY_IMAGE:-getsentry/sentry:$SENTRY_VERSION} COPY . /usr/src/sentry From b651fc7fdaa1e8aed4cf30396adeac2b95ebec59 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 23 May 2020 22:38:30 +0300 Subject: [PATCH 115/417] fix(versioning): Fix leftover snuba:latest in cleanup Follow up to #509. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f0fce4ec68..39cce8e691 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,7 +124,7 @@ services: build: context: ./cron args: - BASE_IMAGE: 'getsentry/snuba:latest' + BASE_IMAGE: 'getsentry/snuba:$SENTRY_VERSION' command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: << : *restart_policy From 9793bb71570b26db8bc7da4b7c6fa09fa547a071 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 23 May 2020 22:49:31 +0300 Subject: [PATCH 116/417] meta(versioning): Reword versioning sect. now that we have version tags (#510) --- README.md | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 81525c89ba..8e2eaa9bf0 100644 --- a/README.md +++ b/README.md @@ -29,21 +29,15 @@ If you have any issues or questions, our [Community Forum](https://forum.sentry. ## Versioning -We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:latest`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: - -```shell -SENTRY_IMAGE=getsentry/sentry:10 ./install.sh -``` +If you want to install a specific release of Sentry, use the tags/releases on this repo. -or +We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:latest`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh ``` -If you want to use different or specific images for other services, you may create a `docker-compose.overrides.yaml` file in the repo and override the `image` field for the corresponding services. - -We strongly recommend keeping the `latest` tags for all, if you are using this repository directly. We also recommend using specific commit tags if you are consuming any of our Docker images in an environment that needs consistent deploys such as [a Helm chart](https://github.com/helm/charts/tree/master/stable/sentry). +Note that this may not work for all commit SHAs as this repository evolves with Sentry and its satellite projects. It is highly recommended to check out a version of this repository that is close to the timestamp of the Sentry commit you are installing. ## Event Retention From fe1f23f10fa656307a933def2fa8a107d120162f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 24 May 2020 23:52:30 +0300 Subject: [PATCH 117/417] fix(zk): Fix zookeper upgrade to 5.5.0 (#511) Fixes the "logs found but no snapshot" error when upgrading from a short-lived older version. Fixes #472. See https://issues.apache.org/jira/browse/ZOOKEEPER-3056. --- install.sh | 10 ++++++++++ zookeeper/snapshot.0 | Bin 0 -> 424 bytes 2 files changed, 10 insertions(+) create mode 100644 zookeeper/snapshot.0 diff --git a/install.sh b/install.sh index 3aa3b8921b..c0419b14ed 100755 --- a/install.sh +++ b/install.sh @@ -164,6 +164,16 @@ $dc build --force-rm --parallel echo "" echo "Docker images built." + +ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') +ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') +# This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 +if [ "$ZOOKEEPER_LOG_FILE_COUNT" -gt "0" ] && [ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq "0" ]; then + $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' + $dc run -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper +fi + + echo "Bootstrapping and migrating Snuba..." $dcr snuba-api bootstrap --force echo "" diff --git a/zookeeper/snapshot.0 b/zookeeper/snapshot.0 new file mode 100644 index 0000000000000000000000000000000000000000..3e6deee02b83966e6e933ea33a686a7c11223515 GIT binary patch literal 424 zcma#@4)$YUU|{+W1wb|kFhVFW4Pt{ZYk7WAP7090mY7$WpO*?%!hog~CQK^LrC*hw zpPiaokXi)NN+nYT5vJ-V=jWwmrX#rv=1U|Y>`ahju*e7?8D3bLUy=w?LnVhZ>Vpj1 MvB=sJNHOXI0LqX(R{#J2 literal 0 HcmV?d00001 From 3e7df7be17a07dd82ee7496df5dd736ab447baf8 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 25 May 2020 01:11:19 +0300 Subject: [PATCH 118/417] fix(install): Skip pull when SENTRY_IMAGE is set (#514) This also uses SENTRY_VERSION instead of hard-coding `:latest` as the tag when pulling (follow up to #509). --- install.sh | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index c0419b14ed..38336caa17 100755 --- a/install.sh +++ b/install.sh @@ -153,7 +153,12 @@ echo "" # shows a 404 error on the console which is confusing and unnecessary. To overcome this, we add the stderr>stdout # redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true -docker pull ${SENTRY_IMAGE:-getsentry/sentry:latest} + +if [ -z "$SENTRY_IMAGE" ]; then + docker pull getsentry/sentry:${SENTRY_VERSION:-latest} +else + echo "SENTRY_IMAGE is explicitly set, skipped pulling." +fi echo "" echo "Building and tagging Docker images..." From 83160e8bdbebfea2c0f02849b545b9cf1b30c324 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 25 May 2020 10:18:18 +0300 Subject: [PATCH 119/417] fix(gcb): We need to pull SENTRY_IMAGE on GCB Follow up to #514. --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 38336caa17..1893bf75cc 100755 --- a/install.sh +++ b/install.sh @@ -157,7 +157,8 @@ $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true if [ -z "$SENTRY_IMAGE" ]; then docker pull getsentry/sentry:${SENTRY_VERSION:-latest} else - echo "SENTRY_IMAGE is explicitly set, skipped pulling." + # We may not have the set image on the repo (local images) so allow fails + docker pull $SENTRY_IMAGE || true; fi echo "" From 85f267bec78a43f15f0fdc7da0e4bfe646eb8e16 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 28 May 2020 21:33:57 +0300 Subject: [PATCH 120/417] fix(nginx): Increase upload size back to 100M for store Addresses https://github.com/getsentry/onpremise/pull/499#discussion_r431537129 --- nginx/nginx.conf | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 31b68e0333..c04bf2adef 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -30,7 +30,7 @@ http { server_names_hash_bucket_size 64; types_hash_max_size 2048; types_hash_bucket_size 64; - client_max_body_size 5m; + client_max_body_size 100m; proxy_http_version 1.1; proxy_redirect off; @@ -67,6 +67,7 @@ http { proxy_pass http://relay; } location / { + client_max_body_size 5m; proxy_pass http://sentry; } } From 01bec9999612d82328d789e64325d1369aeac4c2 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 28 May 2020 21:42:58 +0300 Subject: [PATCH 121/417] fix(nginx): Increase upload size back to 100M for Sentry too Addresses https://github.com/getsentry/onpremise/pull/499#discussion_r431537129 --- nginx/nginx.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index c04bf2adef..84027d49b3 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -67,7 +67,6 @@ http { proxy_pass http://relay; } location / { - client_max_body_size 5m; proxy_pass http://sentry; } } From 41f8b0f14949e6cca5f4bbbde1ff8628951406b2 Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Mon, 1 Jun 2020 18:05:19 +0300 Subject: [PATCH 122/417] license: Update BSL change date (#522) Co-authored-by: sentry-update-license-date[bot] <57668832+sentry-update-license-date[bot]@users.noreply.github.com> --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index a6bad0ab83..cd8936f1f4 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-05-01 +Change Date: 2023-06-01 Change License: Apache License, Version 2.0 From 3522a123256f78c9bdfe5da3f20df2eb8452878c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 1 Jun 2020 23:34:54 +0300 Subject: [PATCH 123/417] ref(sentry): Remove explicit Discover v2 flags (#523) Discover v2 is now enabled by default: getsentry/sentry#19023 --- sentry/sentry.conf.example.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 32682aa884..05862cbd40 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -206,9 +206,6 @@ for feature in ( "organizations:discover", "organizations:events", - "organizations:discover-basic", - "organizations:discover-query", - "organizations:events-v2", "organizations:global-views", "organizations:integrations-issue-basic", "organizations:integrations-issue-sync", From ff057d1d2cc72250a40f329a08eed5f27b172997 Mon Sep 17 00:00:00 2001 From: Armin Ronacher Date: Tue, 2 Jun 2020 16:04:46 +0200 Subject: [PATCH 124/417] feat: Add sessions snuba consumer to setup (#524) Co-authored-by: Markus Unterwaditzer Co-authored-by: Burak Yigit Kaya --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 39cce8e691..c10665a45d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -17,6 +17,7 @@ x-sentry-defaults: &sentry_defaults - snuba-api - snuba-consumer - snuba-outcomes-consumer + - snuba-sessions-consumer - snuba-replacer - symbolicator - kafka @@ -115,6 +116,10 @@ services: snuba-outcomes-consumer: << : *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 + # Kafka consumer responsible for feeding session data into Clickhouse + snuba-sessions-consumer: + << : *snuba_defaults + command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 snuba-replacer: << : *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 From c2120aafc94e34ee33420d0afb7f3c1641fbe514 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 2 Jun 2020 19:45:07 +0300 Subject: [PATCH 125/417] fix(zookeeper): Temp ZK fix should run in detached mode (#525) Fixes #519. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 1893bf75cc..7bf71b6e5e 100755 --- a/install.sh +++ b/install.sh @@ -176,7 +176,7 @@ ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 if [ "$ZOOKEEPER_LOG_FILE_COUNT" -gt "0" ] && [ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq "0" ]; then $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' - $dc run -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper + $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper fi From 677e753c186c19944cf5406a058eb6211268dec9 Mon Sep 17 00:00:00 2001 From: Alexander Kalyuzhnyy Date: Thu, 11 Jun 2020 22:33:36 +0300 Subject: [PATCH 126/417] fix(relay): Increase Kafka message size limit to 50MB (#527) Based on my forum post https://forum.sentry.io/t/sentry-native-and-kafka-messagesizetoolarge-error/9948 In short I tried to send minidumps more than 1mb(in fact 2mb), and relay service failed to store event, with kafka error `MessageSizeTooLarge`. You can reproduce this if you recreate install this on local machine, create simple native project, and send minidump file more than 1mb via curl like suggested in docs or in web ui. And check the logs. I reason selected value of 50mb on https://github.com/getsentry/onpremise/blob/master/docker-compose.yml#L87, like you already hardcoded on kafka service. --- relay/config.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/relay/config.yml b/relay/config.yml index 8700336189..da00363fba 100644 --- a/relay/config.yml +++ b/relay/config.yml @@ -9,4 +9,5 @@ processing: enabled: true kafka_config: - {name: "bootstrap.servers", value: "kafka:9092"} + - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 From af502d0ba5ed6b0ab674c08fcd06c8edc5fbcaa3 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 14 Jun 2020 17:27:41 +0300 Subject: [PATCH 127/417] build(gha): Add Craft releases as a GitHub action (#538) --- .craft.yml | 11 +++++++++++ .github/workflows/release.yml | 14 ++++++++++++++ scripts/bump-version.sh | 11 +++++++++++ 3 files changed, 36 insertions(+) create mode 100644 .craft.yml create mode 100644 .github/workflows/release.yml create mode 100644 scripts/bump-version.sh diff --git a/.craft.yml b/.craft.yml new file mode 100644 index 0000000000..ff71152a9c --- /dev/null +++ b/.craft.yml @@ -0,0 +1,11 @@ +minVersion: "0.10.0" +github: + owner: getsentry + repo: onpremise +changelogPolicy: none +artifactProvider: + name: none +statusProvider: + name: github +targets: + - name: github diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000000..05ad7c81ed --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,14 @@ +on: + repository_dispatch: + types: [release] +jobs: + release: + runs-on: ubuntu-latest + name: "Release a new version" + steps: + - uses: actions/checkout@v2 + - uses: getsentry/craft-action + with: + action: prepare + version: ${{ github.event.client_payload.version }} + dry_run: ${{ github.event.client_payload.dry_run }} diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh new file mode 100644 index 0000000000..2a197d8f97 --- /dev/null +++ b/scripts/bump-version.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -eux + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $SCRIPT_DIR/.. + +OLD_VERSION="$1" +NEW_VERSION="$2" + +VERSION=$2 +echo "New version: $VERSION" From 8c7b633b90ddff5cf9938ea6792bc3843ce2e9e5 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 14 Jun 2020 17:29:34 +0300 Subject: [PATCH 128/417] fix(gha): Fix indent in yaml file --- .github/workflows/release.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 05ad7c81ed..39547110ca 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,12 +3,12 @@ on: types: [release] jobs: release: - runs-on: ubuntu-latest - name: "Release a new version" - steps: - - uses: actions/checkout@v2 - - uses: getsentry/craft-action - with: - action: prepare - version: ${{ github.event.client_payload.version }} - dry_run: ${{ github.event.client_payload.dry_run }} + runs-on: ubuntu-latest + name: "Release a new version" + steps: + - uses: actions/checkout@v2 + - uses: getsentry/craft-action + with: + action: prepare + version: ${{ github.event.client_payload.version }} + dry_run: ${{ github.event.client_payload.dry_run }} From cfaa3683060ee58ca6fe42ab854557fe4194db83 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 14 Jun 2020 17:36:53 +0300 Subject: [PATCH 129/417] fix(gha): Add version to craft-action --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 39547110ca..1fb717d6a6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ jobs: name: "Release a new version" steps: - uses: actions/checkout@v2 - - uses: getsentry/craft-action + - uses: getsentry/craft-action@master with: action: prepare version: ${{ github.event.client_payload.version }} From 2fc9811c74a517fd4db85dabf0211a4712bb6630 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Jun 2020 12:44:13 +0300 Subject: [PATCH 130/417] build(gha): Add automated CalVer releases (#539) --- .craft.yml | 1 + .env | 1 + .github/workflows/release.yml | 23 ++++++++++++++++++++--- docker-compose.yml | 8 ++++---- scripts/bump-version.sh | 12 +++++++++--- 5 files changed, 35 insertions(+), 10 deletions(-) diff --git a/.craft.yml b/.craft.yml index ff71152a9c..ad41cd872f 100644 --- a/.craft.yml +++ b/.craft.yml @@ -2,6 +2,7 @@ minVersion: "0.10.0" github: owner: getsentry repo: onpremise +releaseBranchPrefix: releases changelogPolicy: none artifactProvider: name: none diff --git a/.env b/.env index b4f5273538..aec59cb371 100644 --- a/.env +++ b/.env @@ -1,3 +1,4 @@ COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 SENTRY_VERSION=latest +SYMBOLICATOR_VERSION=latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1fb717d6a6..0a36cb3ff9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,14 +1,31 @@ on: repository_dispatch: types: [release] + schedule: + # We want the release to be at 10 or 11am Pacific Time + # We also make this an hour after all others such as Sentry, + # Snuba, and Relay to make sure their releases finish. + - cron: '0 18 15 * *' jobs: release: runs-on: ubuntu-latest name: "Release a new version" steps: + - id: calver + if: ${{ !github.event.client_payload.version }} + run: echo "::set-output name=version::$(date +'%y.%-m.0')" - uses: actions/checkout@v2 - - uses: getsentry/craft-action@master + - uses: getsentry/craft@master with: action: prepare - version: ${{ github.event.client_payload.version }} - dry_run: ${{ github.event.client_payload.dry_run }} + version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} + env: + DRY_RUN: ${{ github.event.client_payload.dry_run }} + - uses: getsentry/craft@master + with: + action: publish + version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} + keep_branch: true + no_merge: true + env: + DRY_RUN: ${{ github.event.client_payload.dry_run }} diff --git a/docker-compose.yml b/docker-compose.yml index c10665a45d..c27cdbf71e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -133,7 +133,7 @@ services: command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: << : *restart_policy - image: 'getsentry/symbolicator:latest' + image: 'getsentry/symbolicator:$SYMBOLICATOR_VERSION' volumes: - 'sentry-symbolicator:/data' command: run @@ -143,7 +143,7 @@ services: build: context: ./cron args: - BASE_IMAGE: 'getsentry/symbolicator:latest' + BASE_IMAGE: 'getsentry/symbolicator:$SYMBOLICATOR_VERSION' command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - 'sentry-symbolicator:/data' @@ -174,7 +174,7 @@ services: << : *restart_policy ports: - '9000:80/tcp' - image: "nginx:1.16" + image: 'nginx:1.16' volumes: - type: bind read_only: true @@ -185,7 +185,7 @@ services: - relay relay: << : *restart_policy - image: "getsentry/relay:$SENTRY_VERSION" + image: 'getsentry/relay:$SENTRY_VERSION' volumes: - type: bind read_only: true diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 2a197d8f97..081af99794 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -1,5 +1,5 @@ #!/bin/bash -set -eux +set -eu SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $SCRIPT_DIR/.. @@ -7,5 +7,11 @@ cd $SCRIPT_DIR/.. OLD_VERSION="$1" NEW_VERSION="$2" -VERSION=$2 -echo "New version: $VERSION" +SYMBOLICATOR_VERSION=$(curl -sSL 'https://api.github.com/repos/getsentry/symbolicator/git/refs/heads/master' | grep -Po '(?<=\"sha\": \")([a-f0-9]{5,40})(?=\",?)') + +sed -i -e "s/^SYMBOLICATOR_VERSION=.*\$/SYMBOLICATOR_VERSION=$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^SENTRY_VERSION=.*\$/SENTRY_VERSION=$NEW_VERSION/" .env +sed -i -e "s/^\# Sentry .* On-Premise/# Sentry $NEW_VERSION On-Premise/" README.md + +echo "New version: $NEW_VERSION" +echo "New Symbolicator version: $SYMBOLICATOR_VERSION" From ea2c31f0128552d728a92aa2db7eaf2dd819d8d0 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Jun 2020 22:48:09 +0300 Subject: [PATCH 131/417] build(gha): Add GIT_*_NAME variables --- .github/workflows/release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0a36cb3ff9..38d3f420d0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,9 @@ jobs: version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} env: DRY_RUN: ${{ github.event.client_payload.dry_run }} + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com - uses: getsentry/craft@master with: action: publish @@ -29,3 +32,6 @@ jobs: no_merge: true env: DRY_RUN: ${{ github.event.client_payload.dry_run }} + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com From d83aa55bf800a3362d1d5f66a4fad90779e99aae Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Jun 2020 22:49:00 +0300 Subject: [PATCH 132/417] build(gha): Add name to release action --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 38d3f420d0..a37bced47c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,3 +1,4 @@ +name: release on: repository_dispatch: types: [release] From b2b497e6198424aa9aa6db40a4c157f3241abe13 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Jun 2020 22:55:03 +0300 Subject: [PATCH 133/417] build(gha): Workaround for Craft action params --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a37bced47c..372d4da7fe 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,8 +29,8 @@ jobs: with: action: publish version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} - keep_branch: true - no_merge: true + keep_branch: '--keep-branch' + no_merge: '--no-merge' env: DRY_RUN: ${{ github.event.client_payload.dry_run }} GIT_COMMITTER_NAME: getsentry-bot From ea93a4b8862f1301ceed64c127cfa538590b4722 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Jun 2020 22:58:52 +0300 Subject: [PATCH 134/417] build(gha): Add GH status context --- .craft.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.craft.yml b/.craft.yml index ad41cd872f..37c861bf4a 100644 --- a/.craft.yml +++ b/.craft.yml @@ -8,5 +8,8 @@ artifactProvider: name: none statusProvider: name: github + config: + contexts: + - 'continuous-integration/travis-ci/push' targets: - name: github From f66ca96fa7f5bcaa46c464c46b34edf06d070791 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Jun 2020 23:11:09 +0300 Subject: [PATCH 135/417] build(gha): Add skip_prepare option for manual triggers --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 372d4da7fe..b08ddb0739 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,7 @@ jobs: run: echo "::set-output name=version::$(date +'%y.%-m.0')" - uses: actions/checkout@v2 - uses: getsentry/craft@master + if: ${{ !github.event.client_payload.skip_prepare }} with: action: prepare version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} From 131a324af29c57716830c02996e753ee36d58d1b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 17 Jun 2020 20:56:24 +0300 Subject: [PATCH 136/417] docs: Rename title from 10 to Nightly --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 8e2eaa9bf0..c055c05c0b 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Sentry 10 On-Premise [![Build Status][build-status-image]][build-status-url] +# Sentry Nightly On-Premise [![Build Status][build-status-image]][build-status-url] Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From e75e6f1deeb8dd534700f783fe6c10a4418f42d4 Mon Sep 17 00:00:00 2001 From: strange-developer Date: Mon, 29 Jun 2020 15:16:42 +0200 Subject: [PATCH 137/417] fix(zookeeper): cp: cannot create regular file (#552) Hi, I've been through quite a few different ways of implementing this fix and settled on creating a variable to store the output of checking whether the zookeeper copy target folder exists and copying the snapshot file based on the copy target folder existing. I've ran quite a few manual tests for each option as well. Currently the PR sits on Option 3 from the below options. **Option 1** Judging from the [Jira issue](https://issues.apache.org/jira/browse/ZOOKEEPER-3056), it seems like the work around for zookeeper upgrades could be omitted entirely since the issue relates to upgrades from v3.4.10 to v3.5.4. I've tested removing the zookeeper workaround entirely and that install ran smoothly on a clean install of Sentry (no existing data) as well as an install of Sentry that currently has very minimal amount of log entries (roughly 100 log entries). Could we possibly remove the workaround entirely? **Option 2** The second option was to simply add a check to the currently [existing line](https://github.com/getsentry/onpremise/blob/master/install.sh#L178) of whether the copy target folder exists and perform the snapshot file copy only if the copy target folder exists. This is the least amount of code and possibly the simpler fix while also setting the `ZOOKEEPER_SNAPSHOT_TRUST_EMPTY` env var to `true`, however, some unnecessary calculations will be done to determine the `ZOOKEEPER_LOG_FILE_COUNT` and `ZOOKEEPER_SNAPSHOT_FILE_COUNT`. **Option 3** I've created a variable to store whether the copy target folder exists and proceed with the zookeeper upgrade workaround only if the copy target folder exists. This means that if the copy target folder does not exist, the env var `ZOOKEEPER_SNAPSHOT_TRUST_EMPTY` will not be set. Fixes #547. Co-authored-by: chamirb --- install.sh | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/install.sh b/install.sh index 7bf71b6e5e..bc92d7ed5b 100755 --- a/install.sh +++ b/install.sh @@ -170,16 +170,17 @@ $dc build --force-rm --parallel echo "" echo "Docker images built." - -ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') -ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') -# This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 -if [ "$ZOOKEEPER_LOG_FILE_COUNT" -gt "0" ] && [ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq "0" ]; then - $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' - $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper +ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') +if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then + ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') + ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') + # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 + if [ "$ZOOKEEPER_LOG_FILE_COUNT" -gt "0" ] && [ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq "0" ]; then + $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' + $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper + fi fi - echo "Bootstrapping and migrating Snuba..." $dcr snuba-api bootstrap --force echo "" From a549ab1300f94149bf347df9277cf7bc50f0e5d7 Mon Sep 17 00:00:00 2001 From: "sentry-update-license-date[bot]" <57668832+sentry-update-license-date[bot]@users.noreply.github.com> Date: Fri, 3 Jul 2020 22:06:19 +0300 Subject: [PATCH 138/417] license: Update BSL change date (#557) Co-authored-by: sentry-update-license-date[bot] <57668832+sentry-update-license-date[bot]@users.noreply.github.com> --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index cd8936f1f4..38b25bd08d 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-06-01 +Change Date: 2023-07-01 Change License: Apache License, Version 2.0 From a01d0136172f3b6eb9de4714618f688e33ec68a7 Mon Sep 17 00:00:00 2001 From: Jack Date: Wed, 8 Jul 2020 20:56:49 +0800 Subject: [PATCH 139/417] fix(slack): Add note about legacy app flag for newly created bots (#563) Co-authored-by: jack --- sentry/config.example.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index b75e911c54..6edda6fc18 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -98,4 +98,6 @@ transaction-events.force-disable-internal-project: true # slack.client-id: <'client id'> # slack.client-secret: -# slack.verification-token: \ No newline at end of file +# slack.verification-token: +## only uncomment legacy-app if you made your slack bot after july 2020 +# slack.legacy-app: False \ No newline at end of file From 96889a6175867c8c08090dae17fca9b8e44c2291 Mon Sep 17 00:00:00 2001 From: Mike Purvis Date: Wed, 8 Jul 2020 13:42:33 -0400 Subject: [PATCH 140/417] Add volumed-out config directory for symbolicator. (#566) I would like to be able to customize the configuration for my Sentry 10 symbolicator instance, which this change allows me to easily do. See related: https://github.com/getsentry/symbolicator/issues/245 Co-authored-by: Burak Yigit Kaya --- .gitignore | 1 + docker-compose.yml | 6 +++++- install.sh | 4 +++- symbolicator/config.example.yml | 8 ++++++++ 4 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 symbolicator/config.example.yml diff --git a/.gitignore b/.gitignore index 5931ebe3e3..3f2261c8fe 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,4 @@ sentry/config.yml sentry/*.bak sentry/requirements.txt relay/credentials.json +symbolicator/config.yml diff --git a/docker-compose.yml b/docker-compose.yml index c27cdbf71e..40007504d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -136,7 +136,11 @@ services: image: 'getsentry/symbolicator:$SYMBOLICATOR_VERSION' volumes: - 'sentry-symbolicator:/data' - command: run + - type: bind + read_only: true + source: ./symbolicator + target: /etc/symbolicator + command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: << : *restart_policy image: symbolicator-cleanup-onpremise-local diff --git a/install.sh b/install.sh index bc92d7ed5b..f9e91c4d87 100755 --- a/install.sh +++ b/install.sh @@ -14,6 +14,7 @@ MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' +SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' @@ -95,6 +96,7 @@ echo "" ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS +ensure_file_from_example $SYMBOLICATOR_CONFIG_YML if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then echo "" @@ -171,7 +173,7 @@ echo "" echo "Docker images built." ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') -if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then +if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 diff --git a/symbolicator/config.example.yml b/symbolicator/config.example.yml new file mode 100644 index 0000000000..62cf9b83b7 --- /dev/null +++ b/symbolicator/config.example.yml @@ -0,0 +1,8 @@ +# See: https://getsentry.github.io/symbolicator/#configuration +cache_dir: "/data" +bind: "0.0.0.0:3021" +logging: + level: "warn" +metrics: + statsd: null +sentry_dsn: null # TODO: Automatically fill this with the internal project DSN From 9d44b99c5521239971a4be397fe68c9c3f0ac4f3 Mon Sep 17 00:00:00 2001 From: Jack Date: Sat, 11 Jul 2020 02:10:05 +0800 Subject: [PATCH 141/417] fix(slack): Point to newer docs, better defaults (#571) * added new default going forward uncommented * added link to new guide * added support for new signing-secret * slight rejig to config --- sentry/config.example.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 6edda6fc18..0e74ec55bc 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -94,10 +94,12 @@ transaction-events.force-disable-internal-project: true # Slack Integration # ##################### -# Refer to https://forum.sentry.io/t/how-to-configure-slack-in-your-on-prem-sentry/3463 for setup instructions. +# Refer to https://develop.sentry.dev/integrations/slack/ for setup instructions. # slack.client-id: <'client id'> # slack.client-secret: +# slack.signing-secret: +## If you made your slack bot before july 2020 set legacy-app to True +slack.legacy-app: False +## If legacy-app is True use verfication-token instead of signing-secret # slack.verification-token: -## only uncomment legacy-app if you made your slack bot after july 2020 -# slack.legacy-app: False \ No newline at end of file From 73213bc51f57e665df0077a2c323f9d9ee8d948d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 10 Jul 2020 23:53:50 +0300 Subject: [PATCH 142/417] ref(relay): Remove PK and rely on INTERNAL_IPS (#572) This patch adds `INTERNAL_IPS` definition to `sentry.conf.py` by sniffing the network from eth0 and relies on this for trusted Relays instead of the ALLOWLISTED PKs. This removes the necessity of syncing Relay PKs to `sentry.conf.py`. This PR needs getsentry/sentry#19798 to work. --- .gitignore | 1 + install.sh | 14 +---------- relay/{config.yml => config.example.yml} | 1 - sentry/sentry.conf.example.py | 30 ++++++++++++++++++++++++ 4 files changed, 32 insertions(+), 14 deletions(-) rename relay/{config.yml => config.example.yml} (98%) diff --git a/.gitignore b/.gitignore index 3f2261c8fe..b8ee807d52 100644 --- a/.gitignore +++ b/.gitignore @@ -79,4 +79,5 @@ sentry/config.yml sentry/*.bak sentry/requirements.txt relay/credentials.json +relay/config.yml symbolicator/config.yml diff --git a/install.sh b/install.sh index f9e91c4d87..aea57a6814 100755 --- a/install.sh +++ b/install.sh @@ -97,6 +97,7 @@ ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS ensure_file_from_example $SYMBOLICATOR_CONFIG_YML +ensure_file_from_example $RELAY_CONFIG_YML if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then echo "" @@ -245,19 +246,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi -RELAY_CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") -if [ -z "$RELAY_CREDENTIALS" ]; then - >&2 echo "FAIL: Cannot read credentials back from $RELAY_CREDENTIALS_JSON." - >&2 echo " Please ensure this file is readable and contains valid credentials." - >&2 echo "" - exit 1 -fi - -if ! grep -q "\"$RELAY_CREDENTIALS\"" "$SENTRY_CONFIG_PY"; then - echo "SENTRY_RELAY_WHITELIST_PK = (SENTRY_RELAY_WHITELIST_PK or []) + ([\"$RELAY_CREDENTIALS\"])" >> "$SENTRY_CONFIG_PY" - echo "Relay public key written to $SENTRY_CONFIG_PY" - echo "" -fi cleanup diff --git a/relay/config.yml b/relay/config.example.yml similarity index 98% rename from relay/config.yml rename to relay/config.example.yml index da00363fba..f54c9348ea 100644 --- a/relay/config.yml +++ b/relay/config.example.yml @@ -1,4 +1,3 @@ ---- relay: upstream: "http://web:9000/" host: 0.0.0.0 diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 05862cbd40..7116f9c733 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -3,6 +3,36 @@ from sentry.conf.server import * # NOQA + +# Generously adapted from pynetlinux: https://git.io/JJmga +def get_internal_network(): + import ctypes + import fcntl + import math + import socket + import struct + + iface = 'eth0' + sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + ifreq = struct.pack('16sH14s', iface, socket.AF_INET, b'\x00' * 14) + + try: + ip = struct.unpack( + "!I", struct.unpack('16sH2x4s8x', fcntl.ioctl(sockfd, 0x8915, ifreq))[2] + )[0] + netmask = socket.ntohl( + struct.unpack('16sH2xI8x', fcntl.ioctl(sockfd, 0x891B, ifreq))[2] + ) + except IOError: + return () + base = socket.inet_ntoa(struct.pack("!I", ip & netmask)) + netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) + return ('{0:s}/{1:d}'.format(base, netmask_bits),) + + +INTERNAL_IPS = get_internal_network() +INTERNAL_SYSTEM_IPS = INTERNAL_IPS + DATABASES = { "default": { "ENGINE": "sentry.db.postgres", From 1c9bfd90174c02e98186c830582bcea5048fbfda Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 11 Jul 2020 00:08:14 +0300 Subject: [PATCH 143/417] fix(relay): Fix relay cannot authenticate w/ Sentry (#576) Fixes #486. I finally figured out what the issue was: a missing `post-buffering` option to let Relay finish it's POST request for auth. This PR supersedes #543. --- sentry/sentry.conf.example.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 7116f9c733..37218405ef 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -185,13 +185,13 @@ def get_internal_network(): SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { - # These ase for proper HTTP/1.1 support from uWSGI - # Without these it doesn't do keep-alives causing - # issues with Relay's direct requests. "http-keepalive": True, + "so-keepalive": True, + "http-auto-chunked": True, "http-chunked-input": True, # the number of web workers 'workers': 3, + 'threads': 4, # Turn off memory reporting "memory-report": False, # Some stuff so uwsgi will cycle workers sensibly @@ -203,6 +203,8 @@ def get_internal_network(): 'thunder-lock': True, 'log-x-forwarded-for': False, 'buffer-size': 32768, + # Relay cannot authenticate without the following + 'post-buffering': 32768, 'limit-post': 209715200, 'disable-logging': True, 'reload-on-rss': 600, From 75fe6c073b0368e4143e2184fd4d4e35fa318c13 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 13 Jul 2020 13:07:05 +0300 Subject: [PATCH 144/417] fix(ingest): Fix Relay auth issues and add e2e event ingestion test (#578) This is a long-needed test that tests the whole pipeline from Nginx, Relay, to Kafka, and Snuba. The final missing piece is testing the Symbolicator integration. This PR is also a follow up to #576 as it didn't solve the Relay issues fully (the earlier fix was a coincidence or is not as reliable as it seemed). Fixes #486 (finally?). --- .travis.yml | 2 +- install.sh | 23 ++++++--- sentry/sentry.conf.example.py | 53 +++++++++---------- test.sh | 95 ++++++++++++++++++++++++++++------- 4 files changed, 123 insertions(+), 50 deletions(-) diff --git a/.travis.yml b/.travis.yml index 761d9986d3..eb61651167 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh after_failure: diff --git a/install.sh b/install.sh index aea57a6814..bdd73971c3 100755 --- a/install.sh +++ b/install.sh @@ -19,17 +19,31 @@ RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' +# Courtesy of https://stackoverflow.com/a/2183063/90297 +trap_with_arg() { + func="$1" ; shift + for sig ; do + trap "$func $sig" "$sig" + done +} + DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { if [ "$DID_CLEAN_UP" -eq 1 ]; then return 0; fi - echo "Cleaning up..." - $dc stop &> /dev/null DID_CLEAN_UP=1 + + if [ "$1" != "EXIT" ]; then + echo "An error occurred, caught SIG$1"; + echo "Cleaning up..." + fi + + $dc stop &> /dev/null } -trap cleanup ERR INT TERM +trap_with_arg cleanup ERR INT TERM EXIT + echo "Checking minimum requirements..." @@ -246,9 +260,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi - -cleanup - echo "" echo "----------------" echo "You're all done! Run the following command to get Sentry running:" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 37218405ef..13353c1ed9 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -12,22 +12,22 @@ def get_internal_network(): import socket import struct - iface = 'eth0' + iface = "eth0" sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ifreq = struct.pack('16sH14s', iface, socket.AF_INET, b'\x00' * 14) + ifreq = struct.pack("16sH14s", iface, socket.AF_INET, b"\x00" * 14) try: ip = struct.unpack( - "!I", struct.unpack('16sH2x4s8x', fcntl.ioctl(sockfd, 0x8915, ifreq))[2] + "!I", struct.unpack("16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] )[0] netmask = socket.ntohl( - struct.unpack('16sH2xI8x', fcntl.ioctl(sockfd, 0x891B, ifreq))[2] + struct.unpack("16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] ) except IOError: return () base = socket.inet_ntoa(struct.pack("!I", ip & netmask)) netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) - return ('{0:s}/{1:d}'.format(base, netmask_bits),) + return ("{0:s}/{1:d}".format(base, netmask_bits),) INTERNAL_IPS = get_internal_network() @@ -60,7 +60,7 @@ def get_internal_network(): SENTRY_SINGLE_ORGANIZATION = True SENTRY_OPTIONS["system.event-retention-days"] = int( - env('SENTRY_EVENT_RETENTION_DAYS', '90') + env("SENTRY_EVENT_RETENTION_DAYS", "90") ) ######### @@ -185,32 +185,33 @@ def get_internal_network(): SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { - "http-keepalive": True, + "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), + "protocol": "uwsgi", + # This is need to prevent https://git.io/fj7Lw + "uwsgi-socket": None, "so-keepalive": True, - "http-auto-chunked": True, + # Keep this between 15s-75s as that's what Relay supports + "http-keepalive": 15, "http-chunked-input": True, # the number of web workers - 'workers': 3, - 'threads': 4, - # Turn off memory reporting + "workers": 3, + "threads": 4, "memory-report": False, # Some stuff so uwsgi will cycle workers sensibly - 'max-requests': 100000, - 'max-requests-delta': 500, - 'max-worker-lifetime': 86400, + "max-requests": 100000, + "max-requests-delta": 500, + "max-worker-lifetime": 86400, # Duplicate options from sentry default just so we don't get # bit by sentry changing a default value that we depend on. - 'thunder-lock': True, - 'log-x-forwarded-for': False, - 'buffer-size': 32768, - # Relay cannot authenticate without the following - 'post-buffering': 32768, - 'limit-post': 209715200, - 'disable-logging': True, - 'reload-on-rss': 600, - 'ignore-sigpipe': True, - 'ignore-write-errors': True, - 'disable-write-exception': True, + "thunder-lock": True, + "log-x-forwarded-for": False, + "buffer-size": 32768, + "limit-post": 209715200, + "disable-logging": True, + "reload-on-rss": 600, + "ignore-sigpipe": True, + "ignore-write-errors": True, + "disable-write-exception": True, } ########### @@ -259,7 +260,7 @@ def get_internal_network(): # GitHub Integration # ###################### -GITHUB_EXTENDED_PERMISSIONS = ['repo'] +GITHUB_EXTENDED_PERMISSIONS = ["repo"] ######################### # Bitbucket Integration # diff --git a/test.sh b/test.sh index 41800796f4..ea757a9af5 100755 --- a/test.sh +++ b/test.sh @@ -5,30 +5,91 @@ SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) -declare -a TEST_STRINGS=( + +# Courtesy of https://stackoverflow.com/a/2183063/90297 +trap_with_arg() { + func="$1" ; shift + for sig ; do + trap "$func $sig" "$sig" + done +} + +DID_CLEAN_UP=0 +# the cleanup function will be the exit point +cleanup () { + if [ "$DID_CLEAN_UP" -eq 1 ]; then + return 0; + fi + DID_CLEAN_UP=1 + + if [ "$1" != "EXIT" ]; then + echo "An error occurred, caught SIG$1"; + fi + + echo "Cleaning up..." + rm $COOKIE_FILE + echo "Done." +} +trap_with_arg cleanup ERR INT TERM EXIT + +get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } +sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } + +login () { + INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) + if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then + echo "Initial /auth/login/ redirect failed, exiting..." + echo "$INITIAL_AUTH_REDIRECT" + exit -1 + fi + + CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' + /csrfmiddlewaretoken/ { + print $4 "=" $6; + exit; + }') + + curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; +} + +LOGIN_RESPONSE=$(login); +declare -a LOGIN_TEST_STRINGS=( '"isAuthenticated":true' '"username":"test@example.com"' '"isSuperuser":true' ) +for i in "${LOGIN_TEST_STRINGS[@]}" +do + echo "Testing '$i'..." + echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "Pass." +done -INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) -if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then - echo "Initial /auth/login/ redirect failed, exiting..." - echo "$INITIAL_AUTH_REDIRECT" - exit -1 -fi - -CSRF_TOKEN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' - /csrfmiddlewaretoken/ { - print $4 "=" $6; - exit; - }') -LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" "$SENTRY_TEST_HOST/auth/login/" -H "Referer: $SENTRY_TEST_HOST/auth/login/" -b "$COOKIE_FILE" -c "$COOKIE_FILE") +# Set up initial/required settings (InstallWizard request) +sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null -TEST_RESULT=0 -for i in "${TEST_STRINGS[@]}" +SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') + +TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-') +echo "Created event $TEST_EVENT_ID." + +EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" +export -f sentry_api_request get_csrf_token +export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH +printf "Checking its existence" +timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' +echo ""; + +EVENT_RESPONSE=$(sentry_api_request "projects/sentry/internal/events/$TEST_EVENT_ID/") +declare -a EVENT_TEST_STRINGS=( + '"eventID":"'"$TEST_EVENT_ID"'"' + '"message":"a failure"' + '"title":"a failure"' + '"object":"42"' +) +for i in "${EVENT_TEST_STRINGS[@]}" do echo "Testing '$i'..." - echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done From a2507c10e042726e55184a7cb25aebc18069b3a6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 13 Jul 2020 19:21:52 +0300 Subject: [PATCH 145/417] fix(superuser): Don't set INTERNAL_IPS to Docker network (#581) `INTERNAL_IPS` is used to check whether to allow superuser access or not. Limiting this to the Docker internal network makes it impossible for anyone to reach admin pages with on-premise setup. This is a follow up to #572 and it fixes #577. --- sentry/sentry.conf.example.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 13353c1ed9..216440df43 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -30,8 +30,8 @@ def get_internal_network(): return ("{0:s}/{1:d}".format(base, netmask_bits),) -INTERNAL_IPS = get_internal_network() -INTERNAL_SYSTEM_IPS = INTERNAL_IPS +INTERNAL_SYSTEM_IPS = get_internal_network() + DATABASES = { "default": { From 61ec4166d16f2a9454cbb27cae975fa72465f0d6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 00:31:46 +0300 Subject: [PATCH 146/417] fix(test): Don't use sentry-cli for test event (#585) Running Docker on the host network is not supported on GCB, which is breaking our builds over at getsentry/sentry. This patch removes the use of sentry-cli and does a manual `curl` request to create the test event. --- test.sh | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/test.sh b/test.sh index ea757a9af5..9bb2e90dcf 100755 --- a/test.sh +++ b/test.sh @@ -69,8 +69,17 @@ done sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') +# We ignore the protocol and the host as we already know those +DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\?://\([0-9a-z]\+\)@[^/]\+/\([0-9]\+\)$|\1\n\2|p'`) +SENTRY_KEY=${DSN_PIECES[0]} +PROJECT_ID=${DSN_PIECES[1]} -TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-') +TEST_EVENT_ID=$(uuidgen -r | tr -d '-') +# Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk +curl --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"} }' \ + -H 'Content-Type: application/json' \ + -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" \ + $SENTRY_TEST_HOST/api/$PROJECT_ID/store/ -sf -o /dev/null echo "Created event $TEST_EVENT_ID." EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" From 35e817ceb49f1a595d8a6f7cf389a80a6fff3fd4 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 00:48:35 +0300 Subject: [PATCH 147/417] fix(gcb): Don't use uuidgen as it doesn't exist on GCB --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 9bb2e90dcf..12c75629bf 100755 --- a/test.sh +++ b/test.sh @@ -74,7 +74,7 @@ DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\?://\([0-9a-z]\+\)@[^/]\+/\([0 SENTRY_KEY=${DSN_PIECES[0]} PROJECT_ID=${DSN_PIECES[1]} -TEST_EVENT_ID=$(uuidgen -r | tr -d '-') +TEST_EVENT_ID=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-f0-9" | head -c 32) # Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk curl --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"} }' \ -H 'Content-Type: application/json' \ From f922986403f562547d2d2f8f5118559a44692825 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 01:09:02 +0300 Subject: [PATCH 148/417] test(gcb): Add more debug info for GCB fails --- test.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/test.sh b/test.sh index 12c75629bf..eb6a07c1fa 100755 --- a/test.sh +++ b/test.sh @@ -10,7 +10,7 @@ COOKIE_FILE=$(mktemp) trap_with_arg() { func="$1" ; shift for sig ; do - trap "$func $sig" "$sig" + trap "$func $sig "'$LINENO' "$sig" done } @@ -23,7 +23,7 @@ cleanup () { DID_CLEAN_UP=1 if [ "$1" != "EXIT" ]; then - echo "An error occurred, caught SIG$1"; + echo "An error occurred, caught SIG$1 on line $2"; fi echo "Cleaning up..." @@ -76,11 +76,11 @@ PROJECT_ID=${DSN_PIECES[1]} TEST_EVENT_ID=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-f0-9" | head -c 32) # Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk -curl --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"} }' \ +TEST_EVENT=$(curl --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' \ -H 'Content-Type: application/json' \ -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" \ - $SENTRY_TEST_HOST/api/$PROJECT_ID/store/ -sf -o /dev/null -echo "Created event $TEST_EVENT_ID." + $SENTRY_TEST_HOST/api/$PROJECT_ID/store/ -sf) +echo "Created event: $TEST_EVENT" EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" export -f sentry_api_request get_csrf_token @@ -89,7 +89,7 @@ printf "Checking its existence" timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' echo ""; -EVENT_RESPONSE=$(sentry_api_request "projects/sentry/internal/events/$TEST_EVENT_ID/") +EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH") declare -a EVENT_TEST_STRINGS=( '"eventID":"'"$TEST_EVENT_ID"'"' '"message":"a failure"' From 0ef9bf5721004363df4767cd8c18fe413a98afd6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 01:29:44 +0300 Subject: [PATCH 149/417] test(gcb): Even more debug info on fail --- test.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test.sh b/test.sh index eb6a07c1fa..b15f8e8273 100755 --- a/test.sh +++ b/test.sh @@ -24,6 +24,8 @@ cleanup () { if [ "$1" != "EXIT" ]; then echo "An error occurred, caught SIG$1 on line $2"; + docker-compose ps + docker-compose logs fi echo "Cleaning up..." @@ -76,11 +78,8 @@ PROJECT_ID=${DSN_PIECES[1]} TEST_EVENT_ID=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-f0-9" | head -c 32) # Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk -TEST_EVENT=$(curl --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' \ - -H 'Content-Type: application/json' \ - -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" \ - $SENTRY_TEST_HOST/api/$PROJECT_ID/store/ -sf) -echo "Created event: $TEST_EVENT" +echo "Creating test event..." +curl -f --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' -H 'Content-Type: application/json' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/store/" EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" export -f sentry_api_request get_csrf_token From bb2b37a06585e503fa1a8abe4f67222398585c0a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 01:44:07 +0300 Subject: [PATCH 150/417] test: Improve cosmetics, remove docker-compose logs --- test.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/test.sh b/test.sh index b15f8e8273..a1f33dd891 100755 --- a/test.sh +++ b/test.sh @@ -24,8 +24,6 @@ cleanup () { if [ "$1" != "EXIT" ]; then echo "An error occurred, caught SIG$1 on line $2"; - docker-compose ps - docker-compose logs fi echo "Cleaning up..." @@ -79,12 +77,12 @@ PROJECT_ID=${DSN_PIECES[1]} TEST_EVENT_ID=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-f0-9" | head -c 32) # Thanks @untitaker - https://forum.sentry.io/t/how-can-i-post-with-curl-a-sentry-event-which-authentication-credentials/4759/2?u=byk echo "Creating test event..." -curl -f --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' -H 'Content-Type: application/json' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/store/" +curl -sf --data '{"event_id": "'"$TEST_EVENT_ID"'","level":"error","message":"a failure","extra":{"object":"42"}}' -H 'Content-Type: application/json' -H "X-Sentry-Auth: Sentry sentry_version=7, sentry_key=$SENTRY_KEY, sentry_client=test-bash/0.1" "$SENTRY_TEST_HOST/api/$PROJECT_ID/store/" -o /dev/null EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" export -f sentry_api_request get_csrf_token export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH -printf "Checking its existence" +printf "Getting the test event back" timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' echo ""; From 911f17f0801fc917ef3b487f7c2254d19a5a0139 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Tue, 14 Jul 2020 14:48:22 -0400 Subject: [PATCH 151/417] feat: Enable performance views for on-premise (#586) The discover-basic and discover-query feature flags are enabled by default in the built-in sentry/conf/server.py Closes #582 --- sentry/sentry.conf.example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 216440df43..ec6489510d 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -246,6 +246,7 @@ def get_internal_network(): "organizations:sso-basic", "organizations:sso-rippling", "organizations:sso-saml2", + "organizations:performance-view", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups", From 1a554ba61b89e1b731b7a13c4e852d0cf89383bd Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 22:12:30 +0300 Subject: [PATCH 152/417] meta(release): Update BSL date on release (#584) --- scripts/bump-version.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 081af99794..73da616bb7 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -12,6 +12,7 @@ SYMBOLICATOR_VERSION=$(curl -sSL 'https://api.github.com/repos/getsentry/symboli sed -i -e "s/^SYMBOLICATOR_VERSION=.*\$/SYMBOLICATOR_VERSION=$SYMBOLICATOR_VERSION/" .env sed -i -e "s/^SENTRY_VERSION=.*\$/SENTRY_VERSION=$NEW_VERSION/" .env sed -i -e "s/^\# Sentry .* On-Premise/# Sentry $NEW_VERSION On-Premise/" README.md +sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" From 0a9f12404ad91f18f3a7add3893416bf285ea358 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 14 Jul 2020 22:19:47 +0300 Subject: [PATCH 153/417] fix(license): License change date should be 3 years later, not now Follow up to #584. --- scripts/bump-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 73da616bb7..2a8cd9def4 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -12,7 +12,7 @@ SYMBOLICATOR_VERSION=$(curl -sSL 'https://api.github.com/repos/getsentry/symboli sed -i -e "s/^SYMBOLICATOR_VERSION=.*\$/SYMBOLICATOR_VERSION=$SYMBOLICATOR_VERSION/" .env sed -i -e "s/^SENTRY_VERSION=.*\$/SENTRY_VERSION=$NEW_VERSION/" .env sed -i -e "s/^\# Sentry .* On-Premise/# Sentry $NEW_VERSION On-Premise/" README.md -sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d')/" LICENSE +sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" From 4f39b57a5311882a43952eff0b28f03769bac098 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 15 Jul 2020 20:37:51 +0300 Subject: [PATCH 154/417] install(clickhouse): Ensure we have the newest transactions table (#594) See getsentry/sentry#19882 and getsentry/sentry/#19883. Fixes #587. --- install.sh | 105 ++++++++++++++++++++++++++++++++++------------------- 1 file changed, 67 insertions(+), 38 deletions(-) diff --git a/install.sh b/install.sh index bdd73971c3..d391b2691f 100755 --- a/install.sh +++ b/install.sh @@ -21,10 +21,10 @@ SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' # Courtesy of https://stackoverflow.com/a/2183063/90297 trap_with_arg() { - func="$1" ; shift - for sig ; do - trap "$func $sig" "$sig" - done + func="$1" ; shift + for sig ; do + trap "$func $sig "'$LINENO' "$sig" + done } DID_CLEAN_UP=0 @@ -36,7 +36,7 @@ cleanup () { DID_CLEAN_UP=1 if [ "$1" != "EXIT" ]; then - echo "An error occurred, caught SIG$1"; + echo "An error occurred, caught SIG$1 on line $2"; echo "Cleaning up..." fi @@ -125,40 +125,40 @@ if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then fi replace_tsdb() { - if ( - [ -f "$SENTRY_CONFIG_PY" ] && - ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" - ); then - tsdb_settings="SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" - -# Automatic switchover 90 days after $(date). Can be removed afterwards. -SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}" - - if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then - echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" - else - echo "Attempting to automatically migrate to new TSDB" - # Escape newlines for sed - tsdb_settings="${tsdb_settings//$'\n'/\\n}" - cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" - sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true - - if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then - echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" - return - fi - - echo "Failed to automatically migrate TSDB. Reverting..." - mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" - echo "$SENTRY_CONFIG_PY restored from backup." - fi - - echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" - echo "" - echo "$tsdb_settings" - echo "" - echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" + if ( + [ -f "$SENTRY_CONFIG_PY" ] && + ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" + ); then + tsdb_settings="SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" + + # Automatic switchover 90 days after $(date). Can be removed afterwards. + SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}" + + if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then + echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" + else + echo "Attempting to automatically migrate to new TSDB" + # Escape newlines for sed + tsdb_settings="${tsdb_settings//$'\n'/\\n}" + cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" + sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true + + if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then + echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" + return + fi + + echo "Failed to automatically migrate TSDB. Reverting..." + mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" + echo "$SENTRY_CONFIG_PY restored from backup." fi + + echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" + echo "" + echo "$tsdb_settings" + echo "" + echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" + fi } replace_tsdb @@ -198,6 +198,35 @@ if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then fi fi +# [begin] Snuba/Clickhouse transactions table rebuild +clickhouse_query () { $dcr clickhouse clickhouse-client --host clickhouse -q "$1"; } +$dc up -d clickhouse +set +e +CLICKHOUSE_CLIENT_MAX_RETRY=5 +# Wait until clickhouse server is up +until clickhouse_query 'SELECT 1' > /dev/null; do + ((CLICKHOUSE_CLIENT_MAX_RETRY--)) + [[ CLICKHOUSE_CLIENT_MAX_RETRY -eq 0 ]] && echo "Clickhouse server failed to come up in 5 tries." && exit 1; + echo "Trying again. Remaining tries #$CLICKHOUSE_CLIENT_MAX_RETRY" + sleep 0.5; +done +set -e + +SNUBA_HAS_TRANSACTIONS_TABLE=$(clickhouse_query 'EXISTS TABLE transactions_local' | tr -d '\n\r') +SNUBA_TRANSACTIONS_NEEDS_UPDATE=$([ "$SNUBA_HAS_TRANSACTIONS_TABLE" == "1" ] && clickhouse_query 'SHOW CREATE TABLE transactions_local' | grep -v 'SAMPLE BY' || echo '') + +if [ "$SNUBA_TRANSACTIONS_NEEDS_UPDATE" ]; then + SNUBA_TRANSACTIONS_TABLE_CONTENTS=$(clickhouse_query "SELECT * FROM transactions_local LIMIT 1") + if [ -z $SNUBA_TRANSACTIONS_TABLE_CONTENTS ]; then + echo "Dropping the old transactions table from Clickhouse..."; + clickhouse_query 'DROP TABLE transactions_local' + echo "Done." + else + echo "Seems like your Clickhouse transactions table is old and non-empty. You may experience issues if/when you have more than 10000 records in this table. See https://github.com/getsentry/sentry/pull/19882 for more information and consider disabling the 'discover2.tags_facet_enable_sampling' feature flag."; + fi +fi +# [end] Snuba/Clickhouse transactions table rebuild + echo "Bootstrapping and migrating Snuba..." $dcr snuba-api bootstrap --force echo "" From cb83593d5bc4f2a0ba3aeafcd393b403a988f526 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 15 Jul 2020 21:00:37 +0300 Subject: [PATCH 155/417] feat(snuba): Add transactions consumer (#595) Needed for Performance to work (see #586). Depends on #593. Fixes #588. --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 40007504d6..d6bc992595 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -18,6 +18,7 @@ x-sentry-defaults: &sentry_defaults - snuba-consumer - snuba-outcomes-consumer - snuba-sessions-consumer + - snuba-transactions-consumer - snuba-replacer - symbolicator - kafka @@ -120,6 +121,10 @@ services: snuba-sessions-consumer: << : *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 + # Kafka consumer responsible for feeding transactions data into Clickhouse + snuba-transactions-consumer: + << : *snuba_defaults + command: consumer --storage transactions --auto-offset-reset=latest --max-batch-time-ms 750 snuba-replacer: << : *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 From 9330b36415638c1e3711c08f89155efd42d04212 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 15 Jul 2020 21:06:15 +0300 Subject: [PATCH 156/417] ci(craft): Fix Travis CI check name after .com migration --- .craft.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.craft.yml b/.craft.yml index 37c861bf4a..2e5567fe93 100644 --- a/.craft.yml +++ b/.craft.yml @@ -10,6 +10,6 @@ statusProvider: name: github config: contexts: - - 'continuous-integration/travis-ci/push' + - 'Travis CI - Branch' targets: - name: github From b0577a3b274d6b8863813a04057e049cf6e1bcaa Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 15 Jul 2020 22:42:46 +0300 Subject: [PATCH 157/417] ci(craft): Add delay between prepare and publish --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b08ddb0739..d4e87f2b8c 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,6 +26,10 @@ jobs: GIT_COMMITTER_NAME: getsentry-bot GIT_AUTHOR_NAME: getsentry-bot EMAIL: bot@getsentry.com + # Wait until the builds start. Craft should do this automatically + # but it is broken now. + # TODO: Remove this once getsentry/craft#111 is fixed + - run: sleep 10 - uses: getsentry/craft@master with: action: publish From 0bf688a0a6f86e0c7a1be0ad7a9fceaad7bdfa18 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 15 Jul 2020 22:53:56 +0300 Subject: [PATCH 158/417] ci: Increase timeout for getting test event Travis CI failed waiting on this multiple times so increase the grace period. Turns out during peak times things get slow. --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index a1f33dd891..f8ff082f8b 100755 --- a/test.sh +++ b/test.sh @@ -83,7 +83,7 @@ EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" export -f sentry_api_request get_csrf_token export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH printf "Getting the test event back" -timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' +timeout 30 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' echo ""; EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH") From e82506f5b7731164a03a02ce757988191ff8a52f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 17 Jul 2020 06:50:36 +0300 Subject: [PATCH 159/417] fix(snuba): Add consumer group to transactions consumer (#599) Without the consumer-group option, transactions consumer and events consumer will compete for messages on the same topic and usually events win, which is the first one. This may cause some data loss for performance and make it seem not work. Should address https://forum.sentry.io/t/perfomance-tracing-for-sentry-itself/10405/5?u=byk --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d6bc992595..5048ebc112 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -124,7 +124,7 @@ services: # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: << : *snuba_defaults - command: consumer --storage transactions --auto-offset-reset=latest --max-batch-time-ms 750 + command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 snuba-replacer: << : *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 From 86864d0edf7deb1e70608e551291587892618ed7 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 17 Jul 2020 15:02:45 +0300 Subject: [PATCH 160/417] feat(compose): Add ability to use custom images for any Sentry service (#602) This change allows one to override any Sentry service image, mostly for testing purposes. It also removes the SENTRY_VERSION variable as docker-compose makes it very hard to cascade default values for these. Next step is to have integration tests in getsentry/snuba and getsentry/relay (and possibly for getsentry/symbolicator) for getsentry/onpremise using this PR. Also related: #596. --- .env | 6 ++++-- docker-compose.yml | 11 +++++------ scripts/bump-version.sh | 4 ++-- sentry/Dockerfile | 3 +-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.env b/.env index aec59cb371..405bf676d1 100644 --- a/.env +++ b/.env @@ -1,4 +1,6 @@ COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 -SENTRY_VERSION=latest -SYMBOLICATOR_VERSION=latest +SENTRY_IMAGE=getsentry/sentry:latest +SNUBA_IMAGE=getsentry/snuba:latest +RELAY_IMAGE=getsentry/relay:latest +SYMBOLICATOR_IMAGE=getsentry/symbolicator:eac35a6058c7749bdf20ed219a377e49e02d0b76 diff --git a/docker-compose.yml b/docker-compose.yml index 5048ebc112..0ee8b9bcf9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,7 +7,6 @@ x-sentry-defaults: &sentry_defaults context: ./sentry args: - SENTRY_IMAGE - - SENTRY_VERSION image: sentry-onpremise-local depends_on: - redis @@ -34,7 +33,7 @@ x-snuba-defaults: &snuba_defaults - redis - clickhouse - kafka - image: 'getsentry/snuba:$SENTRY_VERSION' + image: '$SNUBA_IMAGE' environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse @@ -134,11 +133,11 @@ services: build: context: ./cron args: - BASE_IMAGE: 'getsentry/snuba:$SENTRY_VERSION' + BASE_IMAGE: '$SNUBA_IMAGE' command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: << : *restart_policy - image: 'getsentry/symbolicator:$SYMBOLICATOR_VERSION' + image: '$SYMBOLICATOR_IMAGE' volumes: - 'sentry-symbolicator:/data' - type: bind @@ -152,7 +151,7 @@ services: build: context: ./cron args: - BASE_IMAGE: 'getsentry/symbolicator:$SYMBOLICATOR_VERSION' + BASE_IMAGE: '$SYMBOLICATOR_IMAGE' command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - 'sentry-symbolicator:/data' @@ -194,7 +193,7 @@ services: - relay relay: << : *restart_policy - image: 'getsentry/relay:$SENTRY_VERSION' + image: '$RELAY_IMAGE' volumes: - type: bind read_only: true diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 2a8cd9def4..ca5173a991 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -9,8 +9,8 @@ NEW_VERSION="$2" SYMBOLICATOR_VERSION=$(curl -sSL 'https://api.github.com/repos/getsentry/symbolicator/git/refs/heads/master' | grep -Po '(?<=\"sha\": \")([a-f0-9]{5,40})(?=\",?)') -sed -i -e "s/^SYMBOLICATOR_VERSION=.*\$/SYMBOLICATOR_VERSION=$SYMBOLICATOR_VERSION/" .env -sed -i -e "s/^SENTRY_VERSION=.*\$/SENTRY_VERSION=$NEW_VERSION/" .env +sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Sentry .* On-Premise/# Sentry $NEW_VERSION On-Premise/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 406830edeb..f9484f295b 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,6 +1,5 @@ -ARG SENTRY_VERSION=latest ARG SENTRY_IMAGE -FROM ${SENTRY_IMAGE:-getsentry/sentry:$SENTRY_VERSION} +FROM ${SENTRY_IMAGE} COPY . /usr/src/sentry From 094d8e38a3decb530e3231969bd1f152ca9f0e0a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 17 Jul 2020 15:29:55 +0300 Subject: [PATCH 161/417] fix(install): Read and set .env in install.sh (#600) Fixes #597. --- install.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/install.sh b/install.sh index d391b2691f..9ccef09315 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,8 @@ #!/usr/bin/env bash set -e +source <(grep -v '^#' .env | sed -E 's|^(.+)=(.*)$|: ${\1=\2}; export \1|g') + dc="docker-compose --no-ansi" dcr="$dc run --rm" @@ -171,12 +173,8 @@ echo "" # redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true -if [ -z "$SENTRY_IMAGE" ]; then - docker pull getsentry/sentry:${SENTRY_VERSION:-latest} -else - # We may not have the set image on the repo (local images) so allow fails - docker pull $SENTRY_IMAGE || true; -fi +# We may not have the set image on the repo (local images) so allow fails +docker pull $SENTRY_IMAGE || true; echo "" echo "Building and tagging Docker images..." From 17d97e13c01246092bc702d7e81d15b88eae9192 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 17 Jul 2020 22:34:11 +0300 Subject: [PATCH 162/417] ci(release): Smarter and safer auto version setting If the release action is run without an explicit version in the same calendar month more than once, all of them will try to release the same version where the patch version is set to 0. This is never the intended action: if we are making a new release in the same month where an old one exists, it is 100% a patch release. This PR automatically implements patch version increment based on existing versions. --- .github/workflows/release.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d4e87f2b8c..23c825638b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -14,7 +14,13 @@ jobs: steps: - id: calver if: ${{ !github.event.client_payload.version }} - run: echo "::set-output name=version::$(date +'%y.%-m.0')" + run: | + DATE_PART=$(date +'%y.%-m') + PATCH_VERSION=0 + while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do + (( PATCH_VERSION++ )) + done + echo "::set-output name=version::"$DATE_PART.$PATCH_VERSION"" - uses: actions/checkout@v2 - uses: getsentry/craft@master if: ${{ !github.event.client_payload.skip_prepare }} From 4dbfcbcebe9d7ea54bd009bb85a331dc6ef51295 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 18 Jul 2020 01:26:47 +0300 Subject: [PATCH 163/417] ci(release): Fix quotes for auto version setting --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 23c825638b..a82b9ca002 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -20,7 +20,7 @@ jobs: while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do (( PATCH_VERSION++ )) done - echo "::set-output name=version::"$DATE_PART.$PATCH_VERSION"" + echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" - uses: actions/checkout@v2 - uses: getsentry/craft@master if: ${{ !github.event.client_payload.skip_prepare }} From 35ad7dc477a4b5ef1ee4a035364a68593e100ef6 Mon Sep 17 00:00:00 2001 From: Tomasz Kontusz Date: Mon, 3 Aug 2020 22:32:56 +0200 Subject: [PATCH 164/417] feat: Only stop the cluster after building images (#614) This reduces downtime for users with custom Dockerfiles. Fixes #607. --- install.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/install.sh b/install.sh index 9ccef09315..d80036e74e 100755 --- a/install.sh +++ b/install.sh @@ -92,12 +92,6 @@ if (($IS_KVM == 0)); then fi fi -# Clean up old stuff and ensure nothing is working while we install/update -# This is for older versions of on-premise: -$dc -p onpremise down --rmi local --remove-orphans -# This is for newer versions -$dc down --rmi local --remove-orphans - echo "" echo "Creating volumes for persistent storage..." echo "Created $(docker volume create --name=sentry-data)." @@ -185,6 +179,12 @@ $dc build --force-rm --parallel echo "" echo "Docker images built." +# Clean up old stuff and ensure nothing is working while we install/update +# This is for older versions of on-premise: +$dc -p onpremise down --rmi local --remove-orphans +# This is for newer versions +$dc down --rmi local --remove-orphans + ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') From 5d00d613fab7cf1a4ab7903b466e871c5a9a9015 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 12 Aug 2020 13:05:14 +0300 Subject: [PATCH 165/417] ci(release): Move to workflow_dispatch for UI-triggered releases --- .github/workflows/release.yml | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a82b9ca002..704dcb525f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,18 @@ name: release on: - repository_dispatch: - types: [release] + workflow_dispatch: + inputs: + version: + description: Version to release + required: false + skip_prepare: + description: Skip preparation step (assume a release branch is ready) + required: false + default: false + dry_run: + description: Do not actually cut the release + required: false + default: false schedule: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, @@ -13,22 +24,22 @@ jobs: name: "Release a new version" steps: - id: calver - if: ${{ !github.event.client_payload.version }} + if: ${{ !github.event.inputs.version }} run: | DATE_PART=$(date +'%y.%-m') - PATCH_VERSION=0 + declare -i PATCH_VERSION=0 while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do - (( PATCH_VERSION++ )) + PATCH_VERSION+=1 done echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" - uses: actions/checkout@v2 - uses: getsentry/craft@master - if: ${{ !github.event.client_payload.skip_prepare }} + if: ${{ !github.event.inputs.skip_prepare }} with: action: prepare - version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} + version: ${{ github.event.inputs.version || steps.calver.outputs.version }} env: - DRY_RUN: ${{ github.event.client_payload.dry_run }} + DRY_RUN: ${{ github.event.inputs.dry_run }} GIT_COMMITTER_NAME: getsentry-bot GIT_AUTHOR_NAME: getsentry-bot EMAIL: bot@getsentry.com @@ -39,11 +50,11 @@ jobs: - uses: getsentry/craft@master with: action: publish - version: ${{ github.event.client_payload.version || steps.calver.outputs.version }} + version: ${{ github.event.inputs.version || steps.calver.outputs.version }} keep_branch: '--keep-branch' no_merge: '--no-merge' env: - DRY_RUN: ${{ github.event.client_payload.dry_run }} + DRY_RUN: ${{ github.event.inputs.dry_run }} GIT_COMMITTER_NAME: getsentry-bot GIT_AUTHOR_NAME: getsentry-bot EMAIL: bot@getsentry.com From 9c8e2a29eb7ecc3e2f995bfa00222843f8b38edb Mon Sep 17 00:00:00 2001 From: buffcode Date: Fri, 14 Aug 2020 20:59:36 +0200 Subject: [PATCH 166/417] typo: Example configuration comment (#620) Fixes a small typo in a example configuration comment --- sentry/sentry.conf.example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index ec6489510d..6645860222 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -187,7 +187,7 @@ def get_internal_network(): SENTRY_WEB_OPTIONS = { "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), "protocol": "uwsgi", - # This is need to prevent https://git.io/fj7Lw + # This is needed in order to prevent https://git.io/fj7Lw "uwsgi-socket": None, "so-keepalive": True, # Keep this between 15s-75s as that's what Relay supports From cf350bd700e26ba0e96993e24faac025920df888 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 15 Aug 2020 16:40:08 +0300 Subject: [PATCH 167/417] "ci(release): Add killswitch via issues w/ release-blocker label Implements https://app.asana.com/0/1169344595888357/1146357826982899/f which would cancel the workflow (stop the release) when the repo has open issues with the label 'release-blocker'". --- .github/workflows/release.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 704dcb525f..89ba95f052 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,6 +23,12 @@ jobs: runs-on: ubuntu-latest name: "Release a new version" steps: + - id: killswitch + if: ${{ !github.event.inputs.force }} + run: | + if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then + curl -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel + fi - id: calver if: ${{ !github.event.inputs.version }} run: | From 1410d00296894116826100114857909dfb5a5a34 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 15 Aug 2020 16:45:45 +0300 Subject: [PATCH 168/417] ci(release): Add the killswitch option to workflow UI --- .github/workflows/release.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 89ba95f052..88d2f35ea1 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -13,6 +13,10 @@ on: description: Do not actually cut the release required: false default: false + force: + description: Force the release, bypassing the 'release-blocker' issue killswitch + required: false + default: false schedule: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, From 7bf83558155b320e34af4946c32f9325ba16987a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 15 Aug 2020 17:11:46 +0300 Subject: [PATCH 169/417] ci(release): Add note when killswitch is activated --- .github/workflows/release.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 88d2f35ea1..469e12c3bb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -31,7 +31,8 @@ jobs: if: ${{ !github.event.inputs.force }} run: | if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then - curl -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel + echo "Open release-blocking issues found, cancelling release..."; + curl -s -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' 'https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel'; fi - id: calver if: ${{ !github.event.inputs.version }} From 918cee5749b458a7d1218fc1b4a7062ca5322c34 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 15 Aug 2020 17:16:24 +0300 Subject: [PATCH 170/417] ci(release): Fix cancellation URL --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 469e12c3bb..8d99df9be4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -32,7 +32,7 @@ jobs: run: | if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then echo "Open release-blocking issues found, cancelling release..."; - curl -s -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' 'https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel'; + curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; fi - id: calver if: ${{ !github.event.inputs.version }} From 67f70915b59c0af3a03d324138f276484f04b275 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 17 Aug 2020 09:16:06 -0400 Subject: [PATCH 171/417] fix(env): Read and set .env in install.sh, portably (#626) h/t https://unix.stackexchange.com/a/79077 Fixes #622, cf. #600 cc: @NullIsNot0 --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index d80036e74e..6d64c26995 100755 --- a/install.sh +++ b/install.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -e -source <(grep -v '^#' .env | sed -E 's|^(.+)=(.*)$|: ${\1=\2}; export \1|g') +# With a tip o' the hat to https://unix.stackexchange.com/a/79077 +set -a && . ./.env && set +a dc="docker-compose --no-ansi" dcr="$dc run --rm" From fb125a1e4c40701b32f974f6eb2c46a05ca2cd78 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 17 Aug 2020 18:31:15 +0300 Subject: [PATCH 172/417] fix(install): TSDB migration should not create invalid config file (#631) Fixes #624 --- install.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 6d64c26995..9343a742e8 100755 --- a/install.sh +++ b/install.sh @@ -126,10 +126,14 @@ replace_tsdb() { [ -f "$SENTRY_CONFIG_PY" ] && ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" ); then - tsdb_settings="SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" - - # Automatic switchover 90 days after $(date). Can be removed afterwards. - SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}" + # Do NOT indent the following string as it would be reflected in the end result, + # breaking the final config file. See getsentry/onpremise#624. + tsdb_settings="\ +SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" + +# Automatic switchover 90 days after $(date). Can be removed afterwards. +SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}\ +" if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" From 4b8d4244772c22f23a72036a6a0d86ca55bd5831 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 20 Aug 2020 15:05:19 -0400 Subject: [PATCH 173/417] ci(github): Reimplement CI in GitHub Actions (#634) Part of #627, sets us up for expanding the OS matrix we test against. --- .github/workflows/test.yml | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 0000000000..8d864fbc5f --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,33 @@ +name: test +on: push +env: + DOCKER_COMPOSE_VERSION: 1.24.1 +jobs: + test: + runs-on: ubuntu-16.04 + name: "test" + steps: + + - name: Pin docker-compose + run: | + sudo rm /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + chmod +x docker-compose + sudo mv docker-compose /usr/local/bin + + - name: Checkout + uses: actions/checkout@v2 + + - name: Install and test + run: | + ./install.sh + docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST + docker-compose up -d + printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + ./test.sh + + - name: Inspect failure + if: failure() + run: | + docker-compose ps + docker-compose logs From a7b2ddbf2e717cec926331062404739488394197 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 21 Aug 2020 22:34:51 +0300 Subject: [PATCH 174/417] upgrade(clickhouse): Use the Clickhouse version we use in prod (#630) Co-authored-by: josh --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0ee8b9bcf9..b4e1cb000c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -96,7 +96,7 @@ services: - 'sentry-secrets:/etc/kafka/secrets' clickhouse: << : *restart_policy - image: 'yandex/clickhouse-server:19.17' + image: 'yandex/clickhouse-server:20.3.9.70' ulimits: nofile: soft: 262144 From e8d882ca0544d7549b51becf072adf28c53b9321 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Sun, 23 Aug 2020 14:34:01 -0400 Subject: [PATCH 175/417] ci(gha): Fine-tune GHA event bindings (#639) --- .github/workflows/test.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d864fbc5f..188dbdcb0b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,5 +1,13 @@ name: test -on: push +on: + # Run CI on all pushes to the master and release/** branches, and on all new + # pull requests, and on all pushes to pull requests (even if a pull request + # is not against master). + push: + branches: + - "master" + - "releases/**" + pull_request: env: DOCKER_COMPOSE_VERSION: 1.24.1 jobs: From 6a40e33073389ae431eb447a08ca07417ef10fc8 Mon Sep 17 00:00:00 2001 From: MK Date: Mon, 24 Aug 2020 01:56:06 -0400 Subject: [PATCH 176/417] Use the custom Nginx access log format (#642) If we are going to define a custom log format, we should probably be using that format. Fixes #635 --- nginx/nginx.conf | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index 84027d49b3..bd81a5e8df 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -17,6 +17,8 @@ http { '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; + access_log /var/log/nginx/access.log main; + sendfile on; tcp_nopush on; tcp_nodelay on; From 704e4c3b5b7360080f79bcfbe26583e5a95ae675 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 25 Aug 2020 22:10:29 +0300 Subject: [PATCH 177/417] meta(issues): Add issue templates (#645) https://app.asana.com/0/1169344595888357/1190341230301209/f Co-authored-by: Fiona <61481573+PeloWriter@users.noreply.github.com> --- .github/ISSUE_TEMPLATE/bug_report.md | 26 +++++++++++++++++++++ .github/ISSUE_TEMPLATE/config.yml | 8 +++++++ .github/ISSUE_TEMPLATE/feature_request.md | 28 +++++++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/config.yml create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000000..aeceea53c2 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,26 @@ +--- +name: 🐞 Bug Report +about: Report a bug to help improve Sentry On-Premise +--- + +## Version Information + +Version: *VERSION HERE* + + +## Description + +[What happened] + +## Steps to Reproduce + +1. [First Step] +2. [Second Step] +3. and so on. + +## Logs + +Please share any applicable logs: + +- ./install.sh logs +- `docker-compose logs` output diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 0000000000..22655b71b5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: Question about self-hosting/on-premise + url: https://forum.sentry.io + about: Please use the community forums for questions + - name: Report a security vulnerability + url: https://sentry.io/security/#vulnerability-disclosure + about: Please see our guide for responsible disclosure. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000000..bb8c4ee4d7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,28 @@ +--- +name: 🧠 Feature request +about: Suggest an idea for this project + +--- + + + +## Summary + +One paragraph description of the feature. + +## Motivation + +Why should this be worked on? What problems or use cases does it solve or +improve? + +## Additional Context + +Any other context or screenshots or API request payload/responses that +pertain to the feature. From 627c366f27379d4a29ca29367624866909eadcc1 Mon Sep 17 00:00:00 2001 From: Tomasz Kontusz Date: Thu, 27 Aug 2020 17:02:33 +0200 Subject: [PATCH 178/417] install: Keep relay available while upgrading (#615) This continues on the ideas from #607. By "downtime" here I mean "not accepting events" - web, smtp and background processes are out of scope. This PR adds a `--minimize-downtime` option to install.sh. This options changes the behaviour of the script by: 1. keeping nginx and relay running until the very end, 2. disabling cleanup on exit and failure, 3. explicitly reloading nginx configuration, 4. and starting the whole cluster at the end. The results are promising: no downtime if relay version doesn't change, and only a second when it does. So far this was only tested with a curl loop, so I'm still not sure if Relay flushes the events to Sentry before getting recreated by `$dc up`. --- .github/workflows/test.yml | 3 ++ .travis.yml | 3 ++ install.sh | 80 +++++++++++++++++++++++++++++++------- 3 files changed, 73 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 188dbdcb0b..6384cf11f9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,6 +33,9 @@ jobs: docker-compose up -d printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' ./test.sh + printf "Testing in-place upgrade" + ./install.sh --minimize-downtime + ./test.sh - name: Inspect failure if: failure() diff --git a/.travis.yml b/.travis.yml index eb61651167..6a54b57989 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,6 +16,9 @@ script: - docker-compose up -d - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh + - printf "Testing in-place upgrade" + - ./install.sh --minimize-downtime + - ./test.sh after_failure: - docker-compose ps diff --git a/install.sh b/install.sh index 9343a742e8..6a711f1366 100755 --- a/install.sh +++ b/install.sh @@ -21,6 +21,35 @@ SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' +MINIMIZE_DOWNTIME= + +load_options() { + while [[ -n "$@" ]]; do + case "$1" in + -h | --help) show_help; exit;; + --minimize-downtime) MINIMIZE_DOWNTIME=1;; + --) ;; + *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; + esac + shift + done +} + +show_help() { + cat < /dev/null + if [[ ! "$MINIMIZE_DOWNTIME" ]]; then + $dc stop &> /dev/null + fi } trap_with_arg cleanup ERR INT TERM EXIT @@ -184,11 +220,16 @@ $dc build --force-rm --parallel echo "" echo "Docker images built." -# Clean up old stuff and ensure nothing is working while we install/update -# This is for older versions of on-premise: -$dc -p onpremise down --rmi local --remove-orphans -# This is for newer versions -$dc down --rmi local --remove-orphans +if [[ "$MINIMIZE_DOWNTIME" ]]; then + # Stop everything but relay and nginx + $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') +else + # Clean up old stuff and ensure nothing is working while we install/update + # This is for older versions of on-premise: + $dc -p onpremise down --rmi local --remove-orphans + # This is for newer versions + $dc down --rmi local --remove-orphans +fi ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then @@ -292,9 +333,22 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi -echo "" -echo "----------------" -echo "You're all done! Run the following command to get Sentry running:" -echo "" -echo " docker-compose up -d" -echo "" +if [[ "$MINIMIZE_DOWNTIME" ]]; then + # Start the whole setup, except nginx and relay. + $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') + $dc exec -T nginx service nginx reload + + echo "Waiting for Sentry to start..." + docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ + -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' + + # Make sure everything is up. This should only touch relay and nginx + $dc up -d +else + echo "" + echo "----------------" + echo "You're all done! Run the following command to get Sentry running:" + echo "" + echo " docker-compose up -d" + echo "" +fi From a5593e99ae2623dc0b23e098e428e22ad81998ad Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 27 Aug 2020 18:26:39 +0300 Subject: [PATCH 179/417] ci: Remove TravisCI (#648) Since #634 and #639 we are using GitHub Actions for pushes and PRs and don't need Travis CI. Thanks for all the :fish: --- .craft.yml | 2 +- .travis.yml | 25 ------------------------- 2 files changed, 1 insertion(+), 26 deletions(-) delete mode 100644 .travis.yml diff --git a/.craft.yml b/.craft.yml index 2e5567fe93..2ad9b373d0 100644 --- a/.craft.yml +++ b/.craft.yml @@ -10,6 +10,6 @@ statusProvider: name: github config: contexts: - - 'Travis CI - Branch' + - 'test' targets: - name: github diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 6a54b57989..0000000000 --- a/.travis.yml +++ /dev/null @@ -1,25 +0,0 @@ -language: bash -services: docker - -env: - - DOCKER_COMPOSE_VERSION=1.24.1 - -before_install: - - sudo rm /usr/local/bin/docker-compose - - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - - chmod +x docker-compose - - sudo mv docker-compose /usr/local/bin - -script: - - ./install.sh - - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - - docker-compose up -d - - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - - ./test.sh - - printf "Testing in-place upgrade" - - ./install.sh --minimize-downtime - - ./test.sh - -after_failure: - - docker-compose ps - - docker-compose logs From 175ad090570e288025c32e2214c745af589a41dc Mon Sep 17 00:00:00 2001 From: Kevin Waddle Date: Thu, 27 Aug 2020 10:32:20 -0500 Subject: [PATCH 180/417] install: Add flag to skip create user prompt during install (#646) Fixes #611. --- README.md | 2 ++ install.sh | 4 +++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c055c05c0b..615084ccf0 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. +During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --no-user-prompt`. + There may need to be modifications to the included example config files (`sentry/config.example.yml` and `sentry/sentry.conf.example.py`) to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script and copy them without the `.example` extensions in the name (such as `sentry/sentry.conf.py`) before running the `install.sh` script. The recommended way to customize your configuration is using the files below, in that order: diff --git a/install.sh b/install.sh index 6a711f1366..19d3b8bfc6 100755 --- a/install.sh +++ b/install.sh @@ -27,6 +27,7 @@ load_options() { while [[ -n "$@" ]]; do case "$1" in -h | --help) show_help; exit;; + --no-user-prompt) SKIP_USER_PROMPT=1;; --minimize-downtime) MINIMIZE_DOWNTIME=1;; --) ;; *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; @@ -43,6 +44,7 @@ Install Sentry with docker-compose. Options: -h, --help Show this message and exit. + --no-user-prompt Skips the initial user creation prompt (ideal for non-interactive installs). --minimize-downtime EXPERIMENTAL: try to keep accepting events for as long as possible while upgrading. This will disable cleanup on error, and might leave your installation in partially upgraded state. This option might not reload all configuration, and is only meant for in-place upgrades. @@ -298,7 +300,7 @@ fi echo "" echo "Setting up database..." -if [ $CI ]; then +if [ $CI ] || [ $SKIP_USER_PROMPT == 1 ]; then $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." From cb8bb3b2e7d439fd43d0723e1de2c04b655b3617 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 27 Aug 2020 20:27:55 +0300 Subject: [PATCH 181/417] ci: Disable beacon on CI e2e tests (#649) This also brings in common pre-test commands into test.sh file. --- .github/workflows/test.yml | 3 --- test.sh | 8 +++++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6384cf11f9..df81a20803 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,9 +29,6 @@ jobs: - name: Install and test run: | ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' ./test.sh printf "Testing in-place upgrade" ./install.sh --minimize-downtime diff --git a/test.sh b/test.sh index f8ff082f8b..a55d3d8c43 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" +export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) @@ -32,6 +32,12 @@ cleanup () { } trap_with_arg cleanup ERR INT TERM EXIT +# Disable beacon for e2e tests +echo 'SENTRY_BEACON=False' >> sentry/sentry.conf.py +docker-compose run --rm web createuser --superuser --email $TEST_USER --password $TEST_PASS || true +docker-compose up -d +printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' + get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } From 61bc028a176570ab8c9231a534f68c7c5cb1e862 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 27 Aug 2020 21:45:54 +0300 Subject: [PATCH 182/417] ci(craft): Remove the delay between prepare and publish This should be obsolete by getsentry/craft#117. --- .github/workflows/release.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8d99df9be4..d1d289026f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,10 +54,6 @@ jobs: GIT_COMMITTER_NAME: getsentry-bot GIT_AUTHOR_NAME: getsentry-bot EMAIL: bot@getsentry.com - # Wait until the builds start. Craft should do this automatically - # but it is broken now. - # TODO: Remove this once getsentry/craft#111 is fixed - - run: sleep 10 - uses: getsentry/craft@master with: action: publish From 1a9b45fb9f0439801555d0ad4a76bab12300d410 Mon Sep 17 00:00:00 2001 From: Tomasz Kontusz Date: Mon, 31 Aug 2020 13:40:52 +0200 Subject: [PATCH 183/417] EditorConfig and indentation fixes (#650) EditorConfig based on conversation in https://github.com/getsentry/onpremise/pull/615#discussion_r477338006, and with the indentation fixes. This PR will probably conflict with everything, but at least the code is a little bit more consistent :-D --- .editorconfig | 16 ++++++++ .github/workflows/release.yml | 76 +++++++++++++++++------------------ cron/entrypoint.sh | 4 +- install.sh | 76 +++++++++++++++++------------------ relay/config.example.yml | 2 +- test.sh | 62 ++++++++++++++-------------- 6 files changed, 126 insertions(+), 110 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000000..4069959185 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,16 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_style = space +insert_final_newline = true + +[*.sh] +indent_size = 2 + +[*.yml] +indent_size = 2 + +[nginx/*.conf] +indent_style = tab diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index d1d289026f..fc3e3ac119 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,41 +27,41 @@ jobs: runs-on: ubuntu-latest name: "Release a new version" steps: - - id: killswitch - if: ${{ !github.event.inputs.force }} - run: | - if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then - echo "Open release-blocking issues found, cancelling release..."; - curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; - fi - - id: calver - if: ${{ !github.event.inputs.version }} - run: | - DATE_PART=$(date +'%y.%-m') - declare -i PATCH_VERSION=0 - while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do - PATCH_VERSION+=1 - done - echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" - - uses: actions/checkout@v2 - - uses: getsentry/craft@master - if: ${{ !github.event.inputs.skip_prepare }} - with: - action: prepare - version: ${{ github.event.inputs.version || steps.calver.outputs.version }} - env: - DRY_RUN: ${{ github.event.inputs.dry_run }} - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com - - uses: getsentry/craft@master - with: - action: publish - version: ${{ github.event.inputs.version || steps.calver.outputs.version }} - keep_branch: '--keep-branch' - no_merge: '--no-merge' - env: - DRY_RUN: ${{ github.event.inputs.dry_run }} - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com + - id: killswitch + if: ${{ !github.event.inputs.force }} + run: | + if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then + echo "Open release-blocking issues found, cancelling release..."; + curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; + fi + - id: calver + if: ${{ !github.event.inputs.version }} + run: | + DATE_PART=$(date +'%y.%-m') + declare -i PATCH_VERSION=0 + while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do + PATCH_VERSION+=1 + done + echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" + - uses: actions/checkout@v2 + - uses: getsentry/craft@master + if: ${{ !github.event.inputs.skip_prepare }} + with: + action: prepare + version: ${{ github.event.inputs.version || steps.calver.outputs.version }} + env: + DRY_RUN: ${{ github.event.inputs.dry_run }} + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com + - uses: getsentry/craft@master + with: + action: publish + version: ${{ github.event.inputs.version || steps.calver.outputs.version }} + keep_branch: '--keep-branch' + no_merge: '--no-merge' + env: + DRY_RUN: ${{ github.event.inputs.dry_run }} + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh index b0f4d5b75e..baa833a77b 100755 --- a/cron/entrypoint.sh +++ b/cron/entrypoint.sh @@ -9,7 +9,7 @@ declare -p | grep -Ev 'BASHOPTS|BASH_VERSINFO|EUID|PPID|SHELLOPTS|UID' > /contai { for cron_job in "$@"; do echo -e "SHELL=/bin/bash BASH_ENV=/container.env ${cron_job} > /proc/1/fd/1 2>/proc/1/fd/2"; done } \ - | sed --regexp-extended 's/\\(.)/\1/g' \ - | crontab - + | sed --regexp-extended 's/\\(.)/\1/g' \ + | crontab - crontab -l exec cron -f -l -L 15 diff --git a/install.sh b/install.sh index 19d3b8bfc6..bd3587adf2 100755 --- a/install.sh +++ b/install.sh @@ -26,11 +26,11 @@ MINIMIZE_DOWNTIME= load_options() { while [[ -n "$@" ]]; do case "$1" in - -h | --help) show_help; exit;; - --no-user-prompt) SKIP_USER_PROMPT=1;; - --minimize-downtime) MINIMIZE_DOWNTIME=1;; - --) ;; - *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; + -h | --help) show_help; exit;; + --no-user-prompt) SKIP_USER_PROMPT=1;; + --minimize-downtime) MINIMIZE_DOWNTIME=1;; + --) ;; + *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; esac shift done @@ -57,7 +57,7 @@ load_options $(getopt -n "$0" -o 'h' -l 'help,minimize-downtime' -- "$@") trap_with_arg() { func="$1" ; shift for sig ; do - trap "$func $sig "'$LINENO' "$sig" + trap "$func $sig "'$LINENO' "$sig" done } @@ -106,18 +106,18 @@ function ensure_file_from_example { } if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then - echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" - exit 1 + echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" + exit 1 fi if [ $(ver $COMPOSE_VERSION) -lt $(ver $MIN_COMPOSE_VERSION) ]; then - echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" - exit 1 + echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" + exit 1 fi if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then - echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" - exit 1 + echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" + exit 1 fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) @@ -149,14 +149,14 @@ ensure_file_from_example $SYMBOLICATOR_CONFIG_YML ensure_file_from_example $RELAY_CONFIG_YML if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then - echo "" - echo "Generating secret key..." - # This is to escape the secret key to be used in sed below - # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode - # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 - SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') - sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML - echo "Secret key written to $SENTRY_CONFIG_YML" + echo "" + echo "Generating secret key..." + # This is to escape the secret key to be used in sed below + # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode + # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 + SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') + sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML + echo "Secret key written to $SENTRY_CONFIG_YML" fi replace_tsdb() { @@ -253,7 +253,7 @@ CLICKHOUSE_CLIENT_MAX_RETRY=5 until clickhouse_query 'SELECT 1' > /dev/null; do ((CLICKHOUSE_CLIENT_MAX_RETRY--)) [[ CLICKHOUSE_CLIENT_MAX_RETRY -eq 0 ]] && echo "Clickhouse server failed to come up in 5 tries." && exit 1; - echo "Trying again. Remaining tries #$CLICKHOUSE_CLIENT_MAX_RETRY" + echo "Trying again. Remaining tries #$CLICKHOUSE_CLIENT_MAX_RETRY" sleep 0.5; done set -e @@ -279,23 +279,23 @@ echo "" # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then - docker volume rm sentry-postgres-new || true - # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume - docker run --rm \ - -v sentry-postgres:/var/lib/postgresql/9.5/data \ - -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ - tianon/postgres-upgrade:9.5-to-9.6 - - # Get rid of the old volume as we'll rename the new one to that - docker volume rm sentry-postgres - docker volume create --name sentry-postgres - # There's no rename volume in Docker so copy the contents from old to new name - # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` - # doesn't do that automatically. - docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ - "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" - # Finally, remove the new old volume as we are all in sentry-postgres now - docker volume rm sentry-postgres-new + docker volume rm sentry-postgres-new || true + # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume + docker run --rm \ + -v sentry-postgres:/var/lib/postgresql/9.5/data \ + -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ + tianon/postgres-upgrade:9.5-to-9.6 + + # Get rid of the old volume as we'll rename the new one to that + docker volume rm sentry-postgres + docker volume create --name sentry-postgres + # There's no rename volume in Docker so copy the contents from old to new name + # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` + # doesn't do that automatically. + docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ + "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" + # Finally, remove the new old volume as we are all in sentry-postgres now + docker volume rm sentry-postgres-new fi echo "" diff --git a/relay/config.example.yml b/relay/config.example.yml index f54c9348ea..0488ba91a4 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -3,7 +3,7 @@ relay: host: 0.0.0.0 port: 3000 logging: - level: WARN + level: WARN processing: enabled: true kafka_config: diff --git a/test.sh b/test.sh index a55d3d8c43..9d11d9df24 100755 --- a/test.sh +++ b/test.sh @@ -8,10 +8,10 @@ COOKIE_FILE=$(mktemp) # Courtesy of https://stackoverflow.com/a/2183063/90297 trap_with_arg() { - func="$1" ; shift - for sig ; do - trap "$func $sig "'$LINENO' "$sig" - done + func="$1" ; shift + for sig ; do + trap "$func $sig "'$LINENO' "$sig" + done } DID_CLEAN_UP=0 @@ -42,33 +42,33 @@ get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } login () { - INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) - if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then - echo "Initial /auth/login/ redirect failed, exiting..." - echo "$INITIAL_AUTH_REDIRECT" - exit -1 - fi - - CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' - /csrfmiddlewaretoken/ { - print $4 "=" $6; - exit; - }') - - curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; + INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) + if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then + echo "Initial /auth/login/ redirect failed, exiting..." + echo "$INITIAL_AUTH_REDIRECT" + exit -1 + fi + + CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' + /csrfmiddlewaretoken/ { + print $4 "=" $6; + exit; + }') + + curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; } LOGIN_RESPONSE=$(login); declare -a LOGIN_TEST_STRINGS=( - '"isAuthenticated":true' - '"username":"test@example.com"' - '"isSuperuser":true' + '"isAuthenticated":true' + '"username":"test@example.com"' + '"isSuperuser":true' ) for i in "${LOGIN_TEST_STRINGS[@]}" do - echo "Testing '$i'..." - echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null - echo "Pass." + echo "Testing '$i'..." + echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "Pass." done # Set up initial/required settings (InstallWizard request) @@ -94,14 +94,14 @@ echo ""; EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH") declare -a EVENT_TEST_STRINGS=( - '"eventID":"'"$TEST_EVENT_ID"'"' - '"message":"a failure"' - '"title":"a failure"' - '"object":"42"' + '"eventID":"'"$TEST_EVENT_ID"'"' + '"message":"a failure"' + '"title":"a failure"' + '"object":"42"' ) for i in "${EVENT_TEST_STRINGS[@]}" do - echo "Testing '$i'..." - echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null - echo "Pass." + echo "Testing '$i'..." + echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "Pass." done From d55e7dbae3d4e8dd8bc40edf6823b6b735dd30f0 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 1 Sep 2020 23:51:49 +0300 Subject: [PATCH 184/417] meta(readme): Update build badge from TravisCI to GHA Follow up to #648 --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 615084ccf0..32c89c62f8 100644 --- a/README.md +++ b/README.md @@ -64,5 +64,5 @@ The included `install.sh` script is meant to be idempotent and to bring you to t * [Community Forums](https://forum.sentry.io/c/on-premise) -[build-status-image]: https://api.travis-ci.com/getsentry/onpremise.svg?branch=master -[build-status-url]: https://travis-ci.com/getsentry/onpremise +[build-status-image]: https://github.com/getsentry/onpremise/workflows/test/badge.svg +[build-status-url]: https://git.io/JUYkh From 5bcc795964c80ff40eaee17238ce04214ebc5a03 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 1 Sep 2020 23:53:34 +0300 Subject: [PATCH 185/417] meta(readme): Point documentation URL to the new develop site Follow up to getsentry/develop#123 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 32c89c62f8..7d7504712a 100644 --- a/README.md +++ b/README.md @@ -59,7 +59,7 @@ The included `install.sh` script is meant to be idempotent and to bring you to t ## Resources - * [Documentation](https://docs.sentry.io/development/server/) + * [Documentation](https://develop.sentry.dev/onpremise/) * [Bug Tracker](https://github.com/getsentry/onpremise/issues) * [Community Forums](https://forum.sentry.io/c/on-premise) From 59db1f2957505955b7a5ba5b8eb9b9e7b39bce43 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 3 Sep 2020 21:52:26 +0300 Subject: [PATCH 186/417] ref(symbolicator): Tag the latest symbolicator release for releases (#654) --- scripts/bump-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index ca5173a991..9805678569 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -7,7 +7,7 @@ cd $SCRIPT_DIR/.. OLD_VERSION="$1" NEW_VERSION="$2" -SYMBOLICATOR_VERSION=$(curl -sSL 'https://api.github.com/repos/getsentry/symbolicator/git/refs/heads/master' | grep -Po '(?<=\"sha\": \")([a-f0-9]{5,40})(?=\",?)') +SYMBOLICATOR_VERSION=$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")') sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env From 98b011ae88edc7a9b09ddacbacd6540f4786eecf Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 4 Sep 2020 22:49:56 +0300 Subject: [PATCH 187/417] fix[readme): Don't recommend nginx in docker-compose (#657) Fixes an issue where we recommend adding TLS terminating Nginx into the docker-compose file. This is no longer recommended as we already have an nginx instance there for routing purposes. Fixes #653. --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7d7504712a..d37ed366f9 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Sentry comes with a cleanup cron job that prunes events older than `90 days` by If you'd like to protect your Sentry install with SSL/TLS, there are fantastic SSL/TLS proxies like [HAProxy](http://www.haproxy.org/) -and [Nginx](http://nginx.org/). You'll likely want to add this service to your `docker-compose.yml` file. +and [Nginx](http://nginx.org/). Our recommendation is running and external Nginx instance or your choice of load balancer that does the TLS termination and more. Read more over at our [productionalizing self-hosted docs](https://develop.sentry.dev/self-hosted/#productionalizing). ## Updating Sentry From e293a0fc97da5017c63e0a55605914622533e758 Mon Sep 17 00:00:00 2001 From: Tomasz Kontusz Date: Mon, 7 Sep 2020 10:04:43 +0200 Subject: [PATCH 188/417] fix(install): Don't use getopt for options parsing (#660) The code assumed GNU getopt, which is not the default on many platforms, notably OSX. Fixes #656, and replaces #659. --- install.sh | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/install.sh b/install.sh index bd3587adf2..e3881586f2 100755 --- a/install.sh +++ b/install.sh @@ -23,22 +23,9 @@ RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' MINIMIZE_DOWNTIME= -load_options() { - while [[ -n "$@" ]]; do - case "$1" in - -h | --help) show_help; exit;; - --no-user-prompt) SKIP_USER_PROMPT=1;; - --minimize-downtime) MINIMIZE_DOWNTIME=1;; - --) ;; - *) echo "Unexpected argument: $1. Use --help for usage information."; exit 1;; - esac - shift - done -} - show_help() { cat < Date: Tue, 8 Sep 2020 18:52:46 +0300 Subject: [PATCH 189/417] fix(release): Bump the license date on master w/ calver releases (#658) --- .github/workflows/release.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index fc3e3ac119..6bc87c1aff 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -44,6 +44,8 @@ jobs: done echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" - uses: actions/checkout@v2 + with: + token: ${{ secrets.GH_SENTRY_BOT_PAT }} - uses: getsentry/craft@master if: ${{ !github.event.inputs.skip_prepare }} with: @@ -65,3 +67,12 @@ jobs: GIT_COMMITTER_NAME: getsentry-bot GIT_AUTHOR_NAME: getsentry-bot EMAIL: bot@getsentry.com + - id: bump-license-date + if: ${{ !github.event.inputs.dry_run && !github.event.inputs.version }} + env: + GIT_COMMITTER_NAME: getsentry-bot + GIT_AUTHOR_NAME: getsentry-bot + EMAIL: bot@getsentry.com + run: | + sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE + git diff --quiet || git commit -anm 'license: Update BSL change date' && git push From d83432996d963e0a94f6e9077dcba42488a80e04 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 8 Sep 2020 21:06:09 +0300 Subject: [PATCH 190/417] feat(clickhouse): Reduce max memory usage to 30% of RAM (#662) Closes #616, supersedes #651 Adds an option to reduce max memory usage of Clickhouse server. Sets it to 30% of all available RAM as the default. Co-authored-by: Renaud Chaput --- clickhouse/config.xml | 3 +++ docker-compose.yml | 9 +++++++++ 2 files changed, 12 insertions(+) create mode 100644 clickhouse/config.xml diff --git a/clickhouse/config.xml b/clickhouse/config.xml new file mode 100644 index 0000000000..4cd44b7b4a --- /dev/null +++ b/clickhouse/config.xml @@ -0,0 +1,3 @@ + + + diff --git a/docker-compose.yml b/docker-compose.yml index b4e1cb000c..f6818c6882 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -104,6 +104,15 @@ services: volumes: - 'sentry-clickhouse:/var/lib/clickhouse' - 'sentry-clickhouse-log:/var/log/clickhouse-server' + - type: bind + read_only: true + source: ./clickhouse/config.xml + target: /etc/clickhouse-server/config.d/sentry.xml + environment: + # This limits Clickhouse's memory to 30% of the host memory + # If you have high volume and your search return incomplete results + # You might want to change this to a higher value (and ensure your host has enough memory) + MAX_MEMORY_USAGE_RATIO: 0.3 snuba-api: << : *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse From 9b9066fc02543bb1566d8128ca33daf27b1ce921 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Tue, 8 Sep 2020 11:17:55 -0700 Subject: [PATCH 191/417] feat(snuba): Run new migration system (#663) This is for the onpremise release on Sept 15th. The new migration system has a migration to handle recreating the transaction table if the old one is present, we no longer need to do this in `install.sh`. --- install.sh | 32 ++------------------------------ 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/install.sh b/install.sh index e3881586f2..b440576f3d 100755 --- a/install.sh +++ b/install.sh @@ -240,37 +240,9 @@ if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then fi fi -# [begin] Snuba/Clickhouse transactions table rebuild -clickhouse_query () { $dcr clickhouse clickhouse-client --host clickhouse -q "$1"; } -$dc up -d clickhouse -set +e -CLICKHOUSE_CLIENT_MAX_RETRY=5 -# Wait until clickhouse server is up -until clickhouse_query 'SELECT 1' > /dev/null; do - ((CLICKHOUSE_CLIENT_MAX_RETRY--)) - [[ CLICKHOUSE_CLIENT_MAX_RETRY -eq 0 ]] && echo "Clickhouse server failed to come up in 5 tries." && exit 1; - echo "Trying again. Remaining tries #$CLICKHOUSE_CLIENT_MAX_RETRY" - sleep 0.5; -done -set -e - -SNUBA_HAS_TRANSACTIONS_TABLE=$(clickhouse_query 'EXISTS TABLE transactions_local' | tr -d '\n\r') -SNUBA_TRANSACTIONS_NEEDS_UPDATE=$([ "$SNUBA_HAS_TRANSACTIONS_TABLE" == "1" ] && clickhouse_query 'SHOW CREATE TABLE transactions_local' | grep -v 'SAMPLE BY' || echo '') - -if [ "$SNUBA_TRANSACTIONS_NEEDS_UPDATE" ]; then - SNUBA_TRANSACTIONS_TABLE_CONTENTS=$(clickhouse_query "SELECT * FROM transactions_local LIMIT 1") - if [ -z $SNUBA_TRANSACTIONS_TABLE_CONTENTS ]; then - echo "Dropping the old transactions table from Clickhouse..."; - clickhouse_query 'DROP TABLE transactions_local' - echo "Done." - else - echo "Seems like your Clickhouse transactions table is old and non-empty. You may experience issues if/when you have more than 10000 records in this table. See https://github.com/getsentry/sentry/pull/19882 for more information and consider disabling the 'discover2.tags_facet_enable_sampling' feature flag."; - fi -fi -# [end] Snuba/Clickhouse transactions table rebuild - echo "Bootstrapping and migrating Snuba..." -$dcr snuba-api bootstrap --force +$dcr snuba-api bootstrap --no-migrate --force +$dcr snuba-api migrations migrate --force echo "" # Very naively check whether there's an existing sentry-postgres volume and the PG version in it From 23251e79b978436822c1017ab4a2f94d47884e7d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 8 Sep 2020 22:03:04 +0300 Subject: [PATCH 192/417] ci(gha): Fix intermittent test failures (#664) GitHub team pointed us to docker/compose#3586 as the likely root cause and some digging around revealed got us to https://git.io/JUn7p as a potential fix, which we are trying here. --- .github/workflows/test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index df81a20803..34255559ef 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,6 @@ jobs: runs-on: ubuntu-16.04 name: "test" steps: - - name: Pin docker-compose run: | sudo rm /usr/local/bin/docker-compose @@ -27,6 +26,8 @@ jobs: uses: actions/checkout@v2 - name: Install and test + env: + COMPOSE_PARALLEL_LIMIT: 10 run: | ./install.sh ./test.sh From 9b3aee1b1fc18bc6f4f76df4e4c40aaca9f82736 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 9 Sep 2020 22:30:27 +0300 Subject: [PATCH 193/417] ref(self-hosted): More on-premise -> self-hosted (#669) --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- README.md | 2 +- scripts/bump-version.sh | 2 +- sentry/sentry.conf.example.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index aeceea53c2..eaaf895b29 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,6 +1,6 @@ --- name: 🐞 Bug Report -about: Report a bug to help improve Sentry On-Premise +about: Report a bug to help improve Self-Hosted Sentry --- ## Version Information diff --git a/README.md b/README.md index d37ed366f9..8cab9df650 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Sentry Nightly On-Premise [![Build Status][build-status-image]][build-status-url] +# Self-Hosted Sentry Nightly [![Build Status][build-status-image]][build-status-url] Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 9805678569..4172c7e4d3 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -11,7 +11,7 @@ SYMBOLICATOR_VERSION=$(curl -s "https://api.github.com/repos/getsentry/symbolica sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env -sed -i -e "s/^\# Sentry .* On-Premise/# Sentry $NEW_VERSION On-Premise/" README.md +sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 6645860222..9136045864 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -80,7 +80,7 @@ def get_internal_network(): # Queue # ######### -# See https://docs.getsentry.com/on-premise/server/queue/ for more +# See https://develop.sentry.dev/services/queue/ for more # information on configuring your queue broker and workers. Sentry relies # on a Python framework called Celery to manage queues. From 313fef026a675ddc1d851a59b45b485c78d61e8f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 11 Sep 2020 13:29:42 +0300 Subject: [PATCH 194/417] fix(symbolicator): Use the latest Symbolicator version for nightlies (#670) --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index 405bf676d1..28c5a2c614 100644 --- a/.env +++ b/.env @@ -3,4 +3,4 @@ SENTRY_EVENT_RETENTION_DAYS=90 SENTRY_IMAGE=getsentry/sentry:latest SNUBA_IMAGE=getsentry/snuba:latest RELAY_IMAGE=getsentry/relay:latest -SYMBOLICATOR_IMAGE=getsentry/symbolicator:eac35a6058c7749bdf20ed219a377e49e02d0b76 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:latest From d6247e23743667d2047c14088f7115726c1d1e3c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 14 Sep 2020 22:41:53 +0300 Subject: [PATCH 195/417] feat(env): Add SENTRY_BIND var (#512) Closes #279, supercedes #306. This is much simpler and safer now that we have `nginx` in front of everything. Thanks a lot @larsnystrom! --- .env | 3 +++ docker-compose.yml | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 28c5a2c614..85ca9bc038 100644 --- a/.env +++ b/.env @@ -1,5 +1,8 @@ COMPOSE_PROJECT_NAME=sentry_onpremise SENTRY_EVENT_RETENTION_DAYS=90 +# You can either use a port number or an IP:PORT combo for SENTRY_BIND +# See https://docs.docker.com/compose/compose-file/#ports for more +SENTRY_BIND=9000 SENTRY_IMAGE=getsentry/sentry:latest SNUBA_IMAGE=getsentry/snuba:latest RELAY_IMAGE=getsentry/relay:latest diff --git a/docker-compose.yml b/docker-compose.yml index f6818c6882..29593eaeb5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -190,7 +190,7 @@ services: nginx: << : *restart_policy ports: - - '9000:80/tcp' + - '$SENTRY_BIND:80/tcp' image: 'nginx:1.16' volumes: - type: bind From 9a18a4a366fac9c131d8a7b43ee8d1a8c75c7124 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 14 Sep 2020 23:03:44 +0300 Subject: [PATCH 196/417] fix(kafka): Reduce Kafka resource usage (#674) Fixes #502 and applies the suggestions from there: - Number of partitons=1 (from 40) - Log retention to 1 day (from 7 days) These settings should be more suited towards the scale this repo is intended for. NOTE: The partition count change will only affect new installs unless `sentry-kafka` and related volumes are cleaned. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 29593eaeb5..4ac2ee442d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -84,6 +84,8 @@ services: KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: '1' + KAFKA_LOG_RETENTION_HOURS: '24' KAFKA_MESSAGE_MAX_BYTES: '50000000' #50MB or bust KAFKA_MAX_REQUEST_SIZE: '50000000' #50MB on requests apparently too CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' From 5631d4511543dbefaee3ca4425ff746f7f285002 Mon Sep 17 00:00:00 2001 From: Bruno Bronosky Date: Mon, 14 Sep 2020 15:19:20 -0500 Subject: [PATCH 197/417] fix(install): Use proper bash testing (#673) Fixes https://github.com/getsentry/onpremise/issues/672 I split this PR up into 4 commits. The first one is the bare minimum for the issue. The rest are just consistency corrections that we neckbeards at irc://chat.freenode.net/%23bash would always make. --- install.sh | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/install.sh b/install.sh index b440576f3d..e94621bfa5 100755 --- a/install.sh +++ b/install.sh @@ -60,22 +60,22 @@ trap_with_arg() { DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { - if [ "$DID_CLEAN_UP" -eq 1 ]; then + if [[ "$DID_CLEAN_UP" -eq 1 ]]; then return 0; fi DID_CLEAN_UP=1 - if [ "$1" != "EXIT" ]; then + if [[ "$1" != "EXIT" ]]; then echo "An error occurred, caught SIG$1 on line $2"; - if [[ "$MINIMIZE_DOWNTIME" ]]; then + if [[ -n "$MINIMIZE_DOWNTIME" ]]; then echo "*NOT* cleaning up, to clean your environment run \"docker-compose stop\"." else echo "Cleaning up..." fi fi - if [[ ! "$MINIMIZE_DOWNTIME" ]]; then + if [[ -z "$MINIMIZE_DOWNTIME" ]]; then $dc stop &> /dev/null fi } @@ -93,7 +93,7 @@ function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } # Thanks to https://stackoverflow.com/a/25123013/90297 for the quick `sed` pattern function ensure_file_from_example { - if [ -f "$1" ]; then + if [[ -f "$1" ]]; then echo "$1 already exists, skipped creation." else echo "Creating $1..." @@ -101,17 +101,17 @@ function ensure_file_from_example { fi } -if [ $(ver $DOCKER_VERSION) -lt $(ver $MIN_DOCKER_VERSION) ]; then +if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit 1 fi -if [ $(ver $COMPOSE_VERSION) -lt $(ver $MIN_COMPOSE_VERSION) ]; then +if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" exit 1 fi -if [ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]; then +if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]]; then echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" exit 1 fi @@ -119,9 +119,9 @@ fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) # On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) -if (($IS_KVM == 0)); then +if [[ "$IS_KVM" -eq 0 ]]; then SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) - if (($SUPPORTS_SSE42 == 0)); then + if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." exit 1 fi @@ -157,7 +157,7 @@ fi replace_tsdb() { if ( - [ -f "$SENTRY_CONFIG_PY" ] && + [[ -f "$SENTRY_CONFIG_PY" ]] && ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" ); then # Do NOT indent the following string as it would be reflected in the end result, @@ -218,7 +218,7 @@ $dc build --force-rm --parallel echo "" echo "Docker images built." -if [[ "$MINIMIZE_DOWNTIME" ]]; then +if [[ -n "$MINIMIZE_DOWNTIME" ]]; then # Stop everything but relay and nginx $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') else @@ -230,11 +230,11 @@ else fi ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') -if [ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq "1" ]; then +if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 - if [ "$ZOOKEEPER_LOG_FILE_COUNT" -gt "0" ] && [ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq "0" ]; then + if [[ "$ZOOKEEPER_LOG_FILE_COUNT" -gt 0 ]] && [[ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq 0 ]]; then $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper fi @@ -246,7 +246,7 @@ $dcr snuba-api migrations migrate --force echo "" # Very naively check whether there's an existing sentry-postgres volume and the PG version in it -if [[ $(docker volume ls -q --filter name=sentry-postgres) && $(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null) == "9.5" ]]; then +if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then docker volume rm sentry-postgres-new || true # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume docker run --rm \ @@ -268,7 +268,7 @@ fi echo "" echo "Setting up database..." -if [ $CI ] || [ $SKIP_USER_PROMPT == 1 ]; then +if [[ -n "$CI" || "$SKIP_USER_PROMPT" == 1 ]]; then $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." @@ -282,7 +282,7 @@ fi SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") -if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then +if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then echo "Migrating file storage..." # Use the web (Sentry) image so the file owners are kept as sentry:sentry # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. @@ -291,7 +291,7 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then fi -if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then +if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then echo "" echo "Generating Relay credentials..." From f8a2e48ed66f29d66fbaa4d1cab2b1b35cb4bb7a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 15 Sep 2020 19:40:42 +0300 Subject: [PATCH 198/417] build(docker): Bump min Docker & Compose versions (#676) We need `docker-compose ps -a` for CI so we were already using 1.24.1, this aligns the rest with that. For Docker, there are a bunch of network-related fixes in 19.03.12 and prior (DNS fallback and IPv6 advertising) that we'd like to have to see if they are going to fix some reported connectivity issues w/ onpremise. --- .github/workflows/test.yml | 2 +- README.md | 4 ++-- install.sh | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 34255559ef..e53def0b6e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,7 +12,7 @@ env: DOCKER_COMPOSE_VERSION: 1.24.1 jobs: test: - runs-on: ubuntu-16.04 + runs-on: ubuntu-18.04 name: "test" steps: - name: Pin docker-compose diff --git a/README.md b/README.md index 8cab9df650..d15ec10c3a 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,8 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements - * Docker 17.05.0+ - * Compose 1.23.0+ + * Docker 19.03.12+ + * Compose 1.24.1+ ## Minimum Hardware Requirements: diff --git a/install.sh b/install.sh index e94621bfa5..c51335edbc 100755 --- a/install.sh +++ b/install.sh @@ -11,8 +11,8 @@ dcr="$dc run --rm" log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") -MIN_DOCKER_VERSION='17.05.0' -MIN_COMPOSE_VERSION='1.23.0' +MIN_DOCKER_VERSION='19.03.12' +MIN_COMPOSE_VERSION='1.24.1' MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' From 49dbeabb2ba1b2a1e5194d482d94885a3124118f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 15 Sep 2020 20:19:58 +0300 Subject: [PATCH 199/417] ci(gcb): Drop Docker version to 19.03.8 as that's the latest on GCB --- README.md | 2 +- install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index d15ec10c3a..bdd3654d2c 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements - * Docker 19.03.12+ + * Docker 19.03.8+ * Compose 1.24.1+ ## Minimum Hardware Requirements: diff --git a/install.sh b/install.sh index c51335edbc..e8d55099f9 100755 --- a/install.sh +++ b/install.sh @@ -11,7 +11,7 @@ dcr="$dc run --rm" log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") -MIN_DOCKER_VERSION='19.03.12' +MIN_DOCKER_VERSION='19.03.8' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM=2400 # MB From 21d1b6d42eee14d0de93ba77fe981f9f2a82447e Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Tue, 15 Sep 2020 21:21:14 +0000 Subject: [PATCH 200/417] license: Update BSL change date --- LICENSE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/LICENSE b/LICENSE index 38b25bd08d..18ffae0c3f 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-07-01 +Change Date: 2023-09-15 Change License: Apache License, Version 2.0 From ce8538e83f12ae3cb2317af02acdc233595f52b2 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 23 Sep 2020 00:04:42 +0300 Subject: [PATCH 201/417] fix(ingest-consumer): Create the missing Kafka topics (#685) Fixes #683, supercedes #684. --- install.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/install.sh b/install.sh index e8d55099f9..50786a3121 100755 --- a/install.sh +++ b/install.sh @@ -245,6 +245,18 @@ $dcr snuba-api bootstrap --no-migrate --force $dcr snuba-api migrations migrate --force echo "" +# NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step +# XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) +EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) +NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events" +for topic in $NEEDED_KAFKA_TOPICS; do + if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq ingest-attachments; then + echo "Creating additional Kafka topics..." + $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 + echo "" + fi +done + # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then docker volume rm sentry-postgres-new || true From 74464695ec6cf1f4e0780c161a189f08d2f9b33d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 23 Sep 2020 00:17:21 +0300 Subject: [PATCH 202/417] fix(docker): Lower docker version to 19.03.6 (#682) Fixes #689. --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 50786a3121..33f0eadaa6 100755 --- a/install.sh +++ b/install.sh @@ -11,7 +11,7 @@ dcr="$dc run --rm" log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") -MIN_DOCKER_VERSION='19.03.8' +MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM=2400 # MB From 3408e3db99a946e5b7850a845a53894951a276f4 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 23 Sep 2020 00:17:40 +0300 Subject: [PATCH 203/417] fix(redis): Increase file descriptors to 10032 (#681) @xbenjii reported the following error on #629: >You requested maxclients of 10000 requiring at least 10032 max file descriptors. Increasing this limit by default makes sense to make Redis more available to heaveier loads. --- docker-compose.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4ac2ee442d..d0d5ce39f8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -56,6 +56,11 @@ services: image: 'redis:5.0-alpine' volumes: - 'sentry-redis:/data' + ulimits: + nofile: + soft: 10032 + hard: 10032 + postgres: << : *restart_policy image: 'postgres:9.6' From 0a4b2ad033be4f2a99f22b317a01ec112bd2fb36 Mon Sep 17 00:00:00 2001 From: Anton Ovchinnikov Date: Fri, 25 Sep 2020 19:54:52 +0200 Subject: [PATCH 204/417] fix(install): Check that all ingest topics exist (#686) Probably this is not needed anymore after https://github.com/getsentry/sentry/pull/20984 is merged, but just in case --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 33f0eadaa6..88e069a5a4 100755 --- a/install.sh +++ b/install.sh @@ -250,7 +250,7 @@ echo "" EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events" for topic in $NEEDED_KAFKA_TOPICS; do - if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq ingest-attachments; then + if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then echo "Creating additional Kafka topics..." $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 echo "" From dcc7fa4e6fb68383af591130a2ae2801639bd40a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 28 Sep 2020 23:08:41 +0300 Subject: [PATCH 205/417] meta(issues: Improve and clarify install log request --- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index eaaf895b29..5cd0918063 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -22,5 +22,5 @@ Version: *VERSION HERE* Please share any applicable logs: -- ./install.sh logs +- `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` # latest instal logs - `docker-compose logs` output From de2b610fc3dd1377eaf5542ef0a12de8e68e7d61 Mon Sep 17 00:00:00 2001 From: Pataar Date: Fri, 2 Oct 2020 18:40:46 +0200 Subject: [PATCH 206/417] State that config files need to be updated as well (#690) Our relay connection wasn't behaving correctly because of some missing config in our `sentry.conf.py`. Also, we didn't have the performance module enabled because of a missing feature flag. That's why an entry in the documentation regarding this would be nice. --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bdd3654d2c..40e6f997d3 100644 --- a/README.md +++ b/README.md @@ -57,6 +57,8 @@ _You need to be on at least Sentry 9.1.2 to be able to upgrade automatically to The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. +Also make sure to check for any difference between the example config files and your current config files in use. There might be new configuration that has to be added to your adjusted files. E.g. feature flags or server configuration. + ## Resources * [Documentation](https://develop.sentry.dev/onpremise/) From e333dbd56a4a27d30b7c0fb08c40c0514cb2819f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 3 Oct 2020 01:30:56 +0300 Subject: [PATCH 207/417] fix(install): Only use .env for fallbacks This fixes a serious bug in install.sh where it ignored externaly set env variable values such as `SENTRY_IMAGE` in favor of the ones defined in `.env`, essentially making all our e2e tests usless. --- install.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install.sh b/install.sh index 88e069a5a4..9f22a17ead 100755 --- a/install.sh +++ b/install.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash set -e -# With a tip o' the hat to https://unix.stackexchange.com/a/79077 -set -a && . ./.env && set +a +# Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 +t=$(mktemp) && export -p > "$t" && set -a && . ./.env && set +a && . "$t" && rm "$t" && unset t dc="docker-compose --no-ansi" dcr="$dc run --rm" From acbaec64c034b25bcfd373362ca91ab821b95be6 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 3 Oct 2020 02:44:24 +0300 Subject: [PATCH 208/417] build(python): Make default config py3 compatible (#692) --- sentry/sentry.conf.example.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 9136045864..9cd49227bd 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -12,25 +12,25 @@ def get_internal_network(): import socket import struct - iface = "eth0" + iface = b"eth0" sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ifreq = struct.pack("16sH14s", iface, socket.AF_INET, b"\x00" * 14) + ifreq = struct.pack(b"16sH14s", iface, socket.AF_INET, b"\x00" * 14) try: ip = struct.unpack( - "!I", struct.unpack("16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] + b"!I", struct.unpack(b"16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] )[0] netmask = socket.ntohl( - struct.unpack("16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] + struct.unpack(b"16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] ) except IOError: return () - base = socket.inet_ntoa(struct.pack("!I", ip & netmask)) + base = socket.inet_ntoa(struct.pack(b"!I", ip & netmask)) netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) - return ("{0:s}/{1:d}".format(base, netmask_bits),) + return "{0:s}/{1:d}".format(base, netmask_bits) -INTERNAL_SYSTEM_IPS = get_internal_network() +INTERNAL_SYSTEM_IPS = (get_internal_network(),) DATABASES = { From ae9378862caead70b620bb347bf5510833889478 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 5 Oct 2020 23:57:49 +0300 Subject: [PATCH 209/417] fix(py3): Make event ingestion test more robust (#693) We were relying on the public DSN field coming first from the API and with Python 3, that's not the case to the tests were failing. This PR makes DSN extraction a bit more robust while still avoiding a full-fledged JSON parser. --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 9d11d9df24..91cfc80139 100755 --- a/test.sh +++ b/test.sh @@ -74,7 +74,7 @@ done # Set up initial/required settings (InstallWizard request) sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null -SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') +SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" && $4 ~ "^http" { print $4; exit; }') # We ignore the protocol and the host as we already know those DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\?://\([0-9a-z]\+\)@[^/]\+/\([0-9]\+\)$|\1\n\2|p'`) SENTRY_KEY=${DSN_PIECES[0]} From 694977a349407c544cde70d3d4a2a48eb7d7dc1d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 7 Oct 2020 01:49:47 +0300 Subject: [PATCH 210/417] ref(config): Move GitHub config over to config.yml (#697) Follow up to getsentry/sentry#21041 --- sentry/config.example.yml | 1 + sentry/sentry.conf.example.py | 6 ------ 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 0e74ec55bc..f42be825b2 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -76,6 +76,7 @@ transaction-events.force-disable-internal-project: true # GitHub Integration # ###################### +# github-app.extended-permissions: ['repo'] # github-app.id: GITHUB_APP_ID # github-app.name: 'GITHUB_APP_NAME' # github-app.webhook-secret: 'GITHUB_WEBHOOK_SECRET' # Use only if configured in GitHub diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 9cd49227bd..b13893bdb3 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -257,12 +257,6 @@ def get_internal_network(): } ) -###################### -# GitHub Integration # -###################### - -GITHUB_EXTENDED_PERMISSIONS = ["repo"] - ######################### # Bitbucket Integration # ######################## From a9a8b2f98b84bc1de94884dd3226ef3d7d77b29f Mon Sep 17 00:00:00 2001 From: Pataar Date: Wed, 7 Oct 2020 22:14:23 +0200 Subject: [PATCH 211/417] docs: Update README with several improvements (#695) - Change minimal Docker version to the one in `install.sh` - Add an extra link to the documentation - Change the existing link to point to the correct one. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 40e6f997d3..e7d929889e 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements - * Docker 19.03.8+ + * Docker 19.03.6+ * Compose 1.24.1+ ## Minimum Hardware Requirements: @@ -59,9 +59,11 @@ The included `install.sh` script is meant to be idempotent and to bring you to t Also make sure to check for any difference between the example config files and your current config files in use. There might be new configuration that has to be added to your adjusted files. E.g. feature flags or server configuration. +For more information regarding updating your Sentry installation, please visit [our documentation](https://develop.sentry.dev/self-hosted/#upgrading). + ## Resources - * [Documentation](https://develop.sentry.dev/onpremise/) + * [Documentation](https://develop.sentry.dev/self-hosted/) * [Bug Tracker](https://github.com/getsentry/onpremise/issues) * [Community Forums](https://forum.sentry.io/c/on-premise) From 1399be6a68f3adda8f039c4ba463dea1db9e6932 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 14 Oct 2020 21:54:55 +0300 Subject: [PATCH 212/417] feat(py3): Add Python 3 support via SENTRY_PYTHON3 env variable (#702) --- .github/workflows/test.yml | 6 +++++- README.md | 2 ++ docker-compose.yml | 2 +- install.sh | 2 +- sentry/Dockerfile | 3 ++- 5 files changed, 11 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e53def0b6e..f816594436 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,8 +12,11 @@ env: DOCKER_COMPOSE_VERSION: 1.24.1 jobs: test: + strategy: + matrix: + py3: ['', '1'] runs-on: ubuntu-18.04 - name: "test" + name: "test${{ matrix.py3 == '1' && ' PY3' || ''}}" steps: - name: Pin docker-compose run: | @@ -28,6 +31,7 @@ jobs: - name: Install and test env: COMPOSE_PARALLEL_LIMIT: 10 + SENTRY_PYTHON3: ${{ matrix.py3 }} run: | ./install.sh ./test.sh diff --git a/README.md b/README.md index e7d929889e..4d7fea2536 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,8 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. +_If you like trying out new things, you can run `SENTRY_PYTHON3=1 ./install.sh` instead to use our brand new Python 3 images. **Keep in mind that Python 3 support is experimental at this point**_ + During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --no-user-prompt`. There may need to be modifications to the included example config files (`sentry/config.example.yml` and `sentry/sentry.conf.example.py`) to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script and copy them without the `.example` extensions in the name (such as `sentry/sentry.conf.py`) before running the `install.sh` script. diff --git a/docker-compose.yml b/docker-compose.yml index d0d5ce39f8..4a082d801a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ x-sentry-defaults: &sentry_defaults context: ./sentry args: - SENTRY_IMAGE + - SENTRY_PYTHON3 image: sentry-onpremise-local depends_on: - redis @@ -60,7 +61,6 @@ services: nofile: soft: 10032 hard: 10032 - postgres: << : *restart_policy image: 'postgres:9.6' diff --git a/install.sh b/install.sh index 9f22a17ead..d5fafc5bb0 100755 --- a/install.sh +++ b/install.sh @@ -207,7 +207,7 @@ echo "" $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails -docker pull $SENTRY_IMAGE || true; +docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON3:+-py3} || true; echo "" echo "Building and tagging Docker images..." diff --git a/sentry/Dockerfile b/sentry/Dockerfile index f9484f295b..7a5b3a8bf4 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,5 +1,6 @@ ARG SENTRY_IMAGE -FROM ${SENTRY_IMAGE} +ARG SENTRY_PYTHON3 +FROM ${SENTRY_IMAGE}${SENTRY_PYTHON3:+-py3} COPY . /usr/src/sentry From 8512f52ba5c7717519a0fa7198482caec470a007 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Thu, 15 Oct 2020 20:07:46 +0200 Subject: [PATCH 213/417] ci(gha): Mark version as optional in release workflow (#704) --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6bc87c1aff..34725dead8 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ on: workflow_dispatch: inputs: version: - description: Version to release + description: Version to release (optional) required: false skip_prepare: description: Skip preparation step (assume a release branch is ready) @@ -21,7 +21,7 @@ on: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, # Snuba, and Relay to make sure their releases finish. - - cron: '0 18 15 * *' + - cron: "0 18 15 * *" jobs: release: runs-on: ubuntu-latest @@ -60,8 +60,8 @@ jobs: with: action: publish version: ${{ github.event.inputs.version || steps.calver.outputs.version }} - keep_branch: '--keep-branch' - no_merge: '--no-merge' + keep_branch: "--keep-branch" + no_merge: "--no-merge" env: DRY_RUN: ${{ github.event.inputs.dry_run }} GIT_COMMITTER_NAME: getsentry-bot From 2a559485ad0da0f8823699eb4d85cbecb1c313df Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 15 Oct 2020 23:06:24 +0300 Subject: [PATCH 214/417] ref: Switch all nightly images to `:nightly` tag (#703) --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 85ca9bc038..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:latest -SNUBA_IMAGE=getsentry/snuba:latest -RELAY_IMAGE=getsentry/relay:latest -SYMBOLICATOR_IMAGE=getsentry/symbolicator:latest +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index 4d7fea2536..5f89250ce8 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ If you have any issues or questions, our [Community Forum](https://forum.sentry. If you want to install a specific release of Sentry, use the tags/releases on this repo. -We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:latest`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: +We continously push the Docker image for each commit made into [Sentry](https://github.com/getsentry/sentry), and other services such as [Snuba](https://github.com/getsentry/snuba) or [Symbolicator](https://github.com/getsentry/symbolicator) to [our Docker Hub](https://hub.docker.com/u/getsentry) and tag the latest version on master as `:nightly`. This is also usually what we have on sentry.io and what the install script uses. You can use a custom Sentry image, such as a modified version that you have built on your own, or simply a specific commit hash by setting the `SENTRY_IMAGE` environment variable to that image name before running `./install.sh`: ```shell SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh From e9cff2e28867eda078b3d3b103aae68252e19d91 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 16 Oct 2020 00:01:53 +0300 Subject: [PATCH 215/417] ci(release): Remove specific status contexts --- .craft.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.craft.yml b/.craft.yml index 2ad9b373d0..ad41cd872f 100644 --- a/.craft.yml +++ b/.craft.yml @@ -8,8 +8,5 @@ artifactProvider: name: none statusProvider: name: github - config: - contexts: - - 'test' targets: - name: github From 2a1a171233307d33b83121205952795b8cafca6e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 29 Oct 2020 23:30:43 +0300 Subject: [PATCH 216/417] fix(install): Increase stop timeout to 60 seconds (#731) This is to ensure clean shutdown of Celery, with fully drained queues. This is needed as versions may change the event format and not be backwards compatible. FWIW this is a hacky workaround without a strong guarantee that the queues will be empty. Ideally we'd shutdown everything first, spin up the workers and check for queues being drained every second or so. --- install.sh | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index d5fafc5bb0..da0ed59863 100755 --- a/install.sh +++ b/install.sh @@ -15,6 +15,11 @@ MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM=2400 # MB +# Increase the default 10 second SIGTERM timeout +# to ensure celery queues are properly drained +# between upgrades as task signatures may change across +# versions +STOP_TIMEOUT=60 # seconds SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' @@ -76,7 +81,7 @@ cleanup () { fi if [[ -z "$MINIMIZE_DOWNTIME" ]]; then - $dc stop &> /dev/null + $dc stop -t $STOP_TIMEOUT &> /dev/null fi } trap_with_arg cleanup ERR INT TERM EXIT @@ -224,9 +229,9 @@ if [[ -n "$MINIMIZE_DOWNTIME" ]]; then else # Clean up old stuff and ensure nothing is working while we install/update # This is for older versions of on-premise: - $dc -p onpremise down --rmi local --remove-orphans + $dc -p onpremise down -t $STOP_TIMEOUT --rmi local --remove-orphans # This is for newer versions - $dc down --rmi local --remove-orphans + $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans fi ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') From 79c461328e4f03d1d59437f4bacb752134652a5f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 9 Nov 2020 20:12:22 +0300 Subject: [PATCH 217/417] feat(search): Enable advanced search for self-hosted (#734) Addresses https://forum.sentry.io/t/advanced-search-access-in-onpremise-install/11736?u=byk --- sentry/sentry.conf.example.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index b13893bdb3..01b092502a 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -247,6 +247,7 @@ def get_internal_network(): "organizations:sso-rippling", "organizations:sso-saml2", "organizations:performance-view", + "organizations:advanced-search", "projects:custom-inbound-filters", "projects:data-forwarding", "projects:discard-groups", From 245ae95a6ad55690a22a7f00f5f3eefdf46683ad Mon Sep 17 00:00:00 2001 From: Sullivan SENECHAL Date: Wed, 11 Nov 2020 14:14:28 +0100 Subject: [PATCH 218/417] fix(clickhouse): default logger level set to information (#733) Co-authored-by: Burak Yigit Kaya --- clickhouse/config.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/clickhouse/config.xml b/clickhouse/config.xml index 4cd44b7b4a..899814319e 100644 --- a/clickhouse/config.xml +++ b/clickhouse/config.xml @@ -1,3 +1,7 @@ + + information + 1 + From 066bf262aac7b50cdc870b3f4e41d378cc2193db Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Fri, 13 Nov 2020 02:39:11 -0800 Subject: [PATCH 219/417] feat: Allow metric alerts to be used in on-prem (#735) This enables metric alerts for all on-prem users. We just need to start a few consumers and enable the feature. --- docker-compose.yml | 12 ++++++++++++ sentry/sentry.conf.example.py | 2 ++ 2 files changed, 14 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4a082d801a..4754ebc32d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -143,6 +143,12 @@ services: snuba-replacer: << : *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 + snuba-subscription-consumer-events: + << : *snuba_defaults + command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 + snuba-subscription-consumer-transactions: + << : *snuba_defaults + command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=snuba-transactions-consumers --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: << : *snuba_defaults image: snuba-cleanup-onpremise-local @@ -186,6 +192,12 @@ services: << : *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 + subscription-consumer-events: + << : *sentry_defaults + command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results + subscription-consumer-transactions: + << : *sentry_defaults + command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: << : *sentry_defaults image: sentry-cleanup-onpremise-local diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 01b092502a..ca217d117a 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -240,9 +240,11 @@ def get_internal_network(): "organizations:discover", "organizations:events", "organizations:global-views", + "organizations:incidents", "organizations:integrations-issue-basic", "organizations:integrations-issue-sync", "organizations:invite-members", + "organizations:metric-alert-builder-aggregate", "organizations:sso-basic", "organizations:sso-rippling", "organizations:sso-saml2", From a717c11a2554474c7ba8637ebba89750061c2a2f Mon Sep 17 00:00:00 2001 From: Dan Fuller Date: Mon, 16 Nov 2020 21:51:19 -0800 Subject: [PATCH 220/417] fix(metric_alerts): Fix transaction alerts (#739) I only tested error alerts while testing, turns out this was broken in both dev and on-prem. This fixes the issue. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4754ebc32d..0314abde4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -139,7 +139,7 @@ services: # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: << : *snuba_defaults - command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 + command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: << : *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 @@ -148,7 +148,7 @@ services: command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 snuba-subscription-consumer-transactions: << : *snuba_defaults - command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=snuba-transactions-consumers --delay-seconds=60 --schedule-ttl=60 + command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: << : *snuba_defaults image: snuba-cleanup-onpremise-local From bee98fd8973748c2be717cba1162beb1739c6424 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 17 Nov 2020 20:35:55 +0300 Subject: [PATCH 221/417] fix(install): Disable the script on git-bash (#741) We kept getting issue reports that we traced down to `git-bash` which doesn't seem to play nice with Docker for Windows with bind mounts. This PR uses the existence of `$MSYSTEM` to detect `git-bash` and exit early with a relevant message. Co-authored-by: Chad Whitacre --- install.sh | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/install.sh b/install.sh index da0ed59863..c044ac31ba 100755 --- a/install.sh +++ b/install.sh @@ -1,6 +1,11 @@ #!/usr/bin/env bash set -e +if [[ -n "$MSYSTEM" ]]; then + echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; + exit 1 +fi + # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . ./.env && set +a && . "$t" && rm "$t" && unset t From 99c9c529fbead35b2c680e068046e8f91eb10fd9 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 18 Nov 2020 00:27:05 +0300 Subject: [PATCH 222/417] ci(release): Align release workflow with getsentry/sentry (#744) Adds names to steps, adds the sleep between prep and publish, adds the global git user config.. --- .github/workflows/release.yml | 55 ++++++++++++++++++++--------------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 34725dead8..7460c4133e 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,58 +21,65 @@ on: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, # Snuba, and Relay to make sure their releases finish. - - cron: "0 18 15 * *" + - cron: '0 18 15 * *' jobs: release: runs-on: ubuntu-latest - name: "Release a new version" + name: 'Release a new version' steps: - id: killswitch + name: Check release blockers if: ${{ !github.event.inputs.force }} run: | if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then echo "Open release-blocking issues found, cancelling release..."; curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; fi - - id: calver - if: ${{ !github.event.inputs.version }} + - id: set-version + name: Determine version run: | - DATE_PART=$(date +'%y.%-m') - declare -i PATCH_VERSION=0 - while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do - PATCH_VERSION+=1 - done - echo "::set-output name=version::$DATE_PART.$PATCH_VERSION" + if [[ -n '${{ github.event.inputs.version }}' ]]; then + echo 'RELEASE_VERSION=${{ github.event.inputs.version }}' >> $GITHUB_ENV; + else + DATE_PART=$(date +'%y.%-m') + declare -i PATCH_VERSION=0 + while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do + PATCH_VERSION+=1 + done + echo "RELEASE_VERSION=${DATE_PART}.${PATCH_VERSION}" >> $GITHUB_ENV; + fi - uses: actions/checkout@v2 with: token: ${{ secrets.GH_SENTRY_BOT_PAT }} + - id: set-git-user + name: Set git user to getsentry-bot + run: | + git config --global user.name getsentry-bot + git config --global user.email bot@getsentry.com - uses: getsentry/craft@master + name: Craft Prepare if: ${{ !github.event.inputs.skip_prepare }} with: action: prepare - version: ${{ github.event.inputs.version || steps.calver.outputs.version }} + version: ${{ env.RELEASE_VERSION }} env: DRY_RUN: ${{ github.event.inputs.dry_run }} - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com + # Wait until the builds start. Craft should do this automatically + # but it is broken now. + - run: sleep 10 - uses: getsentry/craft@master + name: Craft Publish with: action: publish - version: ${{ github.event.inputs.version || steps.calver.outputs.version }} - keep_branch: "--keep-branch" - no_merge: "--no-merge" + version: ${{ env.RELEASE_VERSION }} + no_merge: '--no-merge' env: DRY_RUN: ${{ github.event.inputs.dry_run }} - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com + # We need this additonal step because we don't merge release branches into master to + # always keep it on nightlies - id: bump-license-date + name: Bump license chage date if: ${{ !github.event.inputs.dry_run && !github.event.inputs.version }} - env: - GIT_COMMITTER_NAME: getsentry-bot - GIT_AUTHOR_NAME: getsentry-bot - EMAIL: bot@getsentry.com run: | sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE git diff --quiet || git commit -anm 'license: Update BSL change date' && git push From 65cb0c3821b1316fdd1317f4eb375185dca0fb7e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 18 Nov 2020 00:35:09 +0300 Subject: [PATCH 223/417] ci(release): Don't use global git conf (#745) --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7460c4133e..2a6aa788b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -54,8 +54,8 @@ jobs: - id: set-git-user name: Set git user to getsentry-bot run: | - git config --global user.name getsentry-bot - git config --global user.email bot@getsentry.com + git config user.name getsentry-bot + git config user.email bot@getsentry.com - uses: getsentry/craft@master name: Craft Prepare if: ${{ !github.event.inputs.skip_prepare }} From bd284d0b7f764f03b5d447405743a88e4c196b03 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 17 Nov 2020 17:21:37 -0500 Subject: [PATCH 224/417] =?UTF-8?q?ci(release):=20fix=20a=20couple=20typos?= =?UTF-8?q?=20=F0=9F=90=AD=20(#746)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 2a6aa788b5..aab3b6751f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -75,10 +75,10 @@ jobs: no_merge: '--no-merge' env: DRY_RUN: ${{ github.event.inputs.dry_run }} - # We need this additonal step because we don't merge release branches into master to + # We need this additional step because we don't merge release branches into master to # always keep it on nightlies - id: bump-license-date - name: Bump license chage date + name: Bump license change date if: ${{ !github.event.inputs.dry_run && !github.event.inputs.version }} run: | sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE From 19f4561a9e2abe32dc5eb5a03a332b50f2265b4b Mon Sep 17 00:00:00 2001 From: Marc Durdin Date: Thu, 19 Nov 2020 20:53:50 +1100 Subject: [PATCH 225/417] Clarify upgrade process (#751) After following the instructions on this repo for upgrading, and breaking my installation, I thought it might be helpful to others if these instructions were clarified a little. --- README.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5f89250ce8..0f1c52d8aa 100644 --- a/README.md +++ b/README.md @@ -57,9 +57,11 @@ and [Nginx](http://nginx.org/). Our recommendation is running and external Nginx _You need to be on at least Sentry 9.1.2 to be able to upgrade automatically to the latest version. If you are not, upgrade to 9.1.2 first by checking out the [9.1.2 tag](https://github.com/getsentry/onpremise/tree/9.1.2) on this repo._ -The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. +We recommend (and sometimes require) you to upgrade Sentry one version at a time. That means if you are running 20.6.0, instead of going directly to 20.8.0, first go through 20.7.0. Skipping versions would work most of the time, but there will be times that we require you to stop at specific versions to ensure essential data migrations along the way. + +Pull the version of the repository that you wish to upgrade to by checking out the tagged release of this repo. Make sure to check for any difference between the example config files and your current config files in use. There might be new configuration that has to be added to your adjusted files such as feature flags or server configuration. -Also make sure to check for any difference between the example config files and your current config files in use. There might be new configuration that has to be added to your adjusted files. E.g. feature flags or server configuration. +The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. For more information regarding updating your Sentry installation, please visit [our documentation](https://develop.sentry.dev/self-hosted/#upgrading). From be214438f7b48ce74886c104e4c52c0c213b5ded Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 1 Dec 2020 21:11:59 +0300 Subject: [PATCH 226/417] fix(sentry): Pass SENTRY_EVENT_RETENTION_DAYS to sentry services (#754) We are already referencing this env var here: https://github.com/getsentry/onpremise/blob/19f4561a9e2abe32dc5eb5a03a332b50f2265b4b/sentry/sentry.conf.example.py#L62-L64 --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 0314abde4d..d42e95ddf2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,6 +25,7 @@ x-sentry-defaults: &sentry_defaults environment: SENTRY_CONF: '/etc/sentry' SNUBA: 'http://snuba-api:1218' + SENTRY_EVENT_RETENTION_DAYS: volumes: - 'sentry-data:/data' - './sentry:/etc/sentry' From 9151f566206adae5a65f5d7f1621178d090d924a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 1 Dec 2020 23:56:24 +0300 Subject: [PATCH 227/417] fix(snuba): Pass SENTRY_EVENT_RETENTION_DAYS to Snuba instances too (#759) Follow up on #754. Depends on getsentry/snuba#1526. --- docker-compose.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index d42e95ddf2..bc1acc679d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,9 @@ x-sentry-defaults: &sentry_defaults environment: SENTRY_CONF: '/etc/sentry' SNUBA: 'http://snuba-api:1218' - SENTRY_EVENT_RETENTION_DAYS: + # Leaving the value empty to just pass whatever is set + # on the host system (or in the .env file) + SENTRY_EVENT_RETENTION_DAYS: volumes: - 'sentry-data:/data' - './sentry:/etc/sentry' @@ -43,6 +45,9 @@ x-snuba-defaults: &snuba_defaults REDIS_HOST: redis UWSGI_MAX_REQUESTS: '10000' UWSGI_DISABLE_LOGGING: 'true' + # Leaving the value empty to just pass whatever is set + # on the host system (or in the .env file) + SENTRY_EVENT_RETENTION_DAYS: services: smtp: << : *restart_policy From 504550d96731577ec121d7801f65e0ffbd330d31 Mon Sep 17 00:00:00 2001 From: Max Wittig Date: Tue, 1 Dec 2020 22:25:47 +0100 Subject: [PATCH 228/417] fix(clickhouse): set enable_mixed_granularity_parts to enabled (#758) I've noticed clickhouse complaining about this missing setting, which stopped clickhouse from starting up with version 20.3.9.70 Related to #726 --- clickhouse/config.xml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/clickhouse/config.xml b/clickhouse/config.xml index 899814319e..55cdbbd82b 100644 --- a/clickhouse/config.xml +++ b/clickhouse/config.xml @@ -4,4 +4,7 @@ information 1 + + 1 + From 41e7d862f1c63412903e296a5d70e7fe8940810d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 3 Dec 2020 20:58:26 +0300 Subject: [PATCH 229/417] ci(release): Use action-prepare-release and rework dry-run (#761) Switches to using getsentry/action-prepare-release for common steps and removes the dry run option from the prepare stage as it prevents us from testing the publish step. Prepare step should be safe anyways and can be skipped with the `skip-prepare` input. Related: https://app.asana.com/0/1198192131329257/1198192131329301, https://app.asana.com/0/1198192131329257/1198192573081717 --- .github/workflows/release.yml | 33 +++++---------------------------- 1 file changed, 5 insertions(+), 28 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index aab3b6751f..3a411c04ac 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -27,43 +27,20 @@ jobs: runs-on: ubuntu-latest name: 'Release a new version' steps: - - id: killswitch - name: Check release blockers - if: ${{ !github.event.inputs.force }} - run: | - if curl -s "https://api.github.com/repos/$GITHUB_REPOSITORY/issues?state=open&labels=release-blocker" | grep -Pzvo '\[[\s\n\r]*\]'; then - echo "Open release-blocking issues found, cancelling release..."; - curl -sf -X POST -H 'Accept: application/vnd.github.v3+json' -H 'Authorization: token ${{ secrets.GITHUB_TOKEN }}' https://api.github.com/repos/$GITHUB_REPOSITORY/actions/runs/${{ github.run_id }}/cancel; - fi - - id: set-version - name: Determine version - run: | - if [[ -n '${{ github.event.inputs.version }}' ]]; then - echo 'RELEASE_VERSION=${{ github.event.inputs.version }}' >> $GITHUB_ENV; - else - DATE_PART=$(date +'%y.%-m') - declare -i PATCH_VERSION=0 - while curl -sf -o /dev/null "https://api.github.com/repos/$GITHUB_REPOSITORY/git/ref/tags/$DATE_PART.$PATCH_VERSION"; do - PATCH_VERSION+=1 - done - echo "RELEASE_VERSION=${DATE_PART}.${PATCH_VERSION}" >> $GITHUB_ENV; - fi + - name: Prepare release + uses: getsentry/action-prepare-release@main + with: + version: ${{ github.event.inputs.version }} + force: ${{ github.event.inputs.force }} - uses: actions/checkout@v2 with: token: ${{ secrets.GH_SENTRY_BOT_PAT }} - - id: set-git-user - name: Set git user to getsentry-bot - run: | - git config user.name getsentry-bot - git config user.email bot@getsentry.com - uses: getsentry/craft@master name: Craft Prepare if: ${{ !github.event.inputs.skip_prepare }} with: action: prepare version: ${{ env.RELEASE_VERSION }} - env: - DRY_RUN: ${{ github.event.inputs.dry_run }} # Wait until the builds start. Craft should do this automatically # but it is broken now. - run: sleep 10 From 06fb0d75de55f0cc771bf9db223c18c01f7166ff Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 4 Dec 2020 22:43:36 +0300 Subject: [PATCH 230/417] ref(py3): Make PY3 the default*, add SENTRY_PYTHON (#763) This is in preparation to make the PY3 version the default* for Docker images and self-hosted. It is part **2/5**: 1. ~~Add `-py2` variants for the Python 2 build tags and introduce the `SENTRY_PYTHON2` env variable usage~~ (getsentry/sentry#22460) 2. __Switch getsentry/onpremise to Python 3 by default*, introducing the `SENTRY_PYTHON2` env var for Py2 builds via the `-py2` suffix__ 3. Move the unsuffixed version of the builds to Python 3 4. Remove the `SENTRY_PYTHON3` env var support and `-py3` prefix usage from getsentry/onpremise 5. Remove tagging of `-py3` builds from here _* this will only happen when item 3 above gets landed_ --- .github/workflows/test.yml | 7 ++++--- README.md | 4 +--- docker-compose.yml | 1 + install.sh | 4 ++-- sentry/Dockerfile | 3 ++- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f816594436..fd5809bc13 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,9 +14,9 @@ jobs: test: strategy: matrix: - py3: ['', '1'] + py2: ['', '1'] runs-on: ubuntu-18.04 - name: "test${{ matrix.py3 == '1' && ' PY3' || ''}}" + name: "test${{ matrix.py2 == '1' && ' PY2' || ''}}" steps: - name: Pin docker-compose run: | @@ -31,7 +31,8 @@ jobs: - name: Install and test env: COMPOSE_PARALLEL_LIMIT: 10 - SENTRY_PYTHON3: ${{ matrix.py3 }} + SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} + SENTRY_PYTHON3: ${{ matrix.py2 != '1' || ''}} run: | ./install.sh ./test.sh diff --git a/README.md b/README.md index 0f1c52d8aa..a6d261dfc4 100644 --- a/README.md +++ b/README.md @@ -13,9 +13,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Setup -To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. - -_If you like trying out new things, you can run `SENTRY_PYTHON3=1 ./install.sh` instead to use our brand new Python 3 images. **Keep in mind that Python 3 support is experimental at this point**_ +To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020. If you want/need to stick with the Python 2 versions of the images, you can run `SENTRY_PYTHON2=1 ./install.sh` instead. Note that we are planning to end our Python 2 support completely by January 2021. During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --no-user-prompt`. diff --git a/docker-compose.yml b/docker-compose.yml index bc1acc679d..6fb685fdda 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -7,6 +7,7 @@ x-sentry-defaults: &sentry_defaults context: ./sentry args: - SENTRY_IMAGE + - SENTRY_PYTHON2 - SENTRY_PYTHON3 image: sentry-onpremise-local depends_on: diff --git a/install.sh b/install.sh index c044ac31ba..bb123ea683 100755 --- a/install.sh +++ b/install.sh @@ -21,7 +21,7 @@ MIN_COMPOSE_VERSION='1.24.1' MIN_RAM=2400 # MB # Increase the default 10 second SIGTERM timeout -# to ensure celery queues are properly drained +# to ensure celery queues are properly drained # between upgrades as task signatures may change across # versions STOP_TIMEOUT=60 # seconds @@ -217,7 +217,7 @@ echo "" $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails -docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON3:+-py3} || true; +docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2}${SENTRY_PYTHON3:+-py3} || true; echo "" echo "Building and tagging Docker images..." diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 7a5b3a8bf4..8d17146ad8 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,6 +1,7 @@ ARG SENTRY_IMAGE +ARG SENTRY_PYTHON2 ARG SENTRY_PYTHON3 -FROM ${SENTRY_IMAGE}${SENTRY_PYTHON3:+-py3} +FROM ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2}${SENTRY_PYTHON3:+-py3} COPY . /usr/src/sentry From f885eceaec93a04587026cfc40dea501b0257cab Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sat, 5 Dec 2020 01:46:20 +0300 Subject: [PATCH 231/417] ref(py3): Remove SENTRY_PYTHON3 and -py3 versons (#764) This is in preparation to make the PY3 version the default for Docker images and self-hosted. It is part **4/5**: 1. ~~Add `-py2` variants for the Python 2 build tags and introduce the `SENTRY_PYTHON2` env variable usage~~ (getsentry/sentry#22460) 2. ~~Switch getsentry/onpremise to Python 3 by default*, introducing the `SENTRY_PYTHON2` env var for Py2 builds via the `-py2` suffix~~ (getsentry/onpremise#763) 3. ~~Move the unsuffixed version of the builds to Python 3~~ (getsentry/sentry#22466) 4. **Remove the `SENTRY_PYTHON3` env var support and `-py3` prefix usage from getsentry/onpremise** 5. Remove tagging of `-py3` builds from getsentry/sentry --- .github/workflows/test.yml | 1 - docker-compose.yml | 1 - install.sh | 2 +- sentry/Dockerfile | 3 +-- 4 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fd5809bc13..5cd4f64a1a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,6 @@ jobs: env: COMPOSE_PARALLEL_LIMIT: 10 SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} - SENTRY_PYTHON3: ${{ matrix.py2 != '1' || ''}} run: | ./install.sh ./test.sh diff --git a/docker-compose.yml b/docker-compose.yml index 6fb685fdda..5909226ae4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,7 +8,6 @@ x-sentry-defaults: &sentry_defaults args: - SENTRY_IMAGE - SENTRY_PYTHON2 - - SENTRY_PYTHON3 image: sentry-onpremise-local depends_on: - redis diff --git a/install.sh b/install.sh index bb123ea683..2ec1832167 100755 --- a/install.sh +++ b/install.sh @@ -217,7 +217,7 @@ echo "" $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails -docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2}${SENTRY_PYTHON3:+-py3} || true; +docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2} || true; echo "" echo "Building and tagging Docker images..." diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 8d17146ad8..812056ead1 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,7 +1,6 @@ ARG SENTRY_IMAGE ARG SENTRY_PYTHON2 -ARG SENTRY_PYTHON3 -FROM ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2}${SENTRY_PYTHON3:+-py3} +FROM ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2} COPY . /usr/src/sentry From a623e72e7e995a321493e4459d0515322034684a Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 14 Dec 2020 11:56:04 -0500 Subject: [PATCH 232/417] Integrate with MaxMind out of the box (#766) Integrate with MaxMind out of the box --- .gitignore | 3 +++ docker-compose.yml | 13 +++++++++++ geoip/GeoLite2-City.mmdb.empty | Bin 0 -> 1055 bytes install.sh | 4 ++++ install/geoip.sh | 39 +++++++++++++++++++++++++++++++++ nginx/nginx.conf | 2 +- relay/config.example.yml | 1 + sentry/sentry.conf.example.py | 8 ++++++- 8 files changed, 68 insertions(+), 2 deletions(-) create mode 100644 geoip/GeoLite2-City.mmdb.empty create mode 100755 install/geoip.sh diff --git a/.gitignore b/.gitignore index b8ee807d52..707622f425 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ sentry/requirements.txt relay/credentials.json relay/config.yml symbolicator/config.yml +geoip/GeoIP.conf +geoip/*.mmdb +geoip/.geoipupdate.lock diff --git a/docker-compose.yml b/docker-compose.yml index 5909226ae4..160ae6294c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,7 @@ x-sentry-defaults: &sentry_defaults volumes: - 'sentry-data:/data' - './sentry:/etc/sentry' + - './geoip:/geoip:ro' x-snuba-defaults: &snuba_defaults << : *restart_policy depends_on: @@ -126,6 +127,14 @@ services: # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 + geoipupdate: + image: 'maxmindinc/geoipupdate:latest' + # Override the entrypoint in order to avoid using envvars for config. + # Futz with settings so we can keep mmdb and conf in same dir on host + # (image looks for them in separate dirs by default). + entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf'] + volumes: + - './geoip:/sentry' snuba-api: << : *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse @@ -233,6 +242,10 @@ services: read_only: true source: ./relay target: /work/.relay + - type: bind + read_only: true + source: ./geoip + target: /geoip depends_on: - kafka - redis diff --git a/geoip/GeoLite2-City.mmdb.empty b/geoip/GeoLite2-City.mmdb.empty new file mode 100644 index 0000000000000000000000000000000000000000..94f6921fd1c885fac56dcb689bf4467b25d9d78d GIT binary patch literal 1055 zcmZ9`*;dm)0LJmpS~uKxRMfh!RKd~H_-}(2OHXUFNupTG?i-Ebo3{a%=fQ7((_5zw6W$#6%J4|Hb5@0E? z8dzq=<-iK|O1jGI`9s$5<664T9CZQ**&Be3X50jH0lR_CX4%5nN=tMb-A;GVopcw? z{&5duFWpD?(*rcWQaA7kD04Xk#K2)-3^>B2haLs)0>^-Bz;P}o=t+8to~CE$S$dA1 zrx$21y+|(s13(|UpI)X{=vA8EFn7=_6-E^pVp#M#y+O0@-((B}w-~qSh&jDu;+~27 zzys3{fl>A&v*Z_e4A@+rP(dA_#&7|TA!+;Ob)dmTnSGow0r+NVGBgk{LbE>wBGcIe z6VolmB=C$eMN>LWpVJrgC4EI-)9g`i7;ou2;6395{YcyQ&*I Date: Mon, 14 Dec 2020 16:39:08 -0500 Subject: [PATCH 233/417] Lock closed issues/PRs via 3rd-party action (#772) Lock closed issues/PRs via 3rd-party action --- .github/workflows/lock.yml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 .github/workflows/lock.yml diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 0000000000..3584ab4baa --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,15 @@ +name: 'Lock closed issues and pull requests' +on: + schedule: + - cron: '*/5 * * * *' +jobs: + lock: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v2 + with: + github-token: ${{ github.token }} + issue-lock-inactive-days: 15 + issue-lock-reason: '' + pr-lock-inactive-days: 15 + pr-lock-reason: '' From 037f5d74d565a927ddf2dbee26d1e86f76c6d2b5 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 14 Dec 2020 17:14:08 -0500 Subject: [PATCH 234/417] Tighten up name and schedule (#773) --- .github/workflows/lock.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 3584ab4baa..fca2fb9eb0 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,7 +1,7 @@ -name: 'Lock closed issues and pull requests' +name: 'lock closed issues/PRs' on: schedule: - - cron: '*/5 * * * *' + - cron: '*/2 * * * *' jobs: lock: runs-on: ubuntu-latest From c49e4f0993d3c67d98ccc16fed184b19dffc7e45 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 14 Dec 2020 18:05:54 -0500 Subject: [PATCH 235/417] Drop back to hourly for issue/PR locking (#774) --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index fca2fb9eb0..3f2a08a462 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,7 +1,7 @@ name: 'lock closed issues/PRs' on: schedule: - - cron: '*/2 * * * *' + - cron: '11 * * * *' jobs: lock: runs-on: ubuntu-latest From 04d80faf0160d02d582d259062c9c4c8cea71ed6 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 15 Dec 2020 08:42:56 -0500 Subject: [PATCH 236/417] Drop to daily locking now that we're caught up (#776) --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 3f2a08a462..6865759ab9 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,7 +1,7 @@ name: 'lock closed issues/PRs' on: schedule: - - cron: '11 * * * *' + - cron: '11 3 * * *' jobs: lock: runs-on: ubuntu-latest From 500c02b28aacf403a1b829e4c3215f10a2b76e8f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 15 Dec 2020 18:52:24 +0300 Subject: [PATCH 237/417] ci(release): Move to getsentry/publish for releases (#775) A copy of getsentry/sentry#22657 with fixes included. That said this one is a bit different. We used to use the extra option `no-merge` when publishing to keep the release branches and also keep master on nightly versions. If we want to keep this, we need to add per-project overrides to getsentry/publish which would increase complexity at this early stage for this fringe case. Instead, I opted to follow what getsentry/sentry does: merge but then immediately after that update the version. --- .craft.yml | 2 +- .github/workflows/release.yml | 44 +++++++++++------------------------ scripts/bump-version.sh | 2 +- scripts/post-release.sh | 10 ++++++++ 4 files changed, 26 insertions(+), 32 deletions(-) create mode 100755 scripts/post-release.sh diff --git a/.craft.yml b/.craft.yml index ad41cd872f..fcc29e73df 100644 --- a/.craft.yml +++ b/.craft.yml @@ -1,4 +1,4 @@ -minVersion: "0.10.0" +minVersion: "0.14.0" github: owner: getsentry repo: onpremise diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3a411c04ac..e8e93ad7a5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,18 +5,9 @@ on: version: description: Version to release (optional) required: false - skip_prepare: - description: Skip preparation step (assume a release branch is ready) - required: false - default: false - dry_run: - description: Do not actually cut the release - required: false - default: false force: - description: Force the release, bypassing the 'release-blocker' issue killswitch + description: Force a release even when there are release-blockers (optional) required: false - default: false schedule: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, @@ -34,29 +25,22 @@ jobs: force: ${{ github.event.inputs.force }} - uses: actions/checkout@v2 with: - token: ${{ secrets.GH_SENTRY_BOT_PAT }} + token: ${{ secrets.GH_RELEASE_PAT }} + fetch-depth: 0 - uses: getsentry/craft@master name: Craft Prepare - if: ${{ !github.event.inputs.skip_prepare }} with: action: prepare version: ${{ env.RELEASE_VERSION }} - # Wait until the builds start. Craft should do this automatically - # but it is broken now. - - run: sleep 10 - - uses: getsentry/craft@master - name: Craft Publish + - name: Request publish + if: success() + uses: actions/github-script@v3 with: - action: publish - version: ${{ env.RELEASE_VERSION }} - no_merge: '--no-merge' - env: - DRY_RUN: ${{ github.event.inputs.dry_run }} - # We need this additional step because we don't merge release branches into master to - # always keep it on nightlies - - id: bump-license-date - name: Bump license change date - if: ${{ !github.event.inputs.dry_run && !github.event.inputs.version }} - run: | - sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE - git diff --quiet || git commit -anm 'license: Update BSL change date' && git push + github-token: ${{ secrets.GH_RELEASE_PAT }} + script: | + const repoInfo = context.repo; + await github.issues.create({ + owner: repoInfo.owner, + repo: 'publish', + title: `publish: ${repoInfo.repo}@${process.env.RELEASE_VERSION}`, + }); diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 4172c7e4d3..4c8bb5a5d3 100644 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -7,7 +7,7 @@ cd $SCRIPT_DIR/.. OLD_VERSION="$1" NEW_VERSION="$2" -SYMBOLICATOR_VERSION=$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")') +SYMBOLICATOR_VERSION=${SYMBOLICATOR_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env diff --git a/scripts/post-release.sh b/scripts/post-release.sh new file mode 100755 index 0000000000..ff8e7066c2 --- /dev/null +++ b/scripts/post-release.sh @@ -0,0 +1,10 @@ +#!/bin/bash +set -eu + +SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +cd $SCRIPT_DIR/.. + +# Bring master back to nightlies after merge from release branch + +SYMBOLICATOR_VERSION=nightly ./scripts/bump-version.sh '' 'nightly' +git diff --quiet || git commit -anm 'build: Set master version to nightly' && git push From 3d98c89d755d5124a23345b9d59200eaaf012cb8 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Tue, 15 Dec 2020 21:10:16 +0000 Subject: [PATCH 238/417] release: 20.12.1 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..9b28141080 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:20.12.1 +SNUBA_IMAGE=getsentry/snuba:20.12.1 +RELAY_IMAGE=getsentry/relay:20.12.1 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.2 diff --git a/LICENSE b/LICENSE index 18ffae0c3f..1b36183325 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-09-15 +Change Date: 2023-12-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index a6d261dfc4..8979b794b8 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry Nightly [![Build Status][build-status-image]][build-status-url] +# Self-Hosted Sentry 20.12.1 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From bd6817e8fb4f380e187f28bad88ee774e8c60b60 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 16 Dec 2020 00:53:28 +0300 Subject: [PATCH 239/417] build: Set master version to nightly --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 9b28141080..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:20.12.1 -SNUBA_IMAGE=getsentry/snuba:20.12.1 -RELAY_IMAGE=getsentry/relay:20.12.1 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.2 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/LICENSE b/LICENSE index 1b36183325..ca3c614b92 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-12-15 +Change Date: 2023-12-16 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 8979b794b8..7211979b2f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 20.12.1 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From cab86ac60cc658ff1321b9d8c762f6b263ae0e2b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 16 Dec 2020 00:55:07 +0300 Subject: [PATCH 240/417] fix(release): Mark scripts/bump-version.sh as +x --- scripts/bump-version.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 scripts/bump-version.sh diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh old mode 100644 new mode 100755 From 5adff50dedba8c4eefe4f7ea82c79cc99e2f5077 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 17 Dec 2020 12:01:28 -0500 Subject: [PATCH 241/417] meta: Pin lock-threads, crank frequency to test (#778) Per https://github.com/getsentry/sentry/pull/22754#discussion_r544690140. --- .github/workflows/lock.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 6865759ab9..5e62c4f57b 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,12 +1,12 @@ name: 'lock closed issues/PRs' on: schedule: - - cron: '11 3 * * *' + - cron: '*/2 * * * *' jobs: lock: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@v2 + - uses: dessant/lock-threads@63786a6c74ee3cfc4584f36de4360305c55e5127 with: github-token: ${{ github.token }} issue-lock-inactive-days: 15 From 6e275c2e0d4aee88e0237824253e99bb146b86a7 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 17 Dec 2020 13:00:04 -0500 Subject: [PATCH 242/417] Futz with action pin syntax (#779) YOLO --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 5e62c4f57b..e8fe37a264 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -6,7 +6,7 @@ jobs: lock: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@63786a6c74ee3cfc4584f36de4360305c55e5127 + - uses: dessant/lock-threads@63786a6 with: github-token: ${{ github.token }} issue-lock-inactive-days: 15 From 9a80a19fd1ffbddfe795f51fc620ce012fca15db Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 17 Dec 2020 14:09:51 -0500 Subject: [PATCH 243/417] Pin works, dial back to daily (#780) CWhy --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index e8fe37a264..5df567b800 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,7 +1,7 @@ name: 'lock closed issues/PRs' on: schedule: - - cron: '*/2 * * * *' + - cron: '11 3 * * *' jobs: lock: runs-on: ubuntu-latest From 5e39bfd1f6d10b8fff18aa554019214ee9d4e17d Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 4 Jan 2021 16:31:52 -0500 Subject: [PATCH 244/417] meta: Install stalebot (#784) --- .github/workflows/stale.yml | 47 +++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..0e2edeeb7f --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,47 @@ +name: 'close stale issues/PRs' +on: + schedule: + - cron: '* */6 * * *' + workflow_dispatch: +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@87c2b794b9b47a9bec68ae03c01aeb572ffebdb1 + with: + repo-token: ${{ github.token }} + days-before-stale: 21 + days-before-close: 7 + only-labels: "" + operations-per-run: 100 + remove-stale-when-updated: true + debug-only: false + ascending: false + + exempt-issue-labels: "Status: Accepted" + stale-issue-label: "Status: Stale" + stale-issue-message: |- + This issue has gone three weeks without activity. In another week, I will close it. + + But! If you comment or otherwise update it, I will reset the clock, and if you label it `Status: Accepted`, I will leave it alone ... forever! + + ---- + + "A weed is but an unloved flower." ― _Ella Wheeler Wilcox_ 🥀 + skip-stale-issue-message: false + close-issue-label: "" + close-issue-message: "" + + exempt-pr-labels: "Status: Accepted" + stale-pr-label: "Status: Stale" + stale-pr-message: |- + This pull request has gone three weeks without activity. In another week, I will close it. + + But! If you comment or otherwise update it, I will reset the clock, and if you label it `Status: Accepted`, I will leave it alone ... forever! + + ---- + + "A weed is but an unloved flower." ― _Ella Wheeler Wilcox_ 🥀 + skip-stale-pr-message: false + close-pr-label: + close-pr-message: "" From 70877f64bb8213eb87179009c425e8703508b630 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 6 Jan 2021 00:38:03 +0300 Subject: [PATCH 245/417] ci(release): Pin action-prepare-release to 33507ed (#792) This is to avoid any issues when getsentry/action-prepare-release#4 is merged. --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e8e93ad7a5..6120071c31 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,7 +19,7 @@ jobs: name: 'Release a new version' steps: - name: Prepare release - uses: getsentry/action-prepare-release@main + uses: getsentry/action-prepare-release@33507ed with: version: ${{ github.event.inputs.version }} force: ${{ github.event.inputs.force }} From 2dadbda1cc44e5a9a1c1a84f6dd0306258f17a47 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 7 Jan 2021 11:10:04 -0500 Subject: [PATCH 246/417] Bump RAM requirement Low-hanging fruit on https://github.com/getsentry/onpremise/issues/787. --- README.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/README.md b/README.md index 7211979b2f..2ccd7685ed 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke * Docker 19.03.6+ * Compose 1.24.1+ - -## Minimum Hardware Requirements: - - * You need at least 2400MB RAM + * 8 GB RAM ## Setup From 99e71b180c5e432465b397e3be9eaaa219d42df0 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 7 Jan 2021 21:57:40 +0300 Subject: [PATCH 247/417] ci(release): Upgrade action-prepare-release to latest version (#798) * ci(release): Upgrade action-prepare-release to latest version This version reduces the boilerplate needed and offers much better publish request issue context. * use the version tag --- .github/workflows/release.yml | 31 ++++++++----------------------- 1 file changed, 8 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6120071c31..0aa9715915 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -12,35 +12,20 @@ on: # We want the release to be at 10 or 11am Pacific Time # We also make this an hour after all others such as Sentry, # Snuba, and Relay to make sure their releases finish. - - cron: '0 18 15 * *' + - cron: "0 18 15 * *" jobs: release: runs-on: ubuntu-latest - name: 'Release a new version' + name: "Release a new version" steps: - - name: Prepare release - uses: getsentry/action-prepare-release@33507ed - with: - version: ${{ github.event.inputs.version }} - force: ${{ github.event.inputs.force }} - uses: actions/checkout@v2 with: token: ${{ secrets.GH_RELEASE_PAT }} fetch-depth: 0 - - uses: getsentry/craft@master - name: Craft Prepare - with: - action: prepare - version: ${{ env.RELEASE_VERSION }} - - name: Request publish - if: success() - uses: actions/github-script@v3 + - name: Prepare release + uses: getsentry/action-prepare-release@v1 + env: + GITHUB_TOKEN: ${{ secrets.GH_RELEASE_PAT }} with: - github-token: ${{ secrets.GH_RELEASE_PAT }} - script: | - const repoInfo = context.repo; - await github.issues.create({ - owner: repoInfo.owner, - repo: 'publish', - title: `publish: ${repoInfo.repo}@${process.env.RELEASE_VERSION}`, - }); + version: ${{ github.event.inputs.version }} + force: ${{ github.event.inputs.force }} From a64e3b40861b78632644fe602f1be38c6369fd5d Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 7 Jan 2021 14:15:31 -0500 Subject: [PATCH 248/417] Tell stalebot to ignore `Status: On Hold` (#799) --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 0e2edeeb7f..9344ce7957 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -18,7 +18,7 @@ jobs: debug-only: false ascending: false - exempt-issue-labels: "Status: Accepted" + exempt-issue-labels: "Status: Accepted,Status: On Hold" stale-issue-label: "Status: Stale" stale-issue-message: |- This issue has gone three weeks without activity. In another week, I will close it. @@ -32,7 +32,7 @@ jobs: close-issue-label: "" close-issue-message: "" - exempt-pr-labels: "Status: Accepted" + exempt-pr-labels: "Status: Accepted,Status: On Hold" stale-pr-label: "Status: Stale" stale-pr-message: |- This pull request has gone three weeks without activity. In another week, I will close it. From 410e4c84dfc142c233c79dfaabb58502bf37656a Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 7 Jan 2021 15:36:53 -0500 Subject: [PATCH 249/417] meta: Fork lockbot for security (#800) --- .github/workflows/lock.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 5df567b800..36189d3d62 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -2,11 +2,12 @@ name: 'lock closed issues/PRs' on: schedule: - cron: '11 3 * * *' + workflow_dispatch: jobs: lock: runs-on: ubuntu-latest steps: - - uses: dessant/lock-threads@63786a6 + - uses: getsentry/forked-action-lock-threads with: github-token: ${{ github.token }} issue-lock-inactive-days: 15 From ee34516be565bc1386680222a2c6e9336c400efe Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 7 Jan 2021 15:38:25 -0500 Subject: [PATCH 250/417] Fix syntax of uses --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 36189d3d62..840152cdce 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -7,7 +7,7 @@ jobs: lock: runs-on: ubuntu-latest steps: - - uses: getsentry/forked-action-lock-threads + - uses: getsentry/forked-action-lock-threads@latest with: github-token: ${{ github.token }} issue-lock-inactive-days: 15 From fc1c62995ae9e852370b42cc99b82dc85d3ba675 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 7 Jan 2021 23:44:38 +0300 Subject: [PATCH 251/417] ci(release): Enable the CalVer flag for release (#801) We need this for automated version numbers --- .github/workflows/release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0aa9715915..3bf94a1185 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,3 +29,4 @@ jobs: with: version: ${{ github.event.inputs.version }} force: ${{ github.event.inputs.force }} + calver: true From a71a1a9d024dfa98321bbe0d71ba9d24d846815e Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 7 Jan 2021 15:59:39 -0500 Subject: [PATCH 252/417] Fix one more time (#802) --- .github/workflows/lock.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index 840152cdce..d7fa616b46 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -7,7 +7,7 @@ jobs: lock: runs-on: ubuntu-latest steps: - - uses: getsentry/forked-action-lock-threads@latest + - uses: getsentry/forked-action-lock-threads@master with: github-token: ${{ github.token }} issue-lock-inactive-days: 15 From 4399fc8aa889e6fa5a6cb7a2bdd6e9a5647474d4 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 8 Jan 2021 09:56:52 -0500 Subject: [PATCH 253/417] Unfrobulate CI (#768) Co-authored-by: Burak Yigit Kaya --- .github/workflows/test.yml | 9 +++++---- foo | 0 2 files changed, 5 insertions(+), 4 deletions(-) create mode 100644 foo diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5cd4f64a1a..3fdce0fab8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,8 +5,8 @@ on: # is not against master). push: branches: - - "master" - - "releases/**" + - "master" + - "releases/**" pull_request: env: DOCKER_COMPOSE_VERSION: 1.24.1 @@ -14,7 +14,7 @@ jobs: test: strategy: matrix: - py2: ['', '1'] + py2: ["", "1"] runs-on: ubuntu-18.04 name: "test${{ matrix.py2 == '1' && ' PY2' || ''}}" steps: @@ -30,7 +30,8 @@ jobs: - name: Install and test env: - COMPOSE_PARALLEL_LIMIT: 10 + COMPOSE_PARALLEL_LIMIT: 50 + COMPOSE_HTTP_TIMEOUT: 450 SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} run: | ./install.sh diff --git a/foo b/foo new file mode 100644 index 0000000000..e69de29bb2 From e7ec11aa3b7ca7aecb741d568837140756b195e7 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 8 Jan 2021 10:36:34 -0500 Subject: [PATCH 254/417] Bump RAM requirement in install.sh (#803) * Bump RAM requirement in install.sh * Hard requirement vs. soft recommendation --- install.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/install.sh b/install.sh index 44c4f3e4f4..0722b5ed07 100755 --- a/install.sh +++ b/install.sh @@ -18,7 +18,8 @@ exec &> >(tee -a "$log_file") MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' -MIN_RAM=2400 # MB +MIN_RAM_HARD=4000 # MB +MIN_RAM_SOFT=8000 # MB # Increase the default 10 second SIGTERM timeout # to ensure celery queues are properly drained @@ -121,9 +122,11 @@ if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then exit 1 fi -if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM" ]]; then - echo "FAIL: Expected minimum RAM available to Docker to be $MIN_RAM MB but found $RAM_AVAILABLE_IN_DOCKER MB" +if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then + echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB" exit 1 +elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then + echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB" fi #SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) From 8d6893f0bf556d44102bba98b705a5a2176708fb Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 13 Jan 2021 11:28:00 -0500 Subject: [PATCH 255/417] Source install/geoip.sh, to work in more envs (#809) --- install.sh | 5 ++--- install/docker-aliases.sh | 3 +++ install/geoip.sh | 4 +--- 3 files changed, 6 insertions(+), 6 deletions(-) create mode 100755 install/docker-aliases.sh diff --git a/install.sh b/install.sh index 0722b5ed07..fa81b3df5d 100755 --- a/install.sh +++ b/install.sh @@ -9,8 +9,7 @@ fi # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . ./.env && set +a && . "$t" && rm "$t" && unset t -dc="docker-compose --no-ansi" -dcr="$dc run --rm" +source ./install/docker-aliases.sh # Thanks to https://unix.stackexchange.com/a/145654/108960 log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" @@ -329,7 +328,7 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then fi -./install/geoip.sh +source ./install/geoip.sh if [[ "$MINIMIZE_DOWNTIME" ]]; then diff --git a/install/docker-aliases.sh b/install/docker-aliases.sh new file mode 100755 index 0000000000..e19384b6b9 --- /dev/null +++ b/install/docker-aliases.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash +dc="docker-compose --no-ansi" +dcr="$dc run --rm" diff --git a/install/geoip.sh b/install/geoip.sh index ec37b21660..8d7c81a133 100755 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -2,9 +2,7 @@ if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi -dc="docker-compose --no-ansi" -dcr="$dc run --rm" - +source ./install/docker-aliases.sh install_geoip() { local mmdb='geoip/GeoLite2-City.mmdb' From 082cd73976a19af9023b323d8e58ee0a7e9159f5 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 14 Jan 2021 13:52:00 +0300 Subject: [PATCH 256/417] fix: Enable experimental reqwest library for relay (#810) As mentioned on the forum, this is stable enough and solves a big, ongoing issue with relay unable to connect to any internal services: https://forum.sentry.io/t/relay-errors-in-fresh-new-on-premise-install/9804/24 --- relay/config.example.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/relay/config.example.yml b/relay/config.example.yml index 8538bd7d46..0d70d0a2af 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -11,3 +11,5 @@ processing: - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 geoip_path: "/geoip/GeoLite2-City.mmdb" +http: + _client: "reqwest" From f2f1e7762290a5809217977f599d3ad9d2e8e3ad Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 14 Jan 2021 20:04:48 +0300 Subject: [PATCH 257/417] feat: Add Python 2 deprecation warning (w/ style) (#812) * feat: Add Python 2 deprecation warning (w/ style) * ensure successful exit * always succeed at the end * try again; set +x --- install.sh | 2 ++ install/py2-warning.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 29 insertions(+) create mode 100755 install/py2-warning.sh diff --git a/install.sh b/install.sh index fa81b3df5d..58deecb0bd 100755 --- a/install.sh +++ b/install.sh @@ -350,3 +350,5 @@ else echo " docker-compose up -d" echo "" fi + +source ./install/py2-warning.sh diff --git a/install/py2-warning.sh b/install/py2-warning.sh new file mode 100755 index 0000000000..5843c3b77d --- /dev/null +++ b/install/py2-warning.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi + +source ./install/docker-aliases.sh + +py2_warning() { + if [[ -n $($dcr --no-deps --entrypoint python web --version | grep 'Python 2') ]]; then + cat <<"EOW" + _ _ ____ ____ _ _______ ____ _____ _____ ____ _____ ______ _ _ +| || | |_ _| |_ _|/ \ |_ __ \ |_ \|_ _||_ _||_ \|_ _|.' ___ | | || | +| || | \ \ /\ / / / _ \ | |__) | | \ | | | | | \ | | / .' \_| | || | +| || | \ \/ \/ / / ___ \ | __ / | |\ \| | | | | |\ \| | | | ____ | || | +|_||_| \ /\ /_/ / \ \_ _| | \ \_ _| |_\ |_ _| |_ _| |_\ |_\ `.___] ||_||_| +(_)(_) \/ \/|____| |____||____| |___||_____|\____||_____||_____|\____|`._____.' (_)(_) + +EOW + echo 'You are using Sentry with Python 2, which is deprecated.' + echo 'Sentry 21.1 will be the last version with Python 2 support.' + fi +} + +py2_warning +# Run a simple command that would exit with code 0 so the calling script won't think +# there was a failure in this script. (otherwise it fails when Python 2 is *NOT* detected) +# as the exit code for the `grep` call will be `-1` indicating no match found. +echo '' From d8fd74f1d3bbeba5ce8f84a7ea6e47fe646d0184 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 14 Jan 2021 15:24:21 -0500 Subject: [PATCH 258/417] Standardizing on this (#814) --- .github/workflows/lock.yml | 2 +- .github/workflows/release.yml | 2 +- .github/workflows/stale.yml | 2 +- .github/workflows/test.yml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml index d7fa616b46..5d57d6f4f2 100644 --- a/.github/workflows/lock.yml +++ b/.github/workflows/lock.yml @@ -1,4 +1,4 @@ -name: 'lock closed issues/PRs' +name: 'Lock closed issues/PRs' on: schedule: - cron: '11 3 * * *' diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 3bf94a1185..1d2afb65ee 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,4 +1,4 @@ -name: release +name: Release on: workflow_dispatch: inputs: diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 9344ce7957..b2a339a68e 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -1,4 +1,4 @@ -name: 'close stale issues/PRs' +name: 'Close stale issues/PRs' on: schedule: - cron: '* */6 * * *' diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3fdce0fab8..09c9195f4a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,4 +1,4 @@ -name: test +name: Test on: # Run CI on all pushes to the master and release/** branches, and on all new # pull requests, and on all pushes to pull requests (even if a pull request From a1fcdd8db77ef9165d75793a228fba174991511a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 15 Jan 2021 00:21:01 +0300 Subject: [PATCH 259/417] ci(test): Limit concurent jobs to 1 (#815) This should keep the pool open. --- .github/workflows/test.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 09c9195f4a..1699e65c62 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -13,6 +13,8 @@ env: jobs: test: strategy: + # Only run one job at a time as they are quite demanding + max-parallel: 1 matrix: py2: ["", "1"] runs-on: ubuntu-18.04 From 25453b86493c0771b3b3a9645e572b1540e197fc Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 14 Jan 2021 16:33:17 -0500 Subject: [PATCH 260/417] Validate new issues against templates (#811) --- .github/ISSUE_TEMPLATE/bug_report.md | 25 +++++++++------- .github/ISSUE_TEMPLATE/feature_request.md | 20 ++++--------- .github/workflows/validate-new-issue.yml | 36 +++++++++++++++++++++++ 3 files changed, 55 insertions(+), 26 deletions(-) create mode 100644 .github/workflows/validate-new-issue.yml diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 5cd0918063..359d8126a5 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -1,26 +1,29 @@ --- name: 🐞 Bug Report -about: Report a bug to help improve Self-Hosted Sentry +about: Report a bug in Self-Hosted Sentry --- -## Version Information +### Version Information Version: *VERSION HERE* +### Steps to Reproduce -## Description +1. What +2. you +3. did. -[What happened] +### Expected Result -## Steps to Reproduce +What you thought would happen. -1. [First Step] -2. [Second Step] -3. and so on. +### Actual Result -## Logs +What actually happened. Maybe a screenshot/recording? -Please share any applicable logs: +### Logs -- `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` # latest instal logs +What you saw along the way, e.g.: + +- latest install logs: `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` - `docker-compose logs` output diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bb8c4ee4d7..32e32f231f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -1,28 +1,18 @@ --- -name: 🧠 Feature request -about: Suggest an idea for this project - +name: 🧠 Feature Request +about: Suggest an idea for improving Self-Hosted Sentry --- - - -## Summary +### Summary One paragraph description of the feature. -## Motivation +### Motivation Why should this be worked on? What problems or use cases does it solve or improve? -## Additional Context +### Additional Context Any other context or screenshots or API request payload/responses that pertain to the feature. diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml new file mode 100644 index 0000000000..3252396972 --- /dev/null +++ b/.github/workflows/validate-new-issue.yml @@ -0,0 +1,36 @@ +name: Validate new issue +on: + issues: + types: ['opened'] +jobs: + validate-new-issue: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: "Validate issue against templates" + shell: bash + run: | + # Look for a template where the headings match this issue's + echo "${{ github.event.issue.body }}" > issue-body + for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do + echo -n "$(basename $template)? " + # <() is process substitution - https://superuser.com/a/1060002 + if diff -rub <(grep '^#' $template) <(grep '^#' issue-body) > /dev/null; then + echo "👍 💃" + exit 0 + else + echo "👎" + fi + done + + # Failed to find a match! Close the issue. + echo "${{ github.token }}" | gh auth login --with-token + cat << EOF > comment + {"body": "Sorry, friend. As far as this ol' bot can tell, your issue does not use one of this repo's available issue templates. Please [try again using a template](https://github.com/${{ github.repository }}/issues/new/choose) so that we have the best chance of understanding and addressing your issue. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬)\n\n----\n\n[![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA)"} + EOF + + # Might get `gh issue comment` some day - https://github.com/cli/cli/issues/517 + gh api "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" \ + --method POST \ + --input comment + gh issue close ${{ github.event.issue.number }} From 0ac7eed028d12640a4b567115f8d4042d517163c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 15 Jan 2021 01:35:57 +0300 Subject: [PATCH 261/417] fix: Fix Py2 warning (#813) Removes the obsolete `echo ''` at the end, fixes detection as `python --version` outputs to `stderr` instead of `stdout` in versions prior to 3.4 or something. --- .github/workflows/test.yml | 3 +++ install/geoip.sh | 5 ++--- install/py2-warning.sh | 29 ++++++++++++----------------- 3 files changed, 17 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1699e65c62..6222568ca9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,6 +10,9 @@ on: pull_request: env: DOCKER_COMPOSE_VERSION: 1.24.1 +defaults: + run: + shell: bash jobs: test: strategy: diff --git a/install/geoip.sh b/install/geoip.sh index 8d7c81a133..c90b56d7dd 100755 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi +if [[ ! -f 'install.sh' ]]; then echo 'Where are you?'; exit 1; fi source ./install/docker-aliases.sh @@ -25,8 +25,7 @@ install_geoip() { else echo "IP address geolocation is configured for updates." echo "Updating IP address geolocation database ... " - $dcr geoipupdate - if [ $? -gt 0 ]; then + if ! $dcr geoipupdate; then result='Error' fi echo "$result updating IP address geolocation database." diff --git a/install/py2-warning.sh b/install/py2-warning.sh index 5843c3b77d..245789314b 100755 --- a/install/py2-warning.sh +++ b/install/py2-warning.sh @@ -1,27 +1,22 @@ #!/usr/bin/env bash -if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi +if [[ ! -f 'install.sh' ]]; then echo 'Where are you?'; exit 1; fi source ./install/docker-aliases.sh -py2_warning() { - if [[ -n $($dcr --no-deps --entrypoint python web --version | grep 'Python 2') ]]; then - cat <<"EOW" +# Note the stderr>stdout redirection because Python thinks `--version` should +# be on stderr: https://stackoverflow.com/a/31715011/90297 +if $dcr --no-deps --entrypoint python web --version 2>&1 | grep -q 'Python 2'; then + echo " _ _ ____ ____ _ _______ ____ _____ _____ ____ _____ ______ _ _ | || | |_ _| |_ _|/ \ |_ __ \ |_ \|_ _||_ _||_ \|_ _|.' ___ | | || | | || | \ \ /\ / / / _ \ | |__) | | \ | | | | | \ | | / .' \_| | || | | || | \ \/ \/ / / ___ \ | __ / | |\ \| | | | | |\ \| | | | ____ | || | -|_||_| \ /\ /_/ / \ \_ _| | \ \_ _| |_\ |_ _| |_ _| |_\ |_\ `.___] ||_||_| -(_)(_) \/ \/|____| |____||____| |___||_____|\____||_____||_____|\____|`._____.' (_)(_) +|_||_| \ /\ /_/ / \ \_ _| | \ \_ _| |_\ |_ _| |_ _| |_\ |_\ \`.___] ||_||_| +(_)(_) \/ \/|____| |____||____| |___||_____|\____||_____||_____|\____|\`._____.' (_)(_) -EOW - echo 'You are using Sentry with Python 2, which is deprecated.' - echo 'Sentry 21.1 will be the last version with Python 2 support.' - fi -} - -py2_warning -# Run a simple command that would exit with code 0 so the calling script won't think -# there was a failure in this script. (otherwise it fails when Python 2 is *NOT* detected) -# as the exit code for the `grep` call will be `-1` indicating no match found. -echo '' +" + echo '-----------------------------------------------------------' + echo 'You are using Sentry with Python 2, which is deprecated.' + echo 'Sentry 21.1 will be the last version with Python 2 support.' +fi From 7093bb4d776205379b53d33d959ba2683e0ca656 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Fri, 15 Jan 2021 18:45:41 +0000 Subject: [PATCH 262/417] release: 21.1.0 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..adb696ff51 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.1.0 +SNUBA_IMAGE=getsentry/snuba:21.1.0 +RELAY_IMAGE=getsentry/relay:21.1.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.2 diff --git a/LICENSE b/LICENSE index ca3c614b92..17ad4038e7 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2023-12-16 +Change Date: 2024-01-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 2ccd7685ed..f8c480d8f7 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.1.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From cc31349cc4ee043caf132e786f547c795a97848a Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 15 Jan 2021 23:16:49 +0300 Subject: [PATCH 263/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index adb696ff51..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.1.0 -SNUBA_IMAGE=getsentry/snuba:21.1.0 -RELAY_IMAGE=getsentry/relay:21.1.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.2 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index f8c480d8f7..2ccd7685ed 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.1.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 2e4307b553daf77ca751a7360457d6af125468ca Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 15 Jan 2021 23:20:39 +0300 Subject: [PATCH 264/417] ci: Increase pool size to prevent pool closed errors (#819) Based on my reading, this error comes from urllib3. https://urllib3.readthedocs.io/en/latest/advanced-usage.html >The behavior of the pooling for ConnectionPool is different from PoolManager. By default, if a new request is made and there is no free connection in the pool then a new connection will be created. However, this connection will not be saved if more than maxsize connections exist. This means that maxsize does not determine the maximum number of connections that can be open to a particular host, just the maximum number of connections to keep in the pool. I think the ideal solution would be to add `retry` and `block` options to [docker-py](https://github.com/docker/docker-py/blob/ce2669e3edfe5d3215ba501cc9771fc0ffad680a/docker/transport/unixconn.py#L58-L70) but that's a long shot. --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6222568ca9..659fc79f4f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Install and test env: - COMPOSE_PARALLEL_LIMIT: 50 + COMPOSE_PARALLEL_LIMIT: 100 COMPOSE_HTTP_TIMEOUT: 450 SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} run: | From 2e3ad5df88610ea0d329c93e2ec7a1f9655a7ef6 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 15 Jan 2021 15:29:17 -0500 Subject: [PATCH 265/417] Improve on issue validation (#817) --- .github/workflows/validate-new-issue.yml | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index 3252396972..3257296cf4 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -9,7 +9,18 @@ jobs: - uses: actions/checkout@v2 - name: "Validate issue against templates" shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} run: | + + # Trust users who belong to the getsentry org. + if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then + echo "Skipping validation, because ${{ github.actor }} is a member of the getsentry org." + exit 0 + else + echo "${{ github.actor }} is not a member of the getsentry org. 🧐" + fi + # Look for a template where the headings match this issue's echo "${{ github.event.issue.body }}" > issue-body for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do @@ -24,13 +35,14 @@ jobs: done # Failed to find a match! Close the issue. - echo "${{ github.token }}" | gh auth login --with-token cat << EOF > comment {"body": "Sorry, friend. As far as this ol' bot can tell, your issue does not use one of this repo's available issue templates. Please [try again using a template](https://github.com/${{ github.repository }}/issues/new/choose) so that we have the best chance of understanding and addressing your issue. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬)\n\n----\n\n[![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA)"} EOF # Might get `gh issue comment` some day - https://github.com/cli/cli/issues/517 + echo -n "Commented: " gh api "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" \ --method POST \ - --input comment + --input comment \ + | jq .html_url gh issue close ${{ github.event.issue.number }} From 49f8684808435e6447796b97c5cc70288c155951 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 19 Jan 2021 20:43:33 +0300 Subject: [PATCH 266/417] ci(test): Even larger pools (#825) Attepmt to fix #823 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 659fc79f4f..4e4c4d02ea 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,7 +35,7 @@ jobs: - name: Install and test env: - COMPOSE_PARALLEL_LIMIT: 100 + COMPOSE_PARALLEL_LIMIT: 200 COMPOSE_HTTP_TIMEOUT: 450 SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} run: | From af7f50c883bf3d3144bc68db22af6f3657302321 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 19 Jan 2021 20:56:13 +0300 Subject: [PATCH 267/417] ci(release): Make sure to pull before pushing in post-release (#820) Fixes the issue we had over at getsentry/publish#60. This aligns the script with [the one at getsentry/sentry](https://github.com/getsentry/sentry/blob/master/scripts/post-release.sh#L6) --- scripts/post-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/post-release.sh b/scripts/post-release.sh index ff8e7066c2..652a792720 100755 --- a/scripts/post-release.sh +++ b/scripts/post-release.sh @@ -7,4 +7,4 @@ cd $SCRIPT_DIR/.. # Bring master back to nightlies after merge from release branch SYMBOLICATOR_VERSION=nightly ./scripts/bump-version.sh '' 'nightly' -git diff --quiet || git commit -anm 'build: Set master version to nightly' && git push +git diff --quiet || git commit -anm 'build: Set master version to nightly' && git pull --rebase && git push From 65fdb3bd3d361fb84413400bad0bc3e537701b1c Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 19 Jan 2021 14:03:54 -0500 Subject: [PATCH 268/417] Remove injection vector (#822) --- .github/workflows/validate-new-issue.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index 3257296cf4..bfbb492fe1 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -22,7 +22,7 @@ jobs: fi # Look for a template where the headings match this issue's - echo "${{ github.event.issue.body }}" > issue-body + jq -r .issue.body "$GITHUB_EVENT_PATH" > issue-body for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do echo -n "$(basename $template)? " # <() is process substitution - https://superuser.com/a/1060002 From 640e7fe290a1e1962db2291692de29158d7866f7 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 19 Jan 2021 23:43:27 +0300 Subject: [PATCH 269/417] ci(test): Fix 'pool is closed' errors (#826) Fixes #823. In `install.sh` we build a local sentry image that is used by many services from using the build context under the `./sentry` directory. To avoid building this image multiple times, we also give it a specific name which is referred from multiple services. The issue is, we also run `docker-compose build --parallel` which creates a race condition when building this image as `docker-compose` doesn't check whether the image is already there or not. This is the root cause of all these random failures: an unsurprising race condition. --- .github/workflows/test.yml | 4 ---- install.sh | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4e4c4d02ea..11a7a226a0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -16,8 +16,6 @@ defaults: jobs: test: strategy: - # Only run one job at a time as they are quite demanding - max-parallel: 1 matrix: py2: ["", "1"] runs-on: ubuntu-18.04 @@ -35,8 +33,6 @@ jobs: - name: Install and test env: - COMPOSE_PARALLEL_LIMIT: 200 - COMPOSE_HTTP_TIMEOUT: 450 SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} run: | ./install.sh diff --git a/install.sh b/install.sh index 58deecb0bd..d013e16ab7 100755 --- a/install.sh +++ b/install.sh @@ -226,7 +226,7 @@ echo "Building and tagging Docker images..." echo "" # Build the sentry onpremise image first as it is needed for the cron image $dc build --force-rm web -$dc build --force-rm --parallel +$dc build --force-rm echo "" echo "Docker images built." From 612a14c63cc6eb4b4e7d0bbcffd10c6a42b4b25a Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 19 Jan 2021 17:30:07 -0500 Subject: [PATCH 270/417] Add some structure to logging with ::group:: (#827) --- .github/workflows/test.yml | 3 +- foo | 0 install.sh | 88 ++++++++++++++++++++++++-------------- test.sh | 8 ++++ 4 files changed, 67 insertions(+), 32 deletions(-) delete mode 100644 foo diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 11a7a226a0..f64918035f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -35,9 +35,10 @@ jobs: env: SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} run: | + echo "Testing initial install" ./install.sh ./test.sh - printf "Testing in-place upgrade" + echo "Testing in-place upgrade" ./install.sh --minimize-downtime ./test.sh diff --git a/foo b/foo deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/install.sh b/install.sh index d013e16ab7..fc1710c75e 100755 --- a/install.sh +++ b/install.sh @@ -6,15 +6,23 @@ if [[ -n "$MSYSTEM" ]]; then exit 1 fi +# Thanks to https://unix.stackexchange.com/a/145654/108960 +log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" +exec &> >(tee -a "$log_file") +if [ "$GITHUB_ACTIONS" = "true" ]; then + _group="::group::" + _endgroup="::endgroup::" +else + _group="▶ " + _endgroup="" +fi + +echo "${_group}Defining variables and helpers ..." # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . ./.env && set +a && . "$t" && rm "$t" && unset t source ./install/docker-aliases.sh -# Thanks to https://unix.stackexchange.com/a/145654/108960 -log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" -exec &> >(tee -a "$log_file") - MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM_HARD=4000 # MB @@ -32,7 +40,9 @@ RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' MINIMIZE_DOWNTIME= +echo $_endgroup +echo "${_group}Parsing command line ..." show_help() { cat </dev/null | awk '/Mem/ {print $2}'); @@ -138,9 +149,9 @@ if [[ "$IS_KVM" -eq 0 ]]; then exit 1 fi fi +echo "${_endgroup}" -echo "" -echo "Creating volumes for persistent storage..." +echo "${_group}Creating volumes for persistent storage ..." echo "Created $(docker volume create --name=sentry-data)." echo "Created $(docker volume create --name=sentry-postgres)." echo "Created $(docker volume create --name=sentry-redis)." @@ -148,17 +159,18 @@ echo "Created $(docker volume create --name=sentry-zookeeper)." echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." +echo "${_endgroup}" -echo "" +echo "${_group}Ensuring files from examples ..." ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS ensure_file_from_example $SYMBOLICATOR_CONFIG_YML ensure_file_from_example $RELAY_CONFIG_YML +echo "${_endgroup}" +echo "${_group}Generating secret key ..." if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then - echo "" - echo "Generating secret key..." # This is to escape the secret key to be used in sed below # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 @@ -166,7 +178,9 @@ if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML echo "Secret key written to $SENTRY_CONFIG_YML" fi +echo "${_endgroup}" +echo "${_group}Replacing TSDB ..." replace_tsdb() { if ( [[ -f "$SENTRY_CONFIG_PY" ]] && @@ -209,10 +223,9 @@ SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)} } replace_tsdb +echo "${_endgroup}" -echo "" -echo "Fetching and updating Docker images..." -echo "" +echo "${_group}Fetching and updating Docker images ..." # We tag locally built images with an '-onpremise-local' suffix. docker-compose pull tries to pull these too and # shows a 404 error on the console which is confusing and unnecessary. To overcome this, we add the stderr>stdout # redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. @@ -220,16 +233,18 @@ $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2} || true; +echo "${_endgroup}" -echo "" -echo "Building and tagging Docker images..." +echo "${_group}Building and tagging Docker images ..." echo "" # Build the sentry onpremise image first as it is needed for the cron image $dc build --force-rm web $dc build --force-rm echo "" echo "Docker images built." +echo "${_endgroup}" +echo "${_group}Turning things off ..." if [[ -n "$MINIMIZE_DOWNTIME" ]]; then # Stop everything but relay and nginx $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') @@ -240,7 +255,9 @@ else # This is for newer versions $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans fi +echo "${_endgroup}" +echo "${_group}Setting up Zookeeper ..." ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') @@ -251,24 +268,27 @@ if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper fi fi +echo "${_endgroup}" -echo "Bootstrapping and migrating Snuba..." +echo "${_group}Bootstrapping and migrating Snuba ..." $dcr snuba-api bootstrap --no-migrate --force $dcr snuba-api migrations migrate --force -echo "" +echo "${_endgroup}" +echo "${_group}Creating additional Kafka topics ..." # NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step # XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events" for topic in $NEEDED_KAFKA_TOPICS; do if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then - echo "Creating additional Kafka topics..." $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 echo "" fi done +echo "${_endgroup}" +echo "${_group}Ensuring proper PostgreSQL version ..." # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then docker volume rm sentry-postgres-new || true @@ -289,9 +309,9 @@ if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run # Finally, remove the new old volume as we are all in sentry-postgres now docker volume rm sentry-postgres-new fi +echo "${_endgroup}" -echo "" -echo "Setting up database..." +echo "${_group}Setting up database ..." if [[ -n "$CI" || "$SKIP_USER_PROMPT" == 1 ]]; then $dcr web upgrade --noinput echo "" @@ -303,21 +323,20 @@ if [[ -n "$CI" || "$SKIP_USER_PROMPT" == 1 ]]; then else $dcr web upgrade fi +echo "${_endgroup}" - +echo "${_group}Migrating file storage ..." SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then - echo "Migrating file storage..." # Use the web (Sentry) image so the file owners are kept as sentry:sentry # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. $dcr --entrypoint \"/bin/bash\" web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi +echo "${_endgroup}" - +echo "${_group}Generating Relay credentials ..." if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then - echo "" - echo "Generating Relay credentials..." # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but @@ -325,30 +344,37 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then # credentials file before relay runs. $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" + echo "${_endgroup}" fi - +echo "${_group}Setting up GeoIP integration ..." source ./install/geoip.sh - +echo "${_endgroup}" if [[ "$MINIMIZE_DOWNTIME" ]]; then + echo "${_group}Waiting for Sentry to start ..." # Start the whole setup, except nginx and relay. $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') $dc exec -T nginx service nginx reload - echo "Waiting for Sentry to start..." docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' # Make sure everything is up. This should only touch relay and nginx $dc up -d + echo "${_endgroup}" else echo "" - echo "----------------" + echo "-----------------------------------------------------------------" + echo "" echo "You're all done! Run the following command to get Sentry running:" echo "" echo " docker-compose up -d" echo "" + echo "-----------------------------------------------------------------" + echo "" fi +echo "${_group}Checking Python version ..." source ./install/py2-warning.sh +echo "${_endgroup}" diff --git a/test.sh b/test.sh index 91cfc80139..b4028dc518 100755 --- a/test.sh +++ b/test.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -e +echo "::group::Setting up variables and helpers ..." export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' @@ -31,13 +32,17 @@ cleanup () { echo "Done." } trap_with_arg cleanup ERR INT TERM EXIT +echo "::endgroup::" +echo "::group::Starting Sentry for tests ..." # Disable beacon for e2e tests echo 'SENTRY_BEACON=False' >> sentry/sentry.conf.py docker-compose run --rm web createuser --superuser --email $TEST_USER --password $TEST_PASS || true docker-compose up -d printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' +echo "::endgroup::" +echo "::group::Running tests ..." get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } @@ -70,7 +75,9 @@ do echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done +echo "::endgroup::" +echo "::group::Running moar tests !!!" # Set up initial/required settings (InstallWizard request) sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null @@ -105,3 +112,4 @@ do echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done +echo "::endgroup::" From ee53f18ad0b87ece054faa2d343dc300b4e65601 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 20 Jan 2021 18:28:14 +0300 Subject: [PATCH 271/417] breaking: Remove Python 2 support (#833) --- .github/workflows/test.yml | 7 +- README.md | 2 +- docker-compose.yml | 174 ++++++++++++++++++------------------- install.sh | 6 +- install/py2-warning.sh | 22 ----- sentry/Dockerfile | 3 +- 6 files changed, 91 insertions(+), 123 deletions(-) delete mode 100755 install/py2-warning.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f64918035f..794a01d7d0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,11 +15,8 @@ defaults: shell: bash jobs: test: - strategy: - matrix: - py2: ["", "1"] runs-on: ubuntu-18.04 - name: "test${{ matrix.py2 == '1' && ' PY2' || ''}}" + name: "test" steps: - name: Pin docker-compose run: | @@ -32,8 +29,6 @@ jobs: uses: actions/checkout@v2 - name: Install and test - env: - SENTRY_PYTHON2: ${{ matrix.py2 == '1' || '' }} run: | echo "Testing initial install" ./install.sh diff --git a/README.md b/README.md index 2ccd7685ed..04aa8ff5b7 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Setup -To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020. If you want/need to stick with the Python 2 versions of the images, you can run `SENTRY_PYTHON2=1 ./install.sh` instead. Note that we are planning to end our Python 2 support completely by January 2021. +To get started with all the defaults, simply clone the repo and run `./install.sh` in your local check-out. Sentry uses Python 3 by default since December 4th, 2020 and Sentry 21.1.0 is the last version to support Python 2. During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --no-user-prompt`. diff --git a/docker-compose.yml b/docker-compose.yml index 160ae6294c..d25e054498 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,13 +1,12 @@ -version: '3.4' +version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped x-sentry-defaults: &sentry_defaults - << : *restart_policy + <<: *restart_policy build: context: ./sentry args: - SENTRY_IMAGE - - SENTRY_PYTHON2 image: sentry-onpremise-local depends_on: - redis @@ -23,101 +22,101 @@ x-sentry-defaults: &sentry_defaults - symbolicator - kafka environment: - SENTRY_CONF: '/etc/sentry' - SNUBA: 'http://snuba-api:1218' + SENTRY_CONF: "/etc/sentry" + SNUBA: "http://snuba-api:1218" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: volumes: - - 'sentry-data:/data' - - './sentry:/etc/sentry' - - './geoip:/geoip:ro' + - "sentry-data:/data" + - "./sentry:/etc/sentry" + - "./geoip:/geoip:ro" x-snuba-defaults: &snuba_defaults - << : *restart_policy + <<: *restart_policy depends_on: - redis - clickhouse - kafka - image: '$SNUBA_IMAGE' + image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker CLICKHOUSE_HOST: clickhouse - DEFAULT_BROKERS: 'kafka:9092' + DEFAULT_BROKERS: "kafka:9092" REDIS_HOST: redis - UWSGI_MAX_REQUESTS: '10000' - UWSGI_DISABLE_LOGGING: 'true' + UWSGI_MAX_REQUESTS: "10000" + UWSGI_DISABLE_LOGGING: "true" # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: services: smtp: - << : *restart_policy + <<: *restart_policy image: tianon/exim4 volumes: - - 'sentry-smtp:/var/spool/exim4' - - 'sentry-smtp-log:/var/log/exim4' + - "sentry-smtp:/var/spool/exim4" + - "sentry-smtp-log:/var/log/exim4" memcached: - << : *restart_policy - image: 'memcached:1.5-alpine' + <<: *restart_policy + image: "memcached:1.5-alpine" redis: - << : *restart_policy - image: 'redis:5.0-alpine' + <<: *restart_policy + image: "redis:5.0-alpine" volumes: - - 'sentry-redis:/data' + - "sentry-redis:/data" ulimits: nofile: soft: 10032 hard: 10032 postgres: - << : *restart_policy - image: 'postgres:9.6' + <<: *restart_policy + image: "postgres:9.6" environment: - POSTGRES_HOST_AUTH_METHOD: 'trust' + POSTGRES_HOST_AUTH_METHOD: "trust" volumes: - - 'sentry-postgres:/var/lib/postgresql/data' + - "sentry-postgres:/var/lib/postgresql/data" zookeeper: - << : *restart_policy - image: 'confluentinc/cp-zookeeper:5.5.0' + <<: *restart_policy + image: "confluentinc/cp-zookeeper:5.5.0" environment: - ZOOKEEPER_CLIENT_PORT: '2181' - CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: 'WARN' - ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: 'WARN' + ZOOKEEPER_CLIENT_PORT: "2181" + CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" + ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" volumes: - - 'sentry-zookeeper:/var/lib/zookeeper/data' - - 'sentry-zookeeper-log:/var/lib/zookeeper/log' - - 'sentry-secrets:/etc/zookeeper/secrets' + - "sentry-zookeeper:/var/lib/zookeeper/data" + - "sentry-zookeeper-log:/var/lib/zookeeper/log" + - "sentry-secrets:/etc/zookeeper/secrets" kafka: - << : *restart_policy + <<: *restart_policy depends_on: - zookeeper - image: 'confluentinc/cp-kafka:5.5.0' + image: "confluentinc/cp-kafka:5.5.0" environment: - KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181' - KAFKA_ADVERTISED_LISTENERS: 'PLAINTEXT://kafka:9092' - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' - KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: '1' - KAFKA_LOG_RETENTION_HOURS: '24' - KAFKA_MESSAGE_MAX_BYTES: '50000000' #50MB or bust - KAFKA_MAX_REQUEST_SIZE: '50000000' #50MB on requests apparently too - CONFLUENT_SUPPORT_METRICS_ENABLE: 'false' - KAFKA_LOG4J_LOGGERS: 'kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN' - KAFKA_LOG4J_ROOT_LOGLEVEL: 'WARN' - KAFKA_TOOLS_LOG4J_LOGLEVEL: 'WARN' + KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: "1" + KAFKA_OFFSETS_TOPIC_NUM_PARTITIONS: "1" + KAFKA_LOG_RETENTION_HOURS: "24" + KAFKA_MESSAGE_MAX_BYTES: "50000000" #50MB or bust + KAFKA_MAX_REQUEST_SIZE: "50000000" #50MB on requests apparently too + CONFLUENT_SUPPORT_METRICS_ENABLE: "false" + KAFKA_LOG4J_LOGGERS: "kafka.cluster=WARN,kafka.controller=WARN,kafka.coordinator=WARN,kafka.log=WARN,kafka.server=WARN,kafka.zookeeper=WARN,state.change.logger=WARN" + KAFKA_LOG4J_ROOT_LOGLEVEL: "WARN" + KAFKA_TOOLS_LOG4J_LOGLEVEL: "WARN" volumes: - - 'sentry-kafka:/var/lib/kafka/data' - - 'sentry-kafka-log:/var/lib/kafka/log' - - 'sentry-secrets:/etc/kafka/secrets' + - "sentry-kafka:/var/lib/kafka/data" + - "sentry-kafka-log:/var/lib/kafka/log" + - "sentry-secrets:/etc/kafka/secrets" clickhouse: - << : *restart_policy - image: 'yandex/clickhouse-server:20.3.9.70' + <<: *restart_policy + image: "yandex/clickhouse-server:20.3.9.70" ulimits: nofile: soft: 262144 hard: 262144 volumes: - - 'sentry-clickhouse:/var/lib/clickhouse' - - 'sentry-clickhouse-log:/var/log/clickhouse-server' + - "sentry-clickhouse:/var/lib/clickhouse" + - "sentry-clickhouse-log:/var/log/clickhouse-server" - type: bind read_only: true source: ./clickhouse/config.xml @@ -128,104 +127,105 @@ services: # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 geoipupdate: - image: 'maxmindinc/geoipupdate:latest' + image: "maxmindinc/geoipupdate:latest" # Override the entrypoint in order to avoid using envvars for config. # Futz with settings so we can keep mmdb and conf in same dir on host # (image looks for them in separate dirs by default). - entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf'] + entrypoint: + ["/usr/bin/geoipupdate", "-d", "/sentry", "-f", "/sentry/GeoIP.conf"] volumes: - - './geoip:/sentry' + - "./geoip:/sentry" snuba-api: - << : *snuba_defaults + <<: *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: - << : *snuba_defaults + <<: *snuba_defaults command: consumer --storage events --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration snuba-outcomes-consumer: - << : *snuba_defaults + <<: *snuba_defaults command: consumer --storage outcomes_raw --auto-offset-reset=earliest --max-batch-time-ms 750 # Kafka consumer responsible for feeding session data into Clickhouse snuba-sessions-consumer: - << : *snuba_defaults + <<: *snuba_defaults command: consumer --storage sessions_raw --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding transactions data into Clickhouse snuba-transactions-consumer: - << : *snuba_defaults + <<: *snuba_defaults command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: - << : *snuba_defaults + <<: *snuba_defaults command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: - << : *snuba_defaults + <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 snuba-subscription-consumer-transactions: - << : *snuba_defaults + <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-transactions-subscriptions-consumers --topic=events --result-topic=transactions-subscription-results --dataset=transactions --commit-log-topic=snuba-commit-log --commit-log-group=transactions_group --delay-seconds=60 --schedule-ttl=60 snuba-cleanup: - << : *snuba_defaults + <<: *snuba_defaults image: snuba-cleanup-onpremise-local build: context: ./cron args: - BASE_IMAGE: '$SNUBA_IMAGE' + BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' symbolicator: - << : *restart_policy - image: '$SYMBOLICATOR_IMAGE' + <<: *restart_policy + image: "$SYMBOLICATOR_IMAGE" volumes: - - 'sentry-symbolicator:/data' + - "sentry-symbolicator:/data" - type: bind read_only: true source: ./symbolicator target: /etc/symbolicator command: run -c /etc/symbolicator/config.yml symbolicator-cleanup: - << : *restart_policy + <<: *restart_policy image: symbolicator-cleanup-onpremise-local build: context: ./cron args: - BASE_IMAGE: '$SYMBOLICATOR_IMAGE' + BASE_IMAGE: "$SYMBOLICATOR_IMAGE" command: '"55 23 * * * gosu symbolicator symbolicator cleanup"' volumes: - - 'sentry-symbolicator:/data' + - "sentry-symbolicator:/data" web: - << : *sentry_defaults + <<: *sentry_defaults cron: - << : *sentry_defaults + <<: *sentry_defaults command: run cron worker: - << : *sentry_defaults + <<: *sentry_defaults command: run worker ingest-consumer: - << : *sentry_defaults + <<: *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: - << : *sentry_defaults + <<: *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. command: run post-process-forwarder --commit-batch-size 1 subscription-consumer-events: - << : *sentry_defaults + <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic events-subscription-results subscription-consumer-transactions: - << : *sentry_defaults + <<: *sentry_defaults command: run query-subscription-consumer --commit-batch-size 1 --topic transactions-subscription-results sentry-cleanup: - << : *sentry_defaults + <<: *sentry_defaults image: sentry-cleanup-onpremise-local build: context: ./cron args: - BASE_IMAGE: 'sentry-onpremise-local' + BASE_IMAGE: "sentry-onpremise-local" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: - << : *restart_policy + <<: *restart_policy ports: - - '$SENTRY_BIND:80/tcp' - image: 'nginx:1.16' + - "$SENTRY_BIND:80/tcp" + image: "nginx:1.16" volumes: - type: bind read_only: true @@ -235,8 +235,8 @@ services: - web - relay relay: - << : *restart_policy - image: '$RELAY_IMAGE' + <<: *restart_policy + image: "$RELAY_IMAGE" volumes: - type: bind read_only: true diff --git a/install.sh b/install.sh index fc1710c75e..2ce16baa57 100755 --- a/install.sh +++ b/install.sh @@ -232,7 +232,7 @@ echo "${_group}Fetching and updating Docker images ..." $dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true # We may not have the set image on the repo (local images) so allow fails -docker pull ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2} || true; +docker pull ${SENTRY_IMAGE} || true; echo "${_endgroup}" echo "${_group}Building and tagging Docker images ..." @@ -374,7 +374,3 @@ else echo "-----------------------------------------------------------------" echo "" fi - -echo "${_group}Checking Python version ..." -source ./install/py2-warning.sh -echo "${_endgroup}" diff --git a/install/py2-warning.sh b/install/py2-warning.sh deleted file mode 100755 index 245789314b..0000000000 --- a/install/py2-warning.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -if [[ ! -f 'install.sh' ]]; then echo 'Where are you?'; exit 1; fi - -source ./install/docker-aliases.sh - -# Note the stderr>stdout redirection because Python thinks `--version` should -# be on stderr: https://stackoverflow.com/a/31715011/90297 -if $dcr --no-deps --entrypoint python web --version 2>&1 | grep -q 'Python 2'; then - echo " - _ _ ____ ____ _ _______ ____ _____ _____ ____ _____ ______ _ _ -| || | |_ _| |_ _|/ \ |_ __ \ |_ \|_ _||_ _||_ \|_ _|.' ___ | | || | -| || | \ \ /\ / / / _ \ | |__) | | \ | | | | | \ | | / .' \_| | || | -| || | \ \/ \/ / / ___ \ | __ / | |\ \| | | | | |\ \| | | | ____ | || | -|_||_| \ /\ /_/ / \ \_ _| | \ \_ _| |_\ |_ _| |_ _| |_\ |_\ \`.___] ||_||_| -(_)(_) \/ \/|____| |____||____| |___||_____|\____||_____||_____|\____|\`._____.' (_)(_) - -" - echo '-----------------------------------------------------------' - echo 'You are using Sentry with Python 2, which is deprecated.' - echo 'Sentry 21.1 will be the last version with Python 2 support.' -fi diff --git a/sentry/Dockerfile b/sentry/Dockerfile index 812056ead1..f9484f295b 100644 --- a/sentry/Dockerfile +++ b/sentry/Dockerfile @@ -1,6 +1,5 @@ ARG SENTRY_IMAGE -ARG SENTRY_PYTHON2 -FROM ${SENTRY_IMAGE}${SENTRY_PYTHON2:+-py2} +FROM ${SENTRY_IMAGE} COPY . /usr/src/sentry From e7a3187cd01b5b7c0b5c504ed41fb6280adeb291 Mon Sep 17 00:00:00 2001 From: Mikhail Paulyshka Date: Mon, 25 Jan 2021 13:47:18 +0300 Subject: [PATCH 272/417] fix(nginx): Remove X-Real-IP header entry (#835) Removes the obsolete and confusing `X-Real-IP` header setting from the Nginx config. --- nginx/nginx.conf | 1 - 1 file changed, 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e8d56443af..ba03fb989e 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -44,7 +44,6 @@ http { # it could be "close" to close a keepalive connection proxy_set_header Connection ''; proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Request-Id $request_id; From 41f7a7e98c89f071e181ca6cb2b91b4d7758ae51 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 25 Jan 2021 16:15:45 +0300 Subject: [PATCH 273/417] docs: Add 20 GB minimum disk space requirement (#836) When trying to install self-hosted Sentry on Google Cloud Compute container-optimized OS, I hit an issue where the disk space was not sufficient to even build local images. Given Kafka and Clickhouse are also quite disk-intensive, it makes sense to add this to the docs. Opted not to add an automated check for this as it is not easy to determine the path Sentry installation will consume, hence the free space there. --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 04aa8ff5b7..3985bd7aee 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke * Docker 19.03.6+ * Compose 1.24.1+ * 8 GB RAM + * 20 GB Free Disk Space ## Setup From 54a20969a62675cb3e5b8a976c573f86806a081b Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 25 Jan 2021 15:06:36 -0500 Subject: [PATCH 274/417] Deploy latest validate-new-issue.yml (#838) --- .github/workflows/validate-new-issue.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index bfbb492fe1..78453de678 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -12,6 +12,7 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | + echo "Validating issue #${{ github.event.issue.number }}." # Trust users who belong to the getsentry org. if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then @@ -21,12 +22,16 @@ jobs: echo "${{ github.actor }} is not a member of the getsentry org. 🧐" fi - # Look for a template where the headings match this issue's - jq -r .issue.body "$GITHUB_EVENT_PATH" > issue-body + # Look for a template where all the headings are also in this issue. + # - extra headings in the issue are fine + # - order doesn't matter + # - case-sensitive tho + function extract-headings { grep '^#' "$1" | sort; } + extract-headings <(jq -r .issue.body "$GITHUB_EVENT_PATH") > headings-in-issue for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do echo -n "$(basename $template)? " - # <() is process substitution - https://superuser.com/a/1060002 - if diff -rub <(grep '^#' $template) <(grep '^#' issue-body) > /dev/null; then + extract-headings "$template" > headings-in-template + if [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then echo "👍 💃" exit 0 else From 2da7e840242154ec62159e7ebd5e26c16efcf35c Mon Sep 17 00:00:00 2001 From: arusa Date: Fri, 29 Jan 2021 14:14:56 +0100 Subject: [PATCH 275/417] Change MIN_RAM_HARD from 4000 to 3800 (#840) On machines with 4gb the available memory is often a little bit lower than 4000 --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 2ce16baa57..20fd83316e 100755 --- a/install.sh +++ b/install.sh @@ -25,7 +25,7 @@ source ./install/docker-aliases.sh MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' -MIN_RAM_HARD=4000 # MB +MIN_RAM_HARD=3800 # MB MIN_RAM_SOFT=8000 # MB # Increase the default 10 second SIGTERM timeout From c66a710182ffe9bc4f4e6a25637816726c7ccf9d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 29 Jan 2021 22:16:41 +0300 Subject: [PATCH 276/417] fix(config): extended-permissions is on github-login, not app (#841) Fixes the issue where we set an invalid option, `github-app.extended-permissions`, instead of the correct one, `github-login.extended-permissions`. Some people mentioned this warning earlier but never clearly enough to point that it was coming from our default settings suggestions. --- sentry/config.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index f42be825b2..9595a7354f 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -76,7 +76,7 @@ transaction-events.force-disable-internal-project: true # GitHub Integration # ###################### -# github-app.extended-permissions: ['repo'] +# github-login.extended-permissions: ['repo'] # github-app.id: GITHUB_APP_ID # github-app.name: 'GITHUB_APP_NAME' # github-app.webhook-secret: 'GITHUB_WEBHOOK_SECRET' # Use only if configured in GitHub From dde0b1d80ffdd29e3cc5adc98f3344a0b7acd24f Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 29 Jan 2021 16:36:20 -0500 Subject: [PATCH 277/417] Deploy action: validate-new-issue.yml (#842) --- .github/workflows/validate-new-issue.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index 78453de678..c99c9ce093 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -26,16 +26,18 @@ jobs: # - extra headings in the issue are fine # - order doesn't matter # - case-sensitive tho - function extract-headings { grep '^#' "$1" | sort; } + function extract-headings { { grep '^#' "$1" || echo -n ''; } | sort; } extract-headings <(jq -r .issue.body "$GITHUB_EVENT_PATH") > headings-in-issue for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do - echo -n "$(basename $template)? " extract-headings "$template" > headings-in-template - if [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then - echo "👍 💃" + echo -n "$(basename $template)? " + if [ ! -s headings-in-template ]; then + echo "No headers in template. 🤷" + elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then + echo "Match! 👍 💃" exit 0 else - echo "👎" + echo "No match. 👎" fi done From ab5b86747453047e758e1d9988eb7fa9dc6a966f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 1 Feb 2021 16:41:12 +0300 Subject: [PATCH 278/417] ref(requirements): Add min CPU requirement, relax soft RAM (#844) * ref(requirements): Add min CPU requirement, relax soft RAM Adds minimum of 4 CPU cores requirement as anything below will perform quite poorly even on lower loads. Relaxes the soft RAM requirement from 8000 MB to 7800 MB as even when there is 8 GB RAM installed, the system reserves some of it to itself and under reports the amount. * pass on CI with soft limit --- README.md | 1 + install.sh | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3985bd7aee..62c9b46a28 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke * Docker 19.03.6+ * Compose 1.24.1+ + * 4 CPU Cores * 8 GB RAM * 20 GB Free Disk Space diff --git a/install.sh b/install.sh index 20fd83316e..4307b201b0 100755 --- a/install.sh +++ b/install.sh @@ -26,7 +26,9 @@ source ./install/docker-aliases.sh MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM_HARD=3800 # MB -MIN_RAM_SOFT=8000 # MB +MIN_RAM_SOFT=7800 # MB +MIN_CPU_HARD=2 +MIN_CPU_SOFT=4 # Increase the default 10 second SIGTERM timeout # to ensure celery queues are properly drained @@ -108,6 +110,7 @@ echo "${_group}Checking minimum requirements ..." DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); +CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } @@ -132,6 +135,13 @@ if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then exit 1 fi +if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then + echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" + exit 1 +elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then + echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT MB, found $CPU_AVAILABLE_IN_DOCKER" +fi + if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB" exit 1 From 020ded270e50133d9a53ecaecd3febab5658887c Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 1 Feb 2021 13:40:48 -0500 Subject: [PATCH 279/417] Deploy action: validate-new-issue.yml (#845) --- .github/workflows/validate-new-issue.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index c99c9ce093..a134fd6045 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -26,16 +26,25 @@ jobs: # - extra headings in the issue are fine # - order doesn't matter # - case-sensitive tho + # - can't post a template unchanged (ignoring whitespace) function extract-headings { { grep '^#' "$1" || echo -n ''; } | sort; } - extract-headings <(jq -r .issue.body "$GITHUB_EVENT_PATH") > headings-in-issue + jq -r .issue.body "$GITHUB_EVENT_PATH" > issue + extract-headings <(cat issue) > headings-in-issue for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do + # Strip front matter. https://stackoverflow.com/a/29292490/14946704 + sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" extract-headings "$template" > headings-in-template echo -n "$(basename $template)? " if [ ! -s headings-in-template ]; then echo "No headers in template. 🤷" elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then echo "Match! 👍 💃" - exit 0 + if diff -Bw "$template" issue > /dev/null; then + echo "... like, an /exact/ match. 😖" + break + else + exit 0 + fi else echo "No match. 👎" fi @@ -48,8 +57,5 @@ jobs: # Might get `gh issue comment` some day - https://github.com/cli/cli/issues/517 echo -n "Commented: " - gh api "https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/comments" \ - --method POST \ - --input comment \ - | jq .html_url + gh issue comment ${{ github.event.issue.number }} --body "$(cat comment)" gh issue close ${{ github.event.issue.number }} From 9e94e37f637be244329ba8bd383843b194f15d59 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 2 Feb 2021 09:37:12 -0500 Subject: [PATCH 280/417] meta(gha): Deploy action validate-new-issue.yml (#848) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I are a bot. 🤖 https://github.com/getsentry/.github/blob/f51bb175b187a48f54f699740cce8acd85c09e92/.github/workflows/validate-new-issue.yml --- .github/workflows/validate-new-issue.yml | 99 ++++++++++++------------ 1 file changed, 51 insertions(+), 48 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index a134fd6045..e6d56b46f3 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -1,61 +1,64 @@ name: Validate new issue on: issues: - types: ['opened'] + types: ["opened"] jobs: validate-new-issue: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - name: "Validate issue against templates" - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - echo "Validating issue #${{ github.event.issue.number }}." + - uses: actions/checkout@v2 + - name: "Validate issue against templates" + shell: bash + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + echo "Validating issue #${{ github.event.issue.number }}." - # Trust users who belong to the getsentry org. - if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then - echo "Skipping validation, because ${{ github.actor }} is a member of the getsentry org." - exit 0 - else - echo "${{ github.actor }} is not a member of the getsentry org. 🧐" - fi + # Trust users who belong to the getsentry org. + if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then + echo "Skipping validation, because ${{ github.actor }} is a member of the getsentry org." + exit 0 + else + echo "${{ github.actor }} is not a member of the getsentry org. 🧐" + fi - # Look for a template where all the headings are also in this issue. - # - extra headings in the issue are fine - # - order doesn't matter - # - case-sensitive tho - # - can't post a template unchanged (ignoring whitespace) - function extract-headings { { grep '^#' "$1" || echo -n ''; } | sort; } - jq -r .issue.body "$GITHUB_EVENT_PATH" > issue - extract-headings <(cat issue) > headings-in-issue - for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do - # Strip front matter. https://stackoverflow.com/a/29292490/14946704 - sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" - extract-headings "$template" > headings-in-template - echo -n "$(basename $template)? " - if [ ! -s headings-in-template ]; then - echo "No headers in template. 🤷" - elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then - echo "Match! 👍 💃" - if diff -Bw "$template" issue > /dev/null; then - echo "... like, an /exact/ match. 😖" - break + # Look for a template where all the headings are also in this issue. + # - extra headings in the issue are fine + # - order doesn't matter + # - case-sensitive tho + # - can't post a template unchanged (ignoring whitespace) + function extract-headings { { sed 's/\r$//' "$1" | grep '^#' || echo -n ''; } | sort; } + jq -r .issue.body "$GITHUB_EVENT_PATH" > issue + extract-headings <(cat issue) > headings-in-issue + for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do + # Strip front matter. https://stackoverflow.com/a/29292490/14946704 + sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" + extract-headings "$template" > headings-in-template + echo -n "$(basename $template)? " + if [ ! -s headings-in-template ]; then + echo "No headers in template. 🤷" + elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then + echo "Match! 👍 💃" + if diff -Bw "$template" issue > /dev/null; then + echo "... like, an /exact/ match. 😖" + break + else + exit 0 + fi else - exit 0 + echo "No match. 👎" fi - else - echo "No match. 👎" - fi - done + done + + # Failed to find a match! Close the issue. + cat << EOF > comment + Sorry, friend. As far as this ol' bot can tell, your issue does not use one of this repo's available issue templates. Please [try again using a template](https://github.com/${{ github.repository }}/issues/new/choose) so that we have the best chance of understanding and addressing your issue. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬) + + ---- - # Failed to find a match! Close the issue. - cat << EOF > comment - {"body": "Sorry, friend. As far as this ol' bot can tell, your issue does not use one of this repo's available issue templates. Please [try again using a template](https://github.com/${{ github.repository }}/issues/new/choose) so that we have the best chance of understanding and addressing your issue. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬)\n\n----\n\n[![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA)"} - EOF + [![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA)" + EOF - # Might get `gh issue comment` some day - https://github.com/cli/cli/issues/517 - echo -n "Commented: " - gh issue comment ${{ github.event.issue.number }} --body "$(cat comment)" - gh issue close ${{ github.event.issue.number }} + echo -n "Commented: " + gh issue comment ${{ github.event.issue.number }} --body "$(cat comment)" + gh issue close ${{ github.event.issue.number }} From 5bad6ed2b492d9773062137e17c6cf3ca79bd6c1 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 2 Feb 2021 14:33:56 -0500 Subject: [PATCH 281/417] meta(gha): Deploy action validate-new-issue.yml (#849) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I are a bot, here to deploy [validate-new-issue.yml](https://github.com/getsentry/.github/blob/f663e1dd631880fa51cba2126f6192c92826c437/.github/workflows/validate-new-issue.yml). 🤖 --- .github/workflows/validate-new-issue.yml | 70 +++++++++++++++--------- 1 file changed, 43 insertions(+), 27 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index e6d56b46f3..b62d636b37 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - name: "Validate issue against templates" + - name: "Validate new issue" shell: bash env: GITHUB_TOKEN: ${{ github.token }} @@ -22,43 +22,59 @@ jobs: echo "${{ github.actor }} is not a member of the getsentry org. 🧐" fi - # Look for a template where all the headings are also in this issue. - # - extra headings in the issue are fine - # - order doesn't matter - # - case-sensitive tho - # - can't post a template unchanged (ignoring whitespace) + # Prep reasons for error message comment. + REASON="your issue does not properly use one of this repo's available issue templates" + REASON_EXACT_MATCH="you created an issue from a template without filling in anything" + REASON_EMPTY="you created an empty issue" + + # Definition of valid: + # - not empty (ignoring whitespace) + # - matches a template + # - all the headings are also in this issue + # - extra headings in the issue are fine + # - order doesn't matter + # - case-sensitive tho + # - not an *exact* match for a template (ignoring whitespace) function extract-headings { { sed 's/\r$//' "$1" | grep '^#' || echo -n ''; } | sort; } jq -r .issue.body "$GITHUB_EVENT_PATH" > issue - extract-headings <(cat issue) > headings-in-issue - for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do - # Strip front matter. https://stackoverflow.com/a/29292490/14946704 - sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" - extract-headings "$template" > headings-in-template - echo -n "$(basename $template)? " - if [ ! -s headings-in-template ]; then - echo "No headers in template. 🤷" - elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then - echo "Match! 👍 💃" - if diff -Bw "$template" issue > /dev/null; then - echo "... like, an /exact/ match. 😖" - break + if ! grep -q '[^[:space:]]' issue; then + REASON="${REASON_EMPTY}" + else + extract-headings <(cat issue) > headings-in-issue + for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do + # Strip front matter. https://stackoverflow.com/a/29292490/14946704 + sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" + extract-headings "$template" > headings-in-template + echo -n "$(basename $template)? " + if [ ! -s headings-in-template ]; then + echo "No headers in template. 🤷" + elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then + echo "Match! 👍 💃" + if diff -Bw "$template" issue > /dev/null; then + echo "... like, an /exact/ match. 😖" + REASON="${REASON_EXACT_MATCH}" + break + else + exit 0 + fi else - exit 0 + echo "No match. 👎" fi - else - echo "No match. 👎" - fi - done + done + fi - # Failed to find a match! Close the issue. + # Failed validation! Close the issue with a comment. cat << EOF > comment - Sorry, friend. As far as this ol' bot can tell, your issue does not use one of this repo's available issue templates. Please [try again using a template](https://github.com/${{ github.repository }}/issues/new/choose) so that we have the best chance of understanding and addressing your issue. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬) + Sorry, friend. As far as this ol' bot can tell, ${REASON}. Please [try again](https://github.com/${{ github.repository }}/issues/new/choose), if you like. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬) ---- - [![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA)" + [![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA) + + ([log](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})) EOF echo -n "Commented: " gh issue comment ${{ github.event.issue.number }} --body "$(cat comment)" gh issue close ${{ github.event.issue.number }} + echo "Closed with: \"${REASON}.\"" From a1c0c1fd0ca8a4f3fef91a3129a81a125bdd48b5 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 4 Feb 2021 15:15:59 +0300 Subject: [PATCH 282/417] ref: Stop building local images for Sentry services (#834) We used to build local images for Sentry services to be able to include required plugins in the image. With this change we instead do this in a custom entrypoint script and use the volume `/data` to store the plugins permanently. This should resolve many issues people have around building local images and pushing them to places like private repositories or swarm clusters. This is not 100% compatible with the old way but it should still be a mostly transparent change to many folks. --- .github/workflows/test.yml | 2 ++ docker-compose.yml | 11 +++++------ install.sh | 2 -- sentry/.dockerignore | 5 ----- sentry/Dockerfile | 7 ------- sentry/entrypoint.sh | 16 ++++++++++++++++ 6 files changed, 23 insertions(+), 20 deletions(-) delete mode 100644 sentry/.dockerignore delete mode 100644 sentry/Dockerfile create mode 100755 sentry/entrypoint.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 794a01d7d0..fb891aea8b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -34,6 +34,8 @@ jobs: ./install.sh ./test.sh echo "Testing in-place upgrade" + # Also test plugin installation here + echo "sentry-auth-oidc" >> sentry/requirements.txt ./install.sh --minimize-downtime ./test.sh diff --git a/docker-compose.yml b/docker-compose.yml index d25e054498..4a2644c904 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,11 +3,7 @@ x-restart-policy: &restart_policy restart: unless-stopped x-sentry-defaults: &sentry_defaults <<: *restart_policy - build: - context: ./sentry - args: - - SENTRY_IMAGE - image: sentry-onpremise-local + image: "$SENTRY_IMAGE" depends_on: - redis - postgres @@ -21,7 +17,10 @@ x-sentry-defaults: &sentry_defaults - snuba-replacer - symbolicator - kafka + entrypoint: "/etc/sentry/entrypoint.sh" + command: ["run", "web"] environment: + PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" # Leaving the value empty to just pass whatever is set @@ -219,7 +218,7 @@ services: build: context: ./cron args: - BASE_IMAGE: "sentry-onpremise-local" + BASE_IMAGE: "$SENTRY_IMAGE" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy diff --git a/install.sh b/install.sh index 4307b201b0..ab978185b4 100755 --- a/install.sh +++ b/install.sh @@ -247,8 +247,6 @@ echo "${_endgroup}" echo "${_group}Building and tagging Docker images ..." echo "" -# Build the sentry onpremise image first as it is needed for the cron image -$dc build --force-rm web $dc build --force-rm echo "" echo "Docker images built." diff --git a/sentry/.dockerignore b/sentry/.dockerignore deleted file mode 100644 index 693a7e0716..0000000000 --- a/sentry/.dockerignore +++ /dev/null @@ -1,5 +0,0 @@ -# Ignore everything -* - -# Only allow requirements.txt -!/requirements.txt diff --git a/sentry/Dockerfile b/sentry/Dockerfile deleted file mode 100644 index f9484f295b..0000000000 --- a/sentry/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -ARG SENTRY_IMAGE -FROM ${SENTRY_IMAGE} - -COPY . /usr/src/sentry - -# Hook for installing additional plugins -RUN if [ -s /usr/src/sentry/requirements.txt ]; then pip install -r /usr/src/sentry/requirements.txt; fi diff --git a/sentry/entrypoint.sh b/sentry/entrypoint.sh new file mode 100755 index 0000000000..55c7e4141a --- /dev/null +++ b/sentry/entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/bash +set -e + +req_file="/etc/sentry/requirements.txt" +plugins_dir="/data/custom-packages" +checksum_file="$plugins_dir/.checksum" + +if [[ -s "$req_file" ]] && ! cat "$req_file" | grep '^[^#[:space:]]' | shasum -s -c "$checksum_file" 2>/dev/null; then + echo "Installing additional dependencies..." + mkdir -p "$plugins_dir" + pip install --user -r "$req_file" + cat "$req_file" | grep '^[^#[:space:]]' | shasum > "$checksum_file" + echo "" +fi + +source /docker-entrypoint.sh From 26f11c425ce1ff6cc04694d9d649d35b70d88455 Mon Sep 17 00:00:00 2001 From: Dmitry Dygalo Date: Thu, 11 Feb 2021 15:20:19 +0100 Subject: [PATCH 283/417] fix: Typo in the "recommended minimum CPU cores" message (#855) --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index ab978185b4..fae93a41f4 100755 --- a/install.sh +++ b/install.sh @@ -139,7 +139,7 @@ if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" exit 1 elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then - echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT MB, found $CPU_AVAILABLE_IN_DOCKER" + echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER" fi if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then From fc7fc5d0ce60e4b29429f1ed31ac3ba669631674 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 15 Feb 2021 18:33:58 +0300 Subject: [PATCH 284/417] fix(cron): Fix sentry-cleanup entrypoint issue (#861) Fixes #860 and adds a test case to ensure all cleanup jobs are working. --- docker-compose.yml | 1 + test.sh | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 4a2644c904..d9d28c4259 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -219,6 +219,7 @@ services: context: ./cron args: BASE_IMAGE: "$SENTRY_IMAGE" + entrypoint: "/entrypoint.sh" command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' nginx: <<: *restart_policy diff --git a/test.sh b/test.sh index b4028dc518..4c6ca2fead 100755 --- a/test.sh +++ b/test.sh @@ -113,3 +113,7 @@ do echo "Pass." done echo "::endgroup::" + +echo "::group::Ensure cleanup crons are working ..." +docker-compose ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+" +echo "::endgroup::" From 21c7ece188bd0ca2e3885742026cf7382459c458 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Mon, 15 Feb 2021 18:20:50 +0000 Subject: [PATCH 285/417] release: 21.2.0 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..1bb63dbd57 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.2.0 +SNUBA_IMAGE=getsentry/snuba:21.2.0 +RELAY_IMAGE=getsentry/relay:21.2.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index 17ad4038e7..e31117941e 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-01-15 +Change Date: 2024-02-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 62c9b46a28..d4906ad6d4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.2.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From f9ab3e084f4df0d2f410333815c42d072d0befa7 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 15 Feb 2021 22:34:03 +0300 Subject: [PATCH 286/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 1bb63dbd57..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.2.0 -SNUBA_IMAGE=getsentry/snuba:21.2.0 -RELAY_IMAGE=getsentry/relay:21.2.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index d4906ad6d4..62c9b46a28 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.2.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 4d7021729bbea1d1dda89696e41b4e05c995bce8 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 15 Feb 2021 22:42:52 +0300 Subject: [PATCH 287/417] fix(deps): Add missing snuba consumer dependencies to Sentry (#862) These were looked over when they were added. This is not a big deal as running `docker-compose up -d` spins up all services but this fix is for correctness sake, especially for folks using this repo as a basis for more complex setups. --- docker-compose.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index d9d28c4259..e4c4de1e0f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -14,6 +14,8 @@ x-sentry-defaults: &sentry_defaults - snuba-outcomes-consumer - snuba-sessions-consumer - snuba-transactions-consumer + - snuba-subscription-consumer-events + - snuba-subscription-consumer-transactions - snuba-replacer - symbolicator - kafka From 81db8a666c111b6881ff55f965ffafef699c72ec Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 17 Feb 2021 11:19:11 -0500 Subject: [PATCH 288/417] meta(gha): Deploy action validate-new-issue.yml (#864) --- .github/workflows/validate-new-issue.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index b62d636b37..1a2d14477a 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -12,7 +12,8 @@ jobs: env: GITHUB_TOKEN: ${{ github.token }} run: | - echo "Validating issue #${{ github.event.issue.number }}." + issue_number=${{ github.event.issue.number }} + echo "Validating issue #${issue_number}." # Trust users who belong to the getsentry org. if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then @@ -55,6 +56,9 @@ jobs: REASON="${REASON_EXACT_MATCH}" break else + gh api "/repos/:owner/:repo/issues/${issue_number}/labels" \ + -X POST \ + --input <(echo '{"labels":["Status: Unrouted"]}') exit 0 fi else From 798e028c28050620546c995d74d9fa078fb3aa39 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 24 Feb 2021 09:35:59 -0500 Subject: [PATCH 289/417] meta(gha): Deploy action validate-new-issue.yml (#869) --- .github/workflows/validate-new-issue.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index 1a2d14477a..3badef86ac 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -28,7 +28,10 @@ jobs: REASON_EXACT_MATCH="you created an issue from a template without filling in anything" REASON_EMPTY="you created an empty issue" + BASE_CASE_TITLE="validation bot is confused" + # Definition of valid: + # - is a report about buggy validation 😅 or ... # - not empty (ignoring whitespace) # - matches a template # - all the headings are also in this issue @@ -36,6 +39,13 @@ jobs: # - order doesn't matter # - case-sensitive tho # - not an *exact* match for a template (ignoring whitespace) + + jq -r .issue.title "$GITHUB_EVENT_PATH" > issue-title + if diff issue-title <(echo "$BASE_CASE_TITLE") > /dev/null; then + echo "Infinite recursion avoided." + exit 0 + fi + function extract-headings { { sed 's/\r$//' "$1" | grep '^#' || echo -n ''; } | sort; } jq -r .issue.body "$GITHUB_EVENT_PATH" > issue if ! grep -q '[^[:space:]]' issue; then @@ -69,7 +79,7 @@ jobs: # Failed validation! Close the issue with a comment. cat << EOF > comment - Sorry, friend. As far as this ol' bot can tell, ${REASON}. Please [try again](https://github.com/${{ github.repository }}/issues/new/choose), if you like. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=template+enforcer+is+confused&body=${{ github.event.issue.html_url }}). 😬) + Sorry, friend. As far as this ol' bot can tell, ${REASON}. Please [try again](https://github.com/${{ github.repository }}/issues/new/choose), if you like. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=$(echo "$BASE_CASE_TITLE" | tr ' ' '+')&body=${{ github.event.issue.html_url }}). 😬) ---- From 7365a034575b3beeb6fc70103f13916b52b7f29a Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 24 Feb 2021 16:44:50 +0100 Subject: [PATCH 290/417] fix(relay): Remove http._client usage (#870) As per https://github.com/getsentry/relay/pull/938 this option no longer exists. Existing values will be ignored, however. --- relay/config.example.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/relay/config.example.yml b/relay/config.example.yml index 0d70d0a2af..8538bd7d46 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -11,5 +11,3 @@ processing: - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 geoip_path: "/geoip/GeoLite2-City.mmdb" -http: - _client: "reqwest" From ff01285059f53e23b1a6b9e776448d0559c3a807 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 9 Mar 2021 15:03:25 -0500 Subject: [PATCH 291/417] meta(gha): Deploy action validate-new-issue.yml (#883) --- .github/workflows/validate-new-issue.yml | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index 3badef86ac..b17f2d9ba0 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -23,6 +23,13 @@ jobs: echo "${{ github.actor }} is not a member of the getsentry org. 🧐" fi + # Helper + function gh-issue-label() { + gh api "/repos/:owner/:repo/issues/${1}/labels" \ + -X POST \ + --input <(echo "{\"labels\":[\"$2\"]}") + } + # Prep reasons for error message comment. REASON="your issue does not properly use one of this repo's available issue templates" REASON_EXACT_MATCH="you created an issue from a template without filling in anything" @@ -66,9 +73,7 @@ jobs: REASON="${REASON_EXACT_MATCH}" break else - gh api "/repos/:owner/:repo/issues/${issue_number}/labels" \ - -X POST \ - --input <(echo '{"labels":["Status: Unrouted"]}') + gh-issue-label "${issue_number}" "Status: Unrouted" exit 0 fi else @@ -77,7 +82,7 @@ jobs: done fi - # Failed validation! Close the issue with a comment. + # Failed validation! Close the issue with a comment and a label. cat << EOF > comment Sorry, friend. As far as this ol' bot can tell, ${REASON}. Please [try again](https://github.com/${{ github.repository }}/issues/new/choose), if you like. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=$(echo "$BASE_CASE_TITLE" | tr ' ' '+')&body=${{ github.event.issue.html_url }}). 😬) @@ -89,6 +94,7 @@ jobs: EOF echo -n "Commented: " - gh issue comment ${{ github.event.issue.number }} --body "$(cat comment)" - gh issue close ${{ github.event.issue.number }} + gh issue comment "${issue_number}" --body "$(cat comment)" + gh-issue-label "${issue_number}" "Status: Invalid" + gh issue close "${issue_number}" echo "Closed with: \"${REASON}.\"" From ae0251d1a8bfcd67378de7dd4ce1a0deefe62041 Mon Sep 17 00:00:00 2001 From: ktmitton Date: Fri, 12 Mar 2021 08:09:20 -0500 Subject: [PATCH 292/417] Fixed CPU soft limit check (#885) --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index fae93a41f4..b0f8d1383d 100755 --- a/install.sh +++ b/install.sh @@ -138,7 +138,7 @@ fi if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" exit 1 -elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then +elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER" fi From 7138703a92975b47092e6f550ce838d5de6ef54c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 15 Mar 2021 21:12:51 +0300 Subject: [PATCH 293/417] fix(config): Remove deprecated `slack.legacy-app` config from defaults (#886) --- sentry/config.example.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 9595a7354f..298929958f 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -100,7 +100,5 @@ transaction-events.force-disable-internal-project: true # slack.client-id: <'client id'> # slack.client-secret: # slack.signing-secret: -## If you made your slack bot before july 2020 set legacy-app to True -slack.legacy-app: False ## If legacy-app is True use verfication-token instead of signing-secret # slack.verification-token: From 48c855aa3def4557ef799d878c75832662b5c67d Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Mon, 15 Mar 2021 18:25:12 +0000 Subject: [PATCH 294/417] release: 21.3.0 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..6f39e7381b 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.3.0 +SNUBA_IMAGE=getsentry/snuba:21.3.0 +RELAY_IMAGE=getsentry/relay:21.3.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index e31117941e..6f6d9275e0 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-02-15 +Change Date: 2024-03-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 62c9b46a28..6826146e79 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.3.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From f77e35d193873320068bed42b30bdc8315565a99 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Mar 2021 00:49:02 +0300 Subject: [PATCH 295/417] build: Set master version to nightly --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 6f39e7381b..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.3.0 -SNUBA_IMAGE=getsentry/snuba:21.3.0 -RELAY_IMAGE=getsentry/relay:21.3.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/LICENSE b/LICENSE index 6f6d9275e0..9a133dcd4e 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-03-15 +Change Date: 2024-03-16 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 6826146e79..62c9b46a28 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.3.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 4a8ca839562b8b58f79e0938dd179d71f97ab1be Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 16 Mar 2021 18:33:30 +0300 Subject: [PATCH 296/417] fix(release): Fix post-release script so it operates on master (#887) We've been leaving onpremise master with the latest release, instead of nightly builds for 2 releases now. Even if the post-release script runs, it bumped the versions to nightly on the release branch, making it effectively a no-op. This should be addressed in Craft via getsentry/craft#115 but until then, we need this extra line. --- scripts/post-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/post-release.sh b/scripts/post-release.sh index 652a792720..a05afe6230 100755 --- a/scripts/post-release.sh +++ b/scripts/post-release.sh @@ -5,6 +5,6 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" cd $SCRIPT_DIR/.. # Bring master back to nightlies after merge from release branch - +git checkout master && git pull SYMBOLICATOR_VERSION=nightly ./scripts/bump-version.sh '' 'nightly' git diff --quiet || git commit -anm 'build: Set master version to nightly' && git pull --rebase && git push From 8a742bc709d4545a2a0575cdae9b91a39aa63e68 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 18 Mar 2021 21:26:34 +0300 Subject: [PATCH 297/417] ci(stalebot): Add new triage labels (#890) This PR adds the 2 new triage labels to replace old ones: 1. `Status: Accepted` -> `Status: In Progress` 2. `Status: On Hold` -> `Status: Backlog` --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index b2a339a68e..dca4aec69b 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -18,7 +18,7 @@ jobs: debug-only: false ascending: false - exempt-issue-labels: "Status: Accepted,Status: On Hold" + exempt-issue-labels: "Status: Accepted,Status: On Hold,Status: Backlog,Status: In Progress" stale-issue-label: "Status: Stale" stale-issue-message: |- This issue has gone three weeks without activity. In another week, I will close it. @@ -32,7 +32,7 @@ jobs: close-issue-label: "" close-issue-message: "" - exempt-pr-labels: "Status: Accepted,Status: On Hold" + exempt-pr-labels: "Status: Accepted,Status: On Hold,Status: Backlog,Status: In Progress" stale-pr-label: "Status: Stale" stale-pr-message: |- This pull request has gone three weeks without activity. In another week, I will close it. From 2ac7e7358f5932af6f2628b52516dd6cb1cf137d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 18 Mar 2021 22:26:39 +0300 Subject: [PATCH 298/417] ci(stalebot): Remove old triage labels (#891) This PR removes the 2 old triage labels replaced with new ones: 1. `Status: Accepted` -> `Status: In Progress` 2. `Status: On Hold` -> `Status: Backlog` --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index dca4aec69b..4638274139 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -18,7 +18,7 @@ jobs: debug-only: false ascending: false - exempt-issue-labels: "Status: Accepted,Status: On Hold,Status: Backlog,Status: In Progress" + exempt-issue-labels: "Status: Backlog,Status: In Progress" stale-issue-label: "Status: Stale" stale-issue-message: |- This issue has gone three weeks without activity. In another week, I will close it. @@ -32,7 +32,7 @@ jobs: close-issue-label: "" close-issue-message: "" - exempt-pr-labels: "Status: Accepted,Status: On Hold,Status: Backlog,Status: In Progress" + exempt-pr-labels: "Status: Backlog,Status: In Progress" stale-pr-label: "Status: Stale" stale-pr-message: |- This pull request has gone three weeks without activity. In another week, I will close it. From 83ef869195fa867a45ded5902322fb908604d875 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 18 Mar 2021 23:09:57 +0300 Subject: [PATCH 299/417] ci(stalebot): Update stale message with the new labels (#892) * ci(stalebot): Update stale message with the new labels * fix everywhere * fix yaml --- .github/workflows/stale.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 4638274139..0a47156580 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -23,7 +23,7 @@ jobs: stale-issue-message: |- This issue has gone three weeks without activity. In another week, I will close it. - But! If you comment or otherwise update it, I will reset the clock, and if you label it `Status: Accepted`, I will leave it alone ... forever! + But! If you comment or otherwise update it, I will reset the clock, and if you label it `Status: Backlog` or `Status: In Progress`, I will leave it alone ... forever! ---- @@ -37,7 +37,7 @@ jobs: stale-pr-message: |- This pull request has gone three weeks without activity. In another week, I will close it. - But! If you comment or otherwise update it, I will reset the clock, and if you label it `Status: Accepted`, I will leave it alone ... forever! + But! If you comment or otherwise update it, I will reset the clock, and if you label it `Status: Backlog` or `Status: In Progress`, I will leave it alone ... forever! ---- From 8e498b42f4edeeb1edd25cc9285fce7e9c4c7cf8 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 19 Mar 2021 14:36:17 +0300 Subject: [PATCH 300/417] meta(gha): Deploy action validate-new-issue.yml (#893) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit I are a bot, here to deploy [validate-new-issue.yml](https://github.com/getsentry/.github/blob/6b97fda4e0d15db639960061da72bd8566692a81/.github/workflows/validate-new-issue.yml). 🤖 --- .github/workflows/validate-new-issue.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml index b17f2d9ba0..328e2e236c 100644 --- a/.github/workflows/validate-new-issue.yml +++ b/.github/workflows/validate-new-issue.yml @@ -41,7 +41,7 @@ jobs: # - is a report about buggy validation 😅 or ... # - not empty (ignoring whitespace) # - matches a template - # - all the headings are also in this issue + # - at least one of the headings are also in this issue # - extra headings in the issue are fine # - order doesn't matter # - case-sensitive tho @@ -66,7 +66,7 @@ jobs: echo -n "$(basename $template)? " if [ ! -s headings-in-template ]; then echo "No headers in template. 🤷" - elif [ -z "$(comm -23 headings-in-template headings-in-issue)" ]; then + elif [ "$(comm -12 headings-in-template headings-in-issue)" ]; then echo "Match! 👍 💃" if diff -Bw "$template" issue > /dev/null; then echo "... like, an /exact/ match. 😖" From bdd2686021cfea07507bc07d2756ac34a775c680 Mon Sep 17 00:00:00 2001 From: Hugo Barrera Date: Tue, 23 Mar 2021 09:23:16 +0000 Subject: [PATCH 301/417] fix: Typo in README (#897) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 62c9b46a28..1d618be3a2 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,7 @@ Sentry comes with a cleanup cron job that prunes events older than `90 days` by If you'd like to protect your Sentry install with SSL/TLS, there are fantastic SSL/TLS proxies like [HAProxy](http://www.haproxy.org/) -and [Nginx](http://nginx.org/). Our recommendation is running and external Nginx instance or your choice of load balancer that does the TLS termination and more. Read more over at our [productionalizing self-hosted docs](https://develop.sentry.dev/self-hosted/#productionalizing). +and [Nginx](http://nginx.org/). Our recommendation is running an external Nginx instance or your choice of load balancer that does the TLS termination and more. Read more over at our [productionalizing self-hosted docs](https://develop.sentry.dev/self-hosted/#productionalizing). ## Updating Sentry From 0ce7856117557590e0caa0bc276c643d0fbf878a Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 26 Mar 2021 10:06:04 -0400 Subject: [PATCH 302/417] Refactor relay config (#900) --- .github/workflows/test.yml | 8 ++++++- install.sh | 39 ++++--------------------------- install/_lib.sh | 36 ++++++++++++++++++++++++++++ install/_test_setup.sh | 39 +++++++++++++++++++++++++++++++ install/docker-aliases.sh | 3 --- install/geoip.sh | 5 +--- install/relay-credentials-test.sh | 24 +++++++++++++++++++ install/relay-credentials.sh | 24 +++++++++++++++++++ 8 files changed, 135 insertions(+), 43 deletions(-) create mode 100644 install/_lib.sh create mode 100644 install/_test_setup.sh delete mode 100755 install/docker-aliases.sh create mode 100755 install/relay-credentials-test.sh create mode 100755 install/relay-credentials.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fb891aea8b..9020edd79e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -28,7 +28,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - - name: Install and test + - name: Integration Test run: | echo "Testing initial install" ./install.sh @@ -39,6 +39,12 @@ jobs: ./install.sh --minimize-downtime ./test.sh + - name: Unit Tests + run: | + cd install + ./relay-credentials-test.sh + cd .. + - name: Inspect failure if: failure() run: | diff --git a/install.sh b/install.sh index b0f8d1383d..c0323275ec 100755 --- a/install.sh +++ b/install.sh @@ -9,20 +9,10 @@ fi # Thanks to https://unix.stackexchange.com/a/145654/108960 log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") -if [ "$GITHUB_ACTIONS" = "true" ]; then - _group="::group::" - _endgroup="::endgroup::" -else - _group="▶ " - _endgroup="" -fi - -echo "${_group}Defining variables and helpers ..." -# Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 -t=$(mktemp) && export -p > "$t" && set -a && . ./.env && set +a && . "$t" && rm "$t" && unset t -source ./install/docker-aliases.sh +source ./install/_lib.sh +echo "${_group}Defining variables and helpers ..." MIN_DOCKER_VERSION='19.03.6' MIN_COMPOSE_VERSION='1.24.1' MIN_RAM_HARD=3800 # MB @@ -38,8 +28,6 @@ STOP_TIMEOUT=60 # seconds SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' -RELAY_CONFIG_YML='relay/config.yml' -RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' MINIMIZE_DOWNTIME= echo $_endgroup @@ -115,16 +103,6 @@ CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); # Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } -# Thanks to https://stackoverflow.com/a/25123013/90297 for the quick `sed` pattern -function ensure_file_from_example { - if [[ -f "$1" ]]; then - echo "$1 already exists, skipped creation." - else - echo "Creating $1..." - cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" - fi -} - if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" exit 1 @@ -176,7 +154,6 @@ ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS ensure_file_from_example $SYMBOLICATOR_CONFIG_YML -ensure_file_from_example $RELAY_CONFIG_YML echo "${_endgroup}" echo "${_group}Generating secret key ..." @@ -344,16 +321,8 @@ fi echo "${_endgroup}" echo "${_group}Generating Relay credentials ..." -if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then - - # We need the ugly hack below as `relay generate credentials` tries to read the config and the credentials - # even with the `--stdout` and `--overwrite` flags and then errors out when the credentials file exists but - # not valid JSON. We hit this case as we redirect output to the same config folder, creating an empty - # credentials file before relay runs. - $dcr --no-deps -v $(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml relay --config /tmp credentials generate --stdout > "$RELAY_CREDENTIALS_JSON" - echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" - echo "${_endgroup}" -fi +source ./install/relay-credentials.sh +echo "${_endgroup}" echo "${_group}Setting up GeoIP integration ..." source ./install/geoip.sh diff --git a/install/_lib.sh b/install/_lib.sh new file mode 100644 index 0000000000..ba3dc68887 --- /dev/null +++ b/install/_lib.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +set -euo pipefail +if [[ ! -d 'install' ]]; then echo 'Where are you?'; exit 1; fi +_ENV="$(realpath ./.env)" + +define_stuff() { + # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 + t=$(mktemp) && export -p > "$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t + + if [ "${GITHUB_ACTIONS:-''}" = "true" ]; then + _group="::group::" + _endgroup="::endgroup::" + else + _group="▶ " + _endgroup="" + fi + + dc="docker-compose --no-ansi" + dcr="$dc run --rm" + + function ensure_file_from_example { + if [[ -f "$1" ]]; then + echo "$1 already exists, skipped creation." + else + echo "Creating $1..." + cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" + # sed from https://stackoverflow.com/a/25123013/90297 + fi + } + + stuff_defined="yes" +} + +if [ "${stuff_defined:-''}" != "" ]; then + define_stuff +fi diff --git a/install/_test_setup.sh b/install/_test_setup.sh new file mode 100644 index 0000000000..b0e5963781 --- /dev/null +++ b/install/_test_setup.sh @@ -0,0 +1,39 @@ +set -euo pipefail +test ${DEBUG:-''} && set -x +cd "$(dirname $0)/.." + +rm -rf /tmp/sentry-onpremise-test-sandbox.* +_SANDBOX="$(mktemp -d /tmp/sentry-onpremise-test-sandbox.XXX)" + +teardown() { + test ${DEBUG:-''} || rm -rf "$_SANDBOX" +} + +setup() { + # Clone the local repo into a temp dir, and propagate local changes. + + git clone --depth=1 "file://$(pwd)" "$_SANDBOX" + + git status --porcelain | while read line; do + local operation="$(cut -f1 -d' ' <(echo $line))" + local filepath="$(cut -f2 -d' ' <(echo $line))" + case $operation in + D) + rm "$_SANDBOX/$filepath" + ;; + A | M | AM) + ln -sf "$(realpath $filepath)" "$_SANDBOX/$filepath" + ;; + **) + echo "Wuh? $line" + exit 77 + ;; + esac + done + + cd "$_SANDBOX" + + trap teardown EXIT +} + +setup diff --git a/install/docker-aliases.sh b/install/docker-aliases.sh deleted file mode 100755 index e19384b6b9..0000000000 --- a/install/docker-aliases.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -dc="docker-compose --no-ansi" -dcr="$dc run --rm" diff --git a/install/geoip.sh b/install/geoip.sh index c90b56d7dd..1aecf02521 100755 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -1,8 +1,5 @@ #!/usr/bin/env bash - -if [[ ! -f 'install.sh' ]]; then echo 'Where are you?'; exit 1; fi - -source ./install/docker-aliases.sh +source ./install/_lib.sh install_geoip() { local mmdb='geoip/GeoLite2-City.mmdb' diff --git a/install/relay-credentials-test.sh b/install/relay-credentials-test.sh new file mode 100755 index 0000000000..a1f4b5febe --- /dev/null +++ b/install/relay-credentials-test.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_test_setup.sh" + +cfg="relay/config.yml" +creds="relay/credentials.json" + +# Relay files don't exist in a clean clone. +test ! -f $cfg +test ! -f $creds + +# Running the install script adds them. +./install/relay-credentials.sh +test -f $cfg +test -f $creds +test "$(jq -r 'keys[2]' $creds)" = "secret_key" + +# If the files exist we don't touch it. +echo GARBAGE > $cfg +echo MOAR GARBAGE > $creds +./install/relay-credentials.sh +test "$(cat $cfg)" = "GARBAGE" +test "$(cat $creds)" = "MOAR GARBAGE" + +echo "$(basename $0) - Success 👍" diff --git a/install/relay-credentials.sh b/install/relay-credentials.sh new file mode 100755 index 0000000000..667545cf07 --- /dev/null +++ b/install/relay-credentials.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +source ./install/_lib.sh + +RELAY_CONFIG_YML="relay/config.yml" +RELAY_CREDENTIALS_JSON="relay/credentials.json" + +ensure_file_from_example $RELAY_CONFIG_YML + +if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then + + # We need the ugly hack below as `relay generate credentials` tries to read + # the config and the credentials even with the `--stdout` and `--overwrite` + # flags and then errors out when the credentials file exists but not valid + # JSON. We hit this case as we redirect output to the same config folder, + # creating an empty credentials file before relay runs. + + $dcr \ + --no-deps \ + --volume "$(pwd)/$RELAY_CONFIG_YML:/tmp/config.yml" \ + relay --config /tmp credentials generate --stdout \ + > "$RELAY_CREDENTIALS_JSON" + + echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" +fi From a868b09044f4136bae40bfb5cfb8b6adffade4b5 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 26 Mar 2021 15:04:54 -0400 Subject: [PATCH 303/417] Factor out volume creation (#901) --- .github/workflows/test.yml | 4 ++-- install.sh | 19 +++---------------- install/_lib.sh | 10 +++++++++- install/_test_setup.sh | 20 +++++++++++++++++--- install/create-docker-volumes-test.sh | 20 ++++++++++++++++++++ install/create-docker-volumes.sh | 13 +++++++++++++ install/geoip.sh | 5 ++++- install/relay-credentials-test.sh | 2 +- install/relay-credentials.sh | 5 ++++- 9 files changed, 73 insertions(+), 25 deletions(-) create mode 100755 install/create-docker-volumes-test.sh create mode 100755 install/create-docker-volumes.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9020edd79e..109a3495c6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -40,10 +40,10 @@ jobs: ./test.sh - name: Unit Tests + working-directory: install run: | - cd install + ./create-docker-volumes-test.sh ./relay-credentials-test.sh - cd .. - name: Inspect failure if: failure() diff --git a/install.sh b/install.sh index c0323275ec..2ac593d9ef 100755 --- a/install.sh +++ b/install.sh @@ -139,15 +139,7 @@ if [[ "$IS_KVM" -eq 0 ]]; then fi echo "${_endgroup}" -echo "${_group}Creating volumes for persistent storage ..." -echo "Created $(docker volume create --name=sentry-data)." -echo "Created $(docker volume create --name=sentry-postgres)." -echo "Created $(docker volume create --name=sentry-redis)." -echo "Created $(docker volume create --name=sentry-zookeeper)." -echo "Created $(docker volume create --name=sentry-kafka)." -echo "Created $(docker volume create --name=sentry-clickhouse)." -echo "Created $(docker volume create --name=sentry-symbolicator)." -echo "${_endgroup}" +./install/create-docker-volumes.sh echo "${_group}Ensuring files from examples ..." ensure_file_from_example $SENTRY_CONFIG_PY @@ -320,13 +312,8 @@ if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then fi echo "${_endgroup}" -echo "${_group}Generating Relay credentials ..." -source ./install/relay-credentials.sh -echo "${_endgroup}" - -echo "${_group}Setting up GeoIP integration ..." -source ./install/geoip.sh -echo "${_endgroup}" +./install/relay-credentials.sh +./install/geoip.sh if [[ "$MINIMIZE_DOWNTIME" ]]; then echo "${_group}Waiting for Sentry to start ..." diff --git a/install/_lib.sh b/install/_lib.sh index ba3dc68887..a32323f6c6 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -1,7 +1,15 @@ #!/usr/bin/env bash set -euo pipefail + +# Work from the onpremise root, no matter which script is called from where. +if [[ "$(basename $0)" = "install.sh" ]]; then + cd "$(dirname $0)" +else + cd "$(dirname $0)/.." +fi if [[ ! -d 'install' ]]; then echo 'Where are you?'; exit 1; fi -_ENV="$(realpath ./.env)" + +_ENV="$(realpath .env)" define_stuff() { # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 diff --git a/install/_test_setup.sh b/install/_test_setup.sh index b0e5963781..8d6c3a8e59 100644 --- a/install/_test_setup.sh +++ b/install/_test_setup.sh @@ -5,6 +5,10 @@ cd "$(dirname $0)/.." rm -rf /tmp/sentry-onpremise-test-sandbox.* _SANDBOX="$(mktemp -d /tmp/sentry-onpremise-test-sandbox.XXX)" +report_success() { + echo "$(basename $0) - Success 👍" +} + teardown() { test ${DEBUG:-''} || rm -rf "$_SANDBOX" } @@ -12,16 +16,26 @@ teardown() { setup() { # Clone the local repo into a temp dir, and propagate local changes. + # FWIW `git clone --local` breaks for me because it depends on hard-linking, + # which doesn't work across devices, and I happen to have my workspace and + # tmp on separate devices. git clone --depth=1 "file://$(pwd)" "$_SANDBOX" git status --porcelain | while read line; do - local operation="$(cut -f1 -d' ' <(echo $line))" + + # $line here is something like `M some-script.sh`. By propagating working + # copy changes to the sandbox, we can provide a pretty nice dev experience: + # edit the files in the working copy, then run `DEBUG=1 some-test.sh` to + # leave the sandbox up for interactive dev/debugging. + local filepath="$(cut -f2 -d' ' <(echo $line))" - case $operation in + local filestatus="$(cut -f1 -d' ' <(echo $line))" + + case $filestatus in D) rm "$_SANDBOX/$filepath" ;; - A | M | AM) + A | M | AM | ??) ln -sf "$(realpath $filepath)" "$_SANDBOX/$filepath" ;; **) diff --git a/install/create-docker-volumes-test.sh b/install/create-docker-volumes-test.sh new file mode 100755 index 0000000000..0c35197bbd --- /dev/null +++ b/install/create-docker-volumes-test.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_test_setup.sh" + +expected=7 +count() { + docker volume ls --quiet | grep '^sentry-.*' | wc -l +} + +# Maybe they exist prior, maybe they don't. Script is idempotent. + +before=$(count) +test $before -eq 0 || test $before -eq $expected + +./install/create-docker-volumes.sh +./install/create-docker-volumes.sh +./install/create-docker-volumes.sh + +test $(count) -eq $expected + +report_success diff --git a/install/create-docker-volumes.sh b/install/create-docker-volumes.sh new file mode 100755 index 0000000000..5a844a128e --- /dev/null +++ b/install/create-docker-volumes.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_lib.sh" +echo "${_group}Creating volumes for persistent storage ..." + +echo "Created $(docker volume create --name=sentry-clickhouse)." +echo "Created $(docker volume create --name=sentry-data)." +echo "Created $(docker volume create --name=sentry-kafka)." +echo "Created $(docker volume create --name=sentry-postgres)." +echo "Created $(docker volume create --name=sentry-redis)." +echo "Created $(docker volume create --name=sentry-symbolicator)." +echo "Created $(docker volume create --name=sentry-zookeeper)." + +echo "${_endgroup}" diff --git a/install/geoip.sh b/install/geoip.sh index 1aecf02521..87c0d6a84f 100755 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -source ./install/_lib.sh +source "$(dirname $0)/_lib.sh" +echo "${_group}Setting up GeoIP integration ..." install_geoip() { local mmdb='geoip/GeoLite2-City.mmdb' @@ -31,3 +32,5 @@ install_geoip() { } install_geoip + +echo "${_endgroup}" diff --git a/install/relay-credentials-test.sh b/install/relay-credentials-test.sh index a1f4b5febe..2fc71575b3 100755 --- a/install/relay-credentials-test.sh +++ b/install/relay-credentials-test.sh @@ -21,4 +21,4 @@ echo MOAR GARBAGE > $creds test "$(cat $cfg)" = "GARBAGE" test "$(cat $creds)" = "MOAR GARBAGE" -echo "$(basename $0) - Success 👍" +report_success diff --git a/install/relay-credentials.sh b/install/relay-credentials.sh index 667545cf07..cd3630fbbf 100755 --- a/install/relay-credentials.sh +++ b/install/relay-credentials.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -source ./install/_lib.sh +source "$(dirname $0)/_lib.sh" +echo "${_group}Generating Relay credentials ..." RELAY_CONFIG_YML="relay/config.yml" RELAY_CREDENTIALS_JSON="relay/credentials.json" @@ -22,3 +23,5 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi + +echo "${_endgroup}" From 88f90e046ecca7c0006302c6e9b1174dff3b6abf Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 26 Mar 2021 16:57:21 -0400 Subject: [PATCH 304/417] Add a partial test for geoip (#902) --- .github/workflows/test.yml | 1 + install/_test_setup.sh | 18 ++++++++---------- install/geoip-test.sh | 16 ++++++++++++++++ 3 files changed, 25 insertions(+), 10 deletions(-) create mode 100755 install/geoip-test.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 109a3495c6..378accc41e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,6 +44,7 @@ jobs: run: | ./create-docker-volumes-test.sh ./relay-credentials-test.sh + ./geoip-test.sh - name: Inspect failure if: failure() diff --git a/install/_test_setup.sh b/install/_test_setup.sh index 8d6c3a8e59..a40a562565 100644 --- a/install/_test_setup.sh +++ b/install/_test_setup.sh @@ -14,19 +14,17 @@ teardown() { } setup() { - # Clone the local repo into a temp dir, and propagate local changes. - - # FWIW `git clone --local` breaks for me because it depends on hard-linking, - # which doesn't work across devices, and I happen to have my workspace and - # tmp on separate devices. + # Clone the local repo into a temp dir. FWIW `git clone --local` breaks for + # me because it depends on hard-linking, which doesn't work across devices, + # and I happen to have my workspace and /tmp on separate devices. git clone --depth=1 "file://$(pwd)" "$_SANDBOX" + # Now propagate any local changes from the working copy to the sandbox. This + # provides a pretty nice dev experience: edit the files in the working copy, + # then run `DEBUG=1 some-test.sh` to leave the sandbox up for interactive + # dev/debugging. git status --porcelain | while read line; do - - # $line here is something like `M some-script.sh`. By propagating working - # copy changes to the sandbox, we can provide a pretty nice dev experience: - # edit the files in the working copy, then run `DEBUG=1 some-test.sh` to - # leave the sandbox up for interactive dev/debugging. + # $line here is something like `M some-script.sh`. local filepath="$(cut -f2 -d' ' <(echo $line))" local filestatus="$(cut -f1 -d' ' <(echo $line))" diff --git a/install/geoip-test.sh b/install/geoip-test.sh new file mode 100755 index 0000000000..79f0c35ec5 --- /dev/null +++ b/install/geoip-test.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_test_setup.sh" + +mmdb="geoip/GeoLite2-City.mmdb" + +# Starts with no mmdb, ends up with empty. +test ! -f $mmdb +./install/geoip.sh +diff -rub $mmdb $mmdb.empty + +# Doesn't clobber existing, though. +echo GARBAGE > $mmdb +./install/geoip.sh +test "$(cat $mmdb)" = "GARBAGE" + +report_success From cf4f21c0397b4aca84d586aebfbff590107f33a1 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 29 Mar 2021 10:41:20 -0400 Subject: [PATCH 305/417] Fix a regression with unset envvars (#905) --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 2ac593d9ef..fa604beb5e 100755 --- a/install.sh +++ b/install.sh @@ -289,7 +289,7 @@ fi echo "${_endgroup}" echo "${_group}Setting up database ..." -if [[ -n "$CI" || "$SKIP_USER_PROMPT" == 1 ]]; then +if [[ -n "${CI:-''}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." From 568f9052b59ff0299beff4d3ba422660a9a34f88 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 29 Mar 2021 17:28:39 -0400 Subject: [PATCH 306/417] Factor out a few more things (#906) --- install.sh | 62 ++------------------------ install/migrate-file-storage.sh | 12 +++++ install/restart-carefully.sh | 14 ++++++ install/set-up-and-migrate-database.sh | 16 +++++++ install/upgrade-postgres.sh | 25 +++++++++++ 5 files changed, 71 insertions(+), 58 deletions(-) create mode 100755 install/migrate-file-storage.sh create mode 100755 install/restart-carefully.sh create mode 100755 install/set-up-and-migrate-database.sh create mode 100755 install/upgrade-postgres.sh diff --git a/install.sh b/install.sh index fa604beb5e..4aecf5d88a 100755 --- a/install.sh +++ b/install.sh @@ -265,68 +265,14 @@ for topic in $NEEDED_KAFKA_TOPICS; do done echo "${_endgroup}" -echo "${_group}Ensuring proper PostgreSQL version ..." -# Very naively check whether there's an existing sentry-postgres volume and the PG version in it -if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then - docker volume rm sentry-postgres-new || true - # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume - docker run --rm \ - -v sentry-postgres:/var/lib/postgresql/9.5/data \ - -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ - tianon/postgres-upgrade:9.5-to-9.6 - - # Get rid of the old volume as we'll rename the new one to that - docker volume rm sentry-postgres - docker volume create --name sentry-postgres - # There's no rename volume in Docker so copy the contents from old to new name - # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` - # doesn't do that automatically. - docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ - "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" - # Finally, remove the new old volume as we are all in sentry-postgres now - docker volume rm sentry-postgres-new -fi -echo "${_endgroup}" - -echo "${_group}Setting up database ..." -if [[ -n "${CI:-''}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then - $dcr web upgrade --noinput - echo "" - echo "Did not prompt for user creation due to non-interactive shell." - echo "Run the following command to create one yourself (recommended):" - echo "" - echo " docker-compose run --rm web createuser" - echo "" -else - $dcr web upgrade -fi -echo "${_endgroup}" - -echo "${_group}Migrating file storage ..." -SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") -if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then - # Use the web (Sentry) image so the file owners are kept as sentry:sentry - # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. - $dcr --entrypoint \"/bin/bash\" web -c \ - "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" -fi -echo "${_endgroup}" - +./install/upgrade-postgres.sh +./install/set-up-and-migrate-database.sh +./install/migrate-file-storage.sh ./install/relay-credentials.sh ./install/geoip.sh if [[ "$MINIMIZE_DOWNTIME" ]]; then - echo "${_group}Waiting for Sentry to start ..." - # Start the whole setup, except nginx and relay. - $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') - $dc exec -T nginx service nginx reload - - docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ - -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' - - # Make sure everything is up. This should only touch relay and nginx - $dc up -d - echo "${_endgroup}" + ./install/restart-carefully.sh else echo "" echo "-----------------------------------------------------------------" diff --git a/install/migrate-file-storage.sh b/install/migrate-file-storage.sh new file mode 100755 index 0000000000..9e1b71e787 --- /dev/null +++ b/install/migrate-file-storage.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_lib.sh" + +echo "${_group}Migrating file storage ..." +SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") +if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then + # Use the web (Sentry) image so the file owners are kept as sentry:sentry + # The `\"` escape pattern is to make this compatible w/ Git Bash on Windows. See #329. + $dcr --entrypoint \"/bin/bash\" web -c \ + "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" +fi +echo "${_endgroup}" diff --git a/install/restart-carefully.sh b/install/restart-carefully.sh new file mode 100755 index 0000000000..98fbc6c48b --- /dev/null +++ b/install/restart-carefully.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_lib.sh" + +echo "${_group}Waiting for Sentry to start ..." +# Start the whole setup, except nginx and relay. +$dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') +$dc exec -T nginx service nginx reload + +docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ + -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' + +# Make sure everything is up. This should only touch relay and nginx +$dc up -d +echo "${_endgroup}" diff --git a/install/set-up-and-migrate-database.sh b/install/set-up-and-migrate-database.sh new file mode 100755 index 0000000000..0fa0100995 --- /dev/null +++ b/install/set-up-and-migrate-database.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_lib.sh" + +echo "${_group}Setting up / migrating database ..." +if [[ -n "${CI:-''}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then + $dcr web upgrade --noinput + echo "" + echo "Did not prompt for user creation due to non-interactive shell." + echo "Run the following command to create one yourself (recommended):" + echo "" + echo " docker-compose run --rm web createuser" + echo "" +else + $dcr web upgrade +fi +echo "${_endgroup}" diff --git a/install/upgrade-postgres.sh b/install/upgrade-postgres.sh new file mode 100755 index 0000000000..43f511aee3 --- /dev/null +++ b/install/upgrade-postgres.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +source "$(dirname $0)/_lib.sh" + +echo "${_group}Ensuring proper PostgreSQL version ..." +# Very naively check whether there's an existing sentry-postgres volume and the PG version in it +if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then + docker volume rm sentry-postgres-new || true + # If this is Postgres 9.5 data, start upgrading it to 9.6 in a new volume + docker run --rm \ + -v sentry-postgres:/var/lib/postgresql/9.5/data \ + -v sentry-postgres-new:/var/lib/postgresql/9.6/data \ + tianon/postgres-upgrade:9.5-to-9.6 + + # Get rid of the old volume as we'll rename the new one to that + docker volume rm sentry-postgres + docker volume create --name sentry-postgres + # There's no rename volume in Docker so copy the contents from old to new name + # Also append the `host all all all trust` line as `tianon/postgres-upgrade:9.5-to-9.6` + # doesn't do that automatically. + docker run --rm -v sentry-postgres-new:/from -v sentry-postgres:/to alpine ash -c \ + "cd /from ; cp -av . /to ; echo 'host all all all trust' >> /to/pg_hba.conf" + # Finally, remove the new old volume as we are all in sentry-postgres now + docker volume rm sentry-postgres-new +fi +echo "${_endgroup}" From f08615754842b8bb86b38e83f34cdf76b8b9f0cf Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 30 Mar 2021 05:21:30 -0400 Subject: [PATCH 307/417] Clean up the refactor (#907) - Use source appropriately (needed for config to propagate properly) - Standardize group/endgroup line-spacing - Clean up envvar defaults --- install.sh | 16 ++++++++-------- install/_lib.sh | 4 ++-- install/_test_setup.sh | 5 ++--- install/create-docker-volumes-test.sh | 6 +++--- install/create-docker-volumes.sh | 2 -- install/geoip-test.sh | 4 ++-- install/geoip.sh | 2 -- install/migrate-file-storage.sh | 5 ++--- install/relay-credentials-test.sh | 4 ++-- install/relay-credentials.sh | 2 -- install/restart-carefully.sh | 5 ++--- install/set-up-and-migrate-database.sh | 7 +++---- install/upgrade-postgres.sh | 5 ++--- 13 files changed, 28 insertions(+), 39 deletions(-) mode change 100755 => 100644 install/create-docker-volumes.sh mode change 100755 => 100644 install/geoip.sh mode change 100755 => 100644 install/migrate-file-storage.sh mode change 100755 => 100644 install/relay-credentials.sh mode change 100755 => 100644 install/restart-carefully.sh mode change 100755 => 100644 install/set-up-and-migrate-database.sh mode change 100755 => 100644 install/upgrade-postgres.sh diff --git a/install.sh b/install.sh index 4aecf5d88a..331fd9a5cc 100755 --- a/install.sh +++ b/install.sh @@ -10,7 +10,7 @@ fi log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") -source ./install/_lib.sh +source "$(dirname $0)/install/_lib.sh" echo "${_group}Defining variables and helpers ..." MIN_DOCKER_VERSION='19.03.6' @@ -139,7 +139,7 @@ if [[ "$IS_KVM" -eq 0 ]]; then fi echo "${_endgroup}" -./install/create-docker-volumes.sh +source ./install/create-docker-volumes.sh echo "${_group}Ensuring files from examples ..." ensure_file_from_example $SENTRY_CONFIG_PY @@ -265,14 +265,14 @@ for topic in $NEEDED_KAFKA_TOPICS; do done echo "${_endgroup}" -./install/upgrade-postgres.sh -./install/set-up-and-migrate-database.sh -./install/migrate-file-storage.sh -./install/relay-credentials.sh -./install/geoip.sh +source ./install/upgrade-postgres.sh +source ./install/set-up-and-migrate-database.sh +source ./install/migrate-file-storage.sh +source ./install/relay-credentials.sh +source ./install/geoip.sh if [[ "$MINIMIZE_DOWNTIME" ]]; then - ./install/restart-carefully.sh + source ./install/restart-carefully.sh else echo "" echo "-----------------------------------------------------------------" diff --git a/install/_lib.sh b/install/_lib.sh index a32323f6c6..d0fad965f0 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -1,5 +1,5 @@ -#!/usr/bin/env bash set -euo pipefail +test "${DEBUG:-}" && set -x # Work from the onpremise root, no matter which script is called from where. if [[ "$(basename $0)" = "install.sh" ]]; then @@ -15,7 +15,7 @@ define_stuff() { # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t - if [ "${GITHUB_ACTIONS:-''}" = "true" ]; then + if [ "${GITHUB_ACTIONS:-}" = "true" ]; then _group="::group::" _endgroup="::endgroup::" else diff --git a/install/_test_setup.sh b/install/_test_setup.sh index a40a562565..3a4409912c 100644 --- a/install/_test_setup.sh +++ b/install/_test_setup.sh @@ -1,6 +1,5 @@ set -euo pipefail -test ${DEBUG:-''} && set -x -cd "$(dirname $0)/.." +source "$(dirname $0)/_lib.sh" rm -rf /tmp/sentry-onpremise-test-sandbox.* _SANDBOX="$(mktemp -d /tmp/sentry-onpremise-test-sandbox.XXX)" @@ -10,7 +9,7 @@ report_success() { } teardown() { - test ${DEBUG:-''} || rm -rf "$_SANDBOX" + test "${DEBUG:-}" || rm -rf "$_SANDBOX" } setup() { diff --git a/install/create-docker-volumes-test.sh b/install/create-docker-volumes-test.sh index 0c35197bbd..86f8b172ee 100755 --- a/install/create-docker-volumes-test.sh +++ b/install/create-docker-volumes-test.sh @@ -11,9 +11,9 @@ count() { before=$(count) test $before -eq 0 || test $before -eq $expected -./install/create-docker-volumes.sh -./install/create-docker-volumes.sh -./install/create-docker-volumes.sh +source ./install/create-docker-volumes.sh +source ./install/create-docker-volumes.sh +source ./install/create-docker-volumes.sh test $(count) -eq $expected diff --git a/install/create-docker-volumes.sh b/install/create-docker-volumes.sh old mode 100755 new mode 100644 index 5a844a128e..ca3ef0b23e --- a/install/create-docker-volumes.sh +++ b/install/create-docker-volumes.sh @@ -1,5 +1,3 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" echo "${_group}Creating volumes for persistent storage ..." echo "Created $(docker volume create --name=sentry-clickhouse)." diff --git a/install/geoip-test.sh b/install/geoip-test.sh index 79f0c35ec5..38650ab787 100755 --- a/install/geoip-test.sh +++ b/install/geoip-test.sh @@ -5,12 +5,12 @@ mmdb="geoip/GeoLite2-City.mmdb" # Starts with no mmdb, ends up with empty. test ! -f $mmdb -./install/geoip.sh +source ./install/geoip.sh diff -rub $mmdb $mmdb.empty # Doesn't clobber existing, though. echo GARBAGE > $mmdb -./install/geoip.sh +source ./install/geoip.sh test "$(cat $mmdb)" = "GARBAGE" report_success diff --git a/install/geoip.sh b/install/geoip.sh old mode 100755 new mode 100644 index 87c0d6a84f..9a1317be45 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -1,5 +1,3 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" echo "${_group}Setting up GeoIP integration ..." install_geoip() { diff --git a/install/migrate-file-storage.sh b/install/migrate-file-storage.sh old mode 100755 new mode 100644 index 9e1b71e787..8623faef51 --- a/install/migrate-file-storage.sh +++ b/install/migrate-file-storage.sh @@ -1,7 +1,5 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" - echo "${_group}Migrating file storage ..." + SENTRY_DATA_NEEDS_MIGRATION=$(docker run --rm -v sentry-data:/data alpine ash -c "[ ! -d '/data/files' ] && ls -A1x /data | wc -l || true") if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then # Use the web (Sentry) image so the file owners are kept as sentry:sentry @@ -9,4 +7,5 @@ if [[ -n "$SENTRY_DATA_NEEDS_MIGRATION" ]]; then $dcr --entrypoint \"/bin/bash\" web -c \ "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi + echo "${_endgroup}" diff --git a/install/relay-credentials-test.sh b/install/relay-credentials-test.sh index 2fc71575b3..8edced6bb4 100755 --- a/install/relay-credentials-test.sh +++ b/install/relay-credentials-test.sh @@ -9,7 +9,7 @@ test ! -f $cfg test ! -f $creds # Running the install script adds them. -./install/relay-credentials.sh +source ./install/relay-credentials.sh test -f $cfg test -f $creds test "$(jq -r 'keys[2]' $creds)" = "secret_key" @@ -17,7 +17,7 @@ test "$(jq -r 'keys[2]' $creds)" = "secret_key" # If the files exist we don't touch it. echo GARBAGE > $cfg echo MOAR GARBAGE > $creds -./install/relay-credentials.sh +source ./install/relay-credentials.sh test "$(cat $cfg)" = "GARBAGE" test "$(cat $creds)" = "MOAR GARBAGE" diff --git a/install/relay-credentials.sh b/install/relay-credentials.sh old mode 100755 new mode 100644 index cd3630fbbf..91fe34df72 --- a/install/relay-credentials.sh +++ b/install/relay-credentials.sh @@ -1,5 +1,3 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" echo "${_group}Generating Relay credentials ..." RELAY_CONFIG_YML="relay/config.yml" diff --git a/install/restart-carefully.sh b/install/restart-carefully.sh old mode 100755 new mode 100644 index 98fbc6c48b..a25da9e19f --- a/install/restart-carefully.sh +++ b/install/restart-carefully.sh @@ -1,7 +1,5 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" - echo "${_group}Waiting for Sentry to start ..." + # Start the whole setup, except nginx and relay. $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') $dc exec -T nginx service nginx reload @@ -11,4 +9,5 @@ docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ # Make sure everything is up. This should only touch relay and nginx $dc up -d + echo "${_endgroup}" diff --git a/install/set-up-and-migrate-database.sh b/install/set-up-and-migrate-database.sh old mode 100755 new mode 100644 index 0fa0100995..38d4093b36 --- a/install/set-up-and-migrate-database.sh +++ b/install/set-up-and-migrate-database.sh @@ -1,8 +1,6 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" - echo "${_group}Setting up / migrating database ..." -if [[ -n "${CI:-''}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then + +if [[ -n "${CI:-}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then $dcr web upgrade --noinput echo "" echo "Did not prompt for user creation due to non-interactive shell." @@ -13,4 +11,5 @@ if [[ -n "${CI:-''}" || "${SKIP_USER_PROMPT:-0}" == 1 ]]; then else $dcr web upgrade fi + echo "${_endgroup}" diff --git a/install/upgrade-postgres.sh b/install/upgrade-postgres.sh old mode 100755 new mode 100644 index 43f511aee3..1faccb82a1 --- a/install/upgrade-postgres.sh +++ b/install/upgrade-postgres.sh @@ -1,7 +1,5 @@ -#!/usr/bin/env bash -source "$(dirname $0)/_lib.sh" - echo "${_group}Ensuring proper PostgreSQL version ..." + # Very naively check whether there's an existing sentry-postgres volume and the PG version in it if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run --rm -v sentry-postgres:/db busybox cat /db/PG_VERSION 2>/dev/null)" == "9.5" ]]; then docker volume rm sentry-postgres-new || true @@ -22,4 +20,5 @@ if [[ -n "$(docker volume ls -q --filter name=sentry-postgres)" && "$(docker run # Finally, remove the new old volume as we are all in sentry-postgres now docker volume rm sentry-postgres-new fi + echo "${_endgroup}" From 7e7401a668987c44863c403548ba3dbf91c701da Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 30 Mar 2021 10:21:05 -0400 Subject: [PATCH 308/417] Refactor most of the rest (#903) --- install.sh | 148 +++----------------------- install/_lib.sh | 56 +++++----- install/bootstrap-snuba.sh | 6 ++ install/build-docker-images.sh | 8 ++ install/create-kafka-topics.sh | 14 +++ install/ensure-files-from-examples.sh | 8 ++ install/generate-secret-key.sh | 12 +++ install/replace-tsdb.sh | 46 ++++++++ install/set-up-zookeeper.sh | 14 +++ install/turn-things-off.sh | 14 +++ install/update-docker-images.sh | 13 +++ 11 files changed, 176 insertions(+), 163 deletions(-) create mode 100644 install/bootstrap-snuba.sh create mode 100644 install/build-docker-images.sh create mode 100644 install/create-kafka-topics.sh create mode 100644 install/ensure-files-from-examples.sh create mode 100644 install/generate-secret-key.sh create mode 100644 install/replace-tsdb.sh create mode 100644 install/set-up-zookeeper.sh create mode 100644 install/turn-things-off.sh create mode 100644 install/update-docker-images.sh diff --git a/install.sh b/install.sh index 331fd9a5cc..ba02bf64f7 100755 --- a/install.sh +++ b/install.sh @@ -19,17 +19,6 @@ MIN_RAM_HARD=3800 # MB MIN_RAM_SOFT=7800 # MB MIN_CPU_HARD=2 MIN_CPU_SOFT=4 - -# Increase the default 10 second SIGTERM timeout -# to ensure celery queues are properly drained -# between upgrades as task signatures may change across -# versions -STOP_TIMEOUT=60 # seconds -SENTRY_CONFIG_PY='sentry/sentry.conf.py' -SENTRY_CONFIG_YML='sentry/config.yml' -SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' -SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' -MINIMIZE_DOWNTIME= echo $_endgroup echo "${_group}Parsing command line ..." @@ -48,6 +37,9 @@ Options: EOF } +SKIP_USER_PROMPT="${SKIP_USER_PROMPT:-}" +MINIMIZE_DOWNTIME="${MINIMIZE_DOWNTIME:-}" + while (( $# )); do case "$1" in -h | --help) show_help; exit;; @@ -140,131 +132,15 @@ fi echo "${_endgroup}" source ./install/create-docker-volumes.sh - -echo "${_group}Ensuring files from examples ..." -ensure_file_from_example $SENTRY_CONFIG_PY -ensure_file_from_example $SENTRY_CONFIG_YML -ensure_file_from_example $SENTRY_EXTRA_REQUIREMENTS -ensure_file_from_example $SYMBOLICATOR_CONFIG_YML -echo "${_endgroup}" - -echo "${_group}Generating secret key ..." -if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then - # This is to escape the secret key to be used in sed below - # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode - # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 - SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') - sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML - echo "Secret key written to $SENTRY_CONFIG_YML" -fi -echo "${_endgroup}" - -echo "${_group}Replacing TSDB ..." -replace_tsdb() { - if ( - [[ -f "$SENTRY_CONFIG_PY" ]] && - ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" - ); then - # Do NOT indent the following string as it would be reflected in the end result, - # breaking the final config file. See getsentry/onpremise#624. - tsdb_settings="\ -SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" - -# Automatic switchover 90 days after $(date). Can be removed afterwards. -SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}\ -" - - if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then - echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" - else - echo "Attempting to automatically migrate to new TSDB" - # Escape newlines for sed - tsdb_settings="${tsdb_settings//$'\n'/\\n}" - cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" - sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true - - if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then - echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" - return - fi - - echo "Failed to automatically migrate TSDB. Reverting..." - mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" - echo "$SENTRY_CONFIG_PY restored from backup." - fi - - echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" - echo "" - echo "$tsdb_settings" - echo "" - echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" - fi -} - -replace_tsdb -echo "${_endgroup}" - -echo "${_group}Fetching and updating Docker images ..." -# We tag locally built images with an '-onpremise-local' suffix. docker-compose pull tries to pull these too and -# shows a 404 error on the console which is confusing and unnecessary. To overcome this, we add the stderr>stdout -# redirection below and pass it through grep, ignoring all lines having this '-onpremise-local' suffix. -$dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true - -# We may not have the set image on the repo (local images) so allow fails -docker pull ${SENTRY_IMAGE} || true; -echo "${_endgroup}" - -echo "${_group}Building and tagging Docker images ..." -echo "" -$dc build --force-rm -echo "" -echo "Docker images built." -echo "${_endgroup}" - -echo "${_group}Turning things off ..." -if [[ -n "$MINIMIZE_DOWNTIME" ]]; then - # Stop everything but relay and nginx - $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') -else - # Clean up old stuff and ensure nothing is working while we install/update - # This is for older versions of on-premise: - $dc -p onpremise down -t $STOP_TIMEOUT --rmi local --remove-orphans - # This is for newer versions - $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans -fi -echo "${_endgroup}" - -echo "${_group}Setting up Zookeeper ..." -ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') -if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then - ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') - ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') - # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 - if [[ "$ZOOKEEPER_LOG_FILE_COUNT" -gt 0 ]] && [[ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq 0 ]]; then - $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' - $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper - fi -fi -echo "${_endgroup}" - -echo "${_group}Bootstrapping and migrating Snuba ..." -$dcr snuba-api bootstrap --no-migrate --force -$dcr snuba-api migrations migrate --force -echo "${_endgroup}" - -echo "${_group}Creating additional Kafka topics ..." -# NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step -# XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) -EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) -NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events" -for topic in $NEEDED_KAFKA_TOPICS; do - if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then - $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 - echo "" - fi -done -echo "${_endgroup}" - +source ./install/ensure-files-from-examples.sh +source ./install/generate-secret-key.sh +source ./install/replace-tsdb.sh +source ./install/update-docker-images.sh +source ./install/build-docker-images.sh +source ./install/turn-things-off.sh +source ./install/set-up-zookeeper.sh +source ./install/bootstrap-snuba.sh +source ./install/create-kafka-topics.sh source ./install/upgrade-postgres.sh source ./install/set-up-and-migrate-database.sh source ./install/migrate-file-storage.sh diff --git a/install/_lib.sh b/install/_lib.sh index d0fad965f0..9c636fdff4 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -11,34 +11,36 @@ if [[ ! -d 'install' ]]; then echo 'Where are you?'; exit 1; fi _ENV="$(realpath .env)" -define_stuff() { - # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 - t=$(mktemp) && export -p > "$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t +# Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 +t=$(mktemp) && export -p > "$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t - if [ "${GITHUB_ACTIONS:-}" = "true" ]; then - _group="::group::" - _endgroup="::endgroup::" - else - _group="▶ " - _endgroup="" - fi - - dc="docker-compose --no-ansi" - dcr="$dc run --rm" +if [ "${GITHUB_ACTIONS:-}" = "true" ]; then + _group="::group::" + _endgroup="::endgroup::" +else + _group="▶ " + _endgroup="" +fi - function ensure_file_from_example { - if [[ -f "$1" ]]; then - echo "$1 already exists, skipped creation." - else - echo "Creating $1..." - cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" - # sed from https://stackoverflow.com/a/25123013/90297 - fi - } +dc="docker-compose --no-ansi" +dcr="$dc run --rm" - stuff_defined="yes" +# A couple of the config files are referenced from other subscripts, so they +# get vars, while multiple subscripts call ensure_file_from_example. +function ensure_file_from_example { + if [[ -f "$1" ]]; then + echo "$1 already exists, skipped creation." + else + echo "Creating $1..." + cp -n $(echo "$1" | sed 's/\.[^.]*$/.example&/') "$1" + # sed from https://stackoverflow.com/a/25123013/90297 + fi } - -if [ "${stuff_defined:-''}" != "" ]; then - define_stuff -fi +SENTRY_CONFIG_PY='sentry/sentry.conf.py' +SENTRY_CONFIG_YML='sentry/config.yml' + +# Increase the default 10 second SIGTERM timeout +# to ensure celery queues are properly drained +# between upgrades as task signatures may change across +# versions +STOP_TIMEOUT=60 # seconds diff --git a/install/bootstrap-snuba.sh b/install/bootstrap-snuba.sh new file mode 100644 index 0000000000..2952ed0b33 --- /dev/null +++ b/install/bootstrap-snuba.sh @@ -0,0 +1,6 @@ +echo "${_group}Bootstrapping and migrating Snuba ..." + +$dcr snuba-api bootstrap --no-migrate --force +$dcr snuba-api migrations migrate --force + +echo "${_endgroup}" diff --git a/install/build-docker-images.sh b/install/build-docker-images.sh new file mode 100644 index 0000000000..4bb96b5ea2 --- /dev/null +++ b/install/build-docker-images.sh @@ -0,0 +1,8 @@ +echo "${_group}Building and tagging Docker images ..." + +echo "" +$dc build --force-rm +echo "" +echo "Docker images built." + +echo "${_endgroup}" diff --git a/install/create-kafka-topics.sh b/install/create-kafka-topics.sh new file mode 100644 index 0000000000..a542cb54d3 --- /dev/null +++ b/install/create-kafka-topics.sh @@ -0,0 +1,14 @@ +echo "${_group}Creating additional Kafka topics ..." + +# NOTE: This step relies on `kafka` being available from the previous `snuba-api bootstrap` step +# XXX(BYK): We cannot use auto.create.topics as Confluence and Apache hates it now (and makes it very hard to enable) +EXISTING_KAFKA_TOPICS=$($dcr kafka kafka-topics --list --bootstrap-server kafka:9092 2>/dev/null) +NEEDED_KAFKA_TOPICS="ingest-attachments ingest-transactions ingest-events" +for topic in $NEEDED_KAFKA_TOPICS; do + if ! echo "$EXISTING_KAFKA_TOPICS" | grep -wq $topic; then + $dcr kafka kafka-topics --create --topic $topic --bootstrap-server kafka:9092 + echo "" + fi +done + +echo "${_endgroup}" diff --git a/install/ensure-files-from-examples.sh b/install/ensure-files-from-examples.sh new file mode 100644 index 0000000000..0a507d634f --- /dev/null +++ b/install/ensure-files-from-examples.sh @@ -0,0 +1,8 @@ +echo "${_group}Ensuring files from examples ..." + +ensure_file_from_example $SENTRY_CONFIG_PY +ensure_file_from_example $SENTRY_CONFIG_YML +ensure_file_from_example 'symbolicator/config.yml' +ensure_file_from_example 'sentry/requirements.txt' + +echo "${_endgroup}" diff --git a/install/generate-secret-key.sh b/install/generate-secret-key.sh new file mode 100644 index 0000000000..de2afbaafd --- /dev/null +++ b/install/generate-secret-key.sh @@ -0,0 +1,12 @@ +echo "${_group}Generating secret key ..." + +if grep -xq "system.secret-key: '!!changeme!!'" $SENTRY_CONFIG_YML ; then + # This is to escape the secret key to be used in sed below + # Note the need to set LC_ALL=C due to BSD tr and sed always trying to decode + # whatever is passed to them. Kudos to https://stackoverflow.com/a/23584470/90297 + SECRET_KEY=$(export LC_ALL=C; head /dev/urandom | tr -dc "a-z0-9@#%^&*(-_=+)" | head -c 50 | sed -e 's/[\/&]/\\&/g') + sed -i -e 's/^system.secret-key:.*$/system.secret-key: '"'$SECRET_KEY'"'/' $SENTRY_CONFIG_YML + echo "Secret key written to $SENTRY_CONFIG_YML" +fi + +echo "${_endgroup}" diff --git a/install/replace-tsdb.sh b/install/replace-tsdb.sh new file mode 100644 index 0000000000..0716bc148b --- /dev/null +++ b/install/replace-tsdb.sh @@ -0,0 +1,46 @@ +echo "${_group}Replacing TSDB ..." + +replace_tsdb() { + if ( + [[ -f "$SENTRY_CONFIG_PY" ]] && + ! grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY" + ); then + # Do NOT indent the following string as it would be reflected in the end result, + # breaking the final config file. See getsentry/onpremise#624. + tsdb_settings="\ +SENTRY_TSDB = \"sentry.tsdb.redissnuba.RedisSnubaTSDB\" + +# Automatic switchover 90 days after $(date). Can be removed afterwards. +SENTRY_TSDB_OPTIONS = {\"switchover_timestamp\": $(date +%s) + (90 * 24 * 3600)}\ +" + + if grep -q 'SENTRY_TSDB_OPTIONS = ' "$SENTRY_CONFIG_PY"; then + echo "Not attempting automatic TSDB migration due to presence of SENTRY_TSDB_OPTIONS" + else + echo "Attempting to automatically migrate to new TSDB" + # Escape newlines for sed + tsdb_settings="${tsdb_settings//$'\n'/\\n}" + cp "$SENTRY_CONFIG_PY" "$SENTRY_CONFIG_PY.bak" + sed -i -e "s/^SENTRY_TSDB = .*$/${tsdb_settings}/g" "$SENTRY_CONFIG_PY" || true + + if grep -xq 'SENTRY_TSDB = "sentry.tsdb.redissnuba.RedisSnubaTSDB"' "$SENTRY_CONFIG_PY"; then + echo "Migrated TSDB to Snuba. Old configuration file backed up to $SENTRY_CONFIG_PY.bak" + return + fi + + echo "Failed to automatically migrate TSDB. Reverting..." + mv "$SENTRY_CONFIG_PY.bak" "$SENTRY_CONFIG_PY" + echo "$SENTRY_CONFIG_PY restored from backup." + fi + + echo "WARN: Your Sentry configuration uses a legacy data store for time-series data. Remove the options SENTRY_TSDB and SENTRY_TSDB_OPTIONS from $SENTRY_CONFIG_PY and add:" + echo "" + echo "$tsdb_settings" + echo "" + echo "For more information please refer to https://github.com/getsentry/onpremise/pull/430" + fi +} + +replace_tsdb + +echo "${_endgroup}" diff --git a/install/set-up-zookeeper.sh b/install/set-up-zookeeper.sh new file mode 100644 index 0000000000..00b633da15 --- /dev/null +++ b/install/set-up-zookeeper.sh @@ -0,0 +1,14 @@ +echo "${_group}Setting up Zookeeper ..." + +ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'') +if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then + ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') + ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') + # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 + if [[ "$ZOOKEEPER_LOG_FILE_COUNT" -gt 0 ]] && [[ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq 0 ]]; then + $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' + $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper + fi +fi + +echo "${_endgroup}" diff --git a/install/turn-things-off.sh b/install/turn-things-off.sh new file mode 100644 index 0000000000..090dc8d396 --- /dev/null +++ b/install/turn-things-off.sh @@ -0,0 +1,14 @@ +echo "${_group}Turning things off ..." + +if [[ -n "$MINIMIZE_DOWNTIME" ]]; then + # Stop everything but relay and nginx + $dc rm -fsv $($dc config --services | grep -v -E '^(nginx|relay)$') +else + # Clean up old stuff and ensure nothing is working while we install/update + # This is for older versions of on-premise: + $dc -p onpremise down -t $STOP_TIMEOUT --rmi local --remove-orphans + # This is for newer versions + $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans +fi + +echo "${_endgroup}" diff --git a/install/update-docker-images.sh b/install/update-docker-images.sh new file mode 100644 index 0000000000..e6d232cac7 --- /dev/null +++ b/install/update-docker-images.sh @@ -0,0 +1,13 @@ +echo "${_group}Fetching and updating Docker images ..." + +# We tag locally built images with an '-onpremise-local' suffix. docker-compose +# pull tries to pull these too and shows a 404 error on the console which is +# confusing and unnecessary. To overcome this, we add the stderr>stdout +# redirection below and pass it through grep, ignoring all lines having this +# '-onpremise-local' suffix. +$dc pull -q --ignore-pull-failures 2>&1 | grep -v -- -onpremise-local || true + +# We may not have the set image on the repo (local images) so allow fails +docker pull ${SENTRY_IMAGE} || true; + +echo "${_endgroup}" From a0a86e400f886cb65471bb6f79600872e0db7ba0 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 30 Mar 2021 10:23:49 -0400 Subject: [PATCH 309/417] Finish the refactor (#908) --- install.sh | 176 +++----------------------- install/_lib.sh | 15 ++- install/avoid-git-bash.sh | 0 install/check-minimum-requirements.sh | 53 ++++++++ install/create-docker-volumes-test.sh | 6 +- install/ensure-files-from-examples.sh | 4 +- install/error-handling.sh | 35 +++++ install/geoip-test.sh | 4 +- install/parse-cli.sh | 32 +++++ install/relay-credentials-test.sh | 4 +- install/restart-carefully.sh | 13 -- install/wrap-up.sh | 25 ++++ 12 files changed, 184 insertions(+), 183 deletions(-) create mode 100644 install/avoid-git-bash.sh create mode 100644 install/check-minimum-requirements.sh create mode 100644 install/error-handling.sh create mode 100644 install/parse-cli.sh delete mode 100644 install/restart-carefully.sh create mode 100644 install/wrap-up.sh diff --git a/install.sh b/install.sh index ba02bf64f7..171851b59d 100755 --- a/install.sh +++ b/install.sh @@ -1,162 +1,28 @@ #!/usr/bin/env bash set -e - if [[ -n "$MSYSTEM" ]]; then echo "Seems like you are using an MSYS2-based system (such as Git Bash) which is not supported. Please use WSL instead."; exit 1 fi -# Thanks to https://unix.stackexchange.com/a/145654/108960 -log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" -exec &> >(tee -a "$log_file") - -source "$(dirname $0)/install/_lib.sh" - -echo "${_group}Defining variables and helpers ..." -MIN_DOCKER_VERSION='19.03.6' -MIN_COMPOSE_VERSION='1.24.1' -MIN_RAM_HARD=3800 # MB -MIN_RAM_SOFT=7800 # MB -MIN_CPU_HARD=2 -MIN_CPU_SOFT=4 -echo $_endgroup - -echo "${_group}Parsing command line ..." -show_help() { - cat < /dev/null - fi -} -trap_with_arg cleanup ERR INT TERM EXIT -echo "${_endgroup}" - -echo "${_group}Checking minimum requirements ..." -DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') -COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') -RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); -CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); - -# Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 -function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } - -if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then - echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" - exit 1 -fi - -if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then - echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" - exit 1 -fi - -if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then - echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" - exit 1 -elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then - echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER" -fi - -if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then - echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB" - exit 1 -elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then - echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB" -fi - -#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) -# On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 -IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) -if [[ "$IS_KVM" -eq 0 ]]; then - SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) - if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then - echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." - exit 1 - fi -fi -echo "${_endgroup}" - -source ./install/create-docker-volumes.sh -source ./install/ensure-files-from-examples.sh -source ./install/generate-secret-key.sh -source ./install/replace-tsdb.sh -source ./install/update-docker-images.sh -source ./install/build-docker-images.sh -source ./install/turn-things-off.sh -source ./install/set-up-zookeeper.sh -source ./install/bootstrap-snuba.sh -source ./install/create-kafka-topics.sh -source ./install/upgrade-postgres.sh -source ./install/set-up-and-migrate-database.sh -source ./install/migrate-file-storage.sh -source ./install/relay-credentials.sh -source ./install/geoip.sh - -if [[ "$MINIMIZE_DOWNTIME" ]]; then - source ./install/restart-carefully.sh -else - echo "" - echo "-----------------------------------------------------------------" - echo "" - echo "You're all done! Run the following command to get Sentry running:" - echo "" - echo " docker-compose up -d" - echo "" - echo "-----------------------------------------------------------------" - echo "" -fi +source "$(dirname $0)/install/_lib.sh" # does a `cd .../install/`, among other things + +source parse-cli.sh +source error-handling.sh +source check-minimum-requirements.sh +source create-docker-volumes.sh +source ensure-files-from-examples.sh +source generate-secret-key.sh +source replace-tsdb.sh +source update-docker-images.sh +source build-docker-images.sh +source turn-things-off.sh +source set-up-zookeeper.sh +source bootstrap-snuba.sh +source create-kafka-topics.sh +source upgrade-postgres.sh +source set-up-and-migrate-database.sh +source migrate-file-storage.sh +source relay-credentials.sh +source geoip.sh +source wrap-up.sh diff --git a/install/_lib.sh b/install/_lib.sh index 9c636fdff4..0e0ad5b48f 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -1,15 +1,18 @@ set -euo pipefail test "${DEBUG:-}" && set -x +# Thanks to https://unix.stackexchange.com/a/145654/108960 +log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" +exec &> >(tee -a "$log_file") + # Work from the onpremise root, no matter which script is called from where. if [[ "$(basename $0)" = "install.sh" ]]; then - cd "$(dirname $0)" + cd "$(dirname $0)/install/" else - cd "$(dirname $0)/.." + cd "$(dirname $0)" # assume we're a *-test.sh script fi -if [[ ! -d 'install' ]]; then echo 'Where are you?'; exit 1; fi -_ENV="$(realpath .env)" +_ENV="$(realpath ../.env)" # Read .env for default values with a tip o' the hat to https://stackoverflow.com/a/59831605/90297 t=$(mktemp) && export -p > "$t" && set -a && . $_ENV && set +a && . "$t" && rm "$t" && unset t @@ -36,8 +39,8 @@ function ensure_file_from_example { # sed from https://stackoverflow.com/a/25123013/90297 fi } -SENTRY_CONFIG_PY='sentry/sentry.conf.py' -SENTRY_CONFIG_YML='sentry/config.yml' +SENTRY_CONFIG_PY='../sentry/sentry.conf.py' +SENTRY_CONFIG_YML='../sentry/config.yml' # Increase the default 10 second SIGTERM timeout # to ensure celery queues are properly drained diff --git a/install/avoid-git-bash.sh b/install/avoid-git-bash.sh new file mode 100644 index 0000000000..e69de29bb2 diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh new file mode 100644 index 0000000000..4527a222ef --- /dev/null +++ b/install/check-minimum-requirements.sh @@ -0,0 +1,53 @@ +echo "${_group}Checking minimum requirements ..." + +MIN_DOCKER_VERSION='19.03.6' +MIN_COMPOSE_VERSION='1.24.1' +MIN_RAM_HARD=3800 # MB +MIN_RAM_SOFT=7800 # MB +MIN_CPU_HARD=2 +MIN_CPU_SOFT=4 + +DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') +COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') +RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); +CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); + +# Compare dot-separated strings - function below is inspired by https://stackoverflow.com/a/37939589/808368 +function ver () { echo "$@" | awk -F. '{ printf("%d%03d%03d", $1,$2,$3); }'; } + +if [[ "$(ver $DOCKER_VERSION)" -lt "$(ver $MIN_DOCKER_VERSION)" ]]; then + echo "FAIL: Expected minimum Docker version to be $MIN_DOCKER_VERSION but found $DOCKER_VERSION" + exit 1 +fi + +if [[ "$(ver $COMPOSE_VERSION)" -lt "$(ver $MIN_COMPOSE_VERSION)" ]]; then + echo "FAIL: Expected minimum docker-compose version to be $MIN_COMPOSE_VERSION but found $COMPOSE_VERSION" + exit 1 +fi + +if [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_HARD" ]]; then + echo "FAIL: Required minimum CPU cores available to Docker is $MIN_CPU_HARD, found $CPU_AVAILABLE_IN_DOCKER" + exit 1 +elif [[ "$CPU_AVAILABLE_IN_DOCKER" -lt "$MIN_CPU_SOFT" ]]; then + echo "WARN: Recommended minimum CPU cores available to Docker is $MIN_CPU_SOFT, found $CPU_AVAILABLE_IN_DOCKER" +fi + +if [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_HARD" ]]; then + echo "FAIL: Required minimum RAM available to Docker is $MIN_RAM_HARD MB, found $RAM_AVAILABLE_IN_DOCKER MB" + exit 1 +elif [[ "$RAM_AVAILABLE_IN_DOCKER" -lt "$MIN_RAM_SOFT" ]]; then + echo "WARN: Recommended minimum RAM available to Docker is $MIN_RAM_SOFT MB, found $RAM_AVAILABLE_IN_DOCKER MB" +fi + +#SSE4.2 required by Clickhouse (https://clickhouse.yandex/docs/en/operations/requirements/) +# On KVM, cpuinfo could falsely not report SSE 4.2 support, so skip the check. https://github.com/ClickHouse/ClickHouse/issues/20#issuecomment-226849297 +IS_KVM=$(docker run --rm busybox grep -c 'Common KVM processor' /proc/cpuinfo || :) +if [[ "$IS_KVM" -eq 0 ]]; then + SUPPORTS_SSE42=$(docker run --rm busybox grep -c sse4_2 /proc/cpuinfo || :) + if [[ "$SUPPORTS_SSE42" -eq 0 ]]; then + echo "FAIL: The CPU your machine is running on does not support the SSE 4.2 instruction set, which is required for one of the services Sentry uses (Clickhouse). See https://git.io/JvLDt for more info." + exit 1 + fi +fi + +echo "${_endgroup}" diff --git a/install/create-docker-volumes-test.sh b/install/create-docker-volumes-test.sh index 86f8b172ee..7cf8969ee0 100755 --- a/install/create-docker-volumes-test.sh +++ b/install/create-docker-volumes-test.sh @@ -11,9 +11,9 @@ count() { before=$(count) test $before -eq 0 || test $before -eq $expected -source ./install/create-docker-volumes.sh -source ./install/create-docker-volumes.sh -source ./install/create-docker-volumes.sh +source create-docker-volumes.sh +source create-docker-volumes.sh +source create-docker-volumes.sh test $(count) -eq $expected diff --git a/install/ensure-files-from-examples.sh b/install/ensure-files-from-examples.sh index 0a507d634f..17958a01b4 100644 --- a/install/ensure-files-from-examples.sh +++ b/install/ensure-files-from-examples.sh @@ -2,7 +2,7 @@ echo "${_group}Ensuring files from examples ..." ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML -ensure_file_from_example 'symbolicator/config.yml' -ensure_file_from_example 'sentry/requirements.txt' +ensure_file_from_example '../symbolicator/config.yml' +ensure_file_from_example '../sentry/requirements.txt' echo "${_endgroup}" diff --git a/install/error-handling.sh b/install/error-handling.sh new file mode 100644 index 0000000000..d25ee01fa0 --- /dev/null +++ b/install/error-handling.sh @@ -0,0 +1,35 @@ +echo "${_group}Setting up error handling ..." + +# Courtesy of https://stackoverflow.com/a/2183063/90297 +trap_with_arg() { + func="$1" ; shift + for sig ; do + trap "$func $sig "'$LINENO' "$sig" + done +} + +DID_CLEAN_UP=0 +# the cleanup function will be the exit point +cleanup () { + if [[ "$DID_CLEAN_UP" -eq 1 ]]; then + return 0; + fi + DID_CLEAN_UP=1 + + if [[ "$1" != "EXIT" ]]; then + echo "An error occurred, caught SIG$1 on line $2"; + + if [[ -n "$MINIMIZE_DOWNTIME" ]]; then + echo "*NOT* cleaning up, to clean your environment run \"docker-compose stop\"." + else + echo "Cleaning up..." + fi + fi + + if [[ -z "$MINIMIZE_DOWNTIME" ]]; then + $dc stop -t $STOP_TIMEOUT &> /dev/null + fi +} +trap_with_arg cleanup ERR INT TERM EXIT + +echo "${_endgroup}" diff --git a/install/geoip-test.sh b/install/geoip-test.sh index 38650ab787..f6c55cc69e 100755 --- a/install/geoip-test.sh +++ b/install/geoip-test.sh @@ -5,12 +5,12 @@ mmdb="geoip/GeoLite2-City.mmdb" # Starts with no mmdb, ends up with empty. test ! -f $mmdb -source ./install/geoip.sh +source geoip.sh diff -rub $mmdb $mmdb.empty # Doesn't clobber existing, though. echo GARBAGE > $mmdb -source ./install/geoip.sh +source geoip.sh test "$(cat $mmdb)" = "GARBAGE" report_success diff --git a/install/parse-cli.sh b/install/parse-cli.sh new file mode 100644 index 0000000000..f1b6218589 --- /dev/null +++ b/install/parse-cli.sh @@ -0,0 +1,32 @@ +echo "${_group}Parsing command line ..." + +show_help() { + cat < $cfg echo MOAR GARBAGE > $creds -source ./install/relay-credentials.sh +source relay-credentials.sh test "$(cat $cfg)" = "GARBAGE" test "$(cat $creds)" = "MOAR GARBAGE" diff --git a/install/restart-carefully.sh b/install/restart-carefully.sh deleted file mode 100644 index a25da9e19f..0000000000 --- a/install/restart-carefully.sh +++ /dev/null @@ -1,13 +0,0 @@ -echo "${_group}Waiting for Sentry to start ..." - -# Start the whole setup, except nginx and relay. -$dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') -$dc exec -T nginx service nginx reload - -docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ - -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' - -# Make sure everything is up. This should only touch relay and nginx -$dc up -d - -echo "${_endgroup}" diff --git a/install/wrap-up.sh b/install/wrap-up.sh new file mode 100644 index 0000000000..2671a3a513 --- /dev/null +++ b/install/wrap-up.sh @@ -0,0 +1,25 @@ +if [[ "$MINIMIZE_DOWNTIME" ]]; then + echo "${_group}Waiting for Sentry to start ..." + + # Start the whole setup, except nginx and relay. + $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') + $dc exec -T nginx service nginx reload + + docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ + -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' + + # Make sure everything is up. This should only touch relay and nginx + $dc up -d + + echo "${_endgroup}" +else + echo "" + echo "-----------------------------------------------------------------" + echo "" + echo "You're all done! Run the following command to get Sentry running:" + echo "" + echo " docker-compose up -d" + echo "" + echo "-----------------------------------------------------------------" + echo "" +fi From 8e34f6e9bc8bd35bcd46abda25623d2a11f622ec Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 30 Mar 2021 10:45:05 -0400 Subject: [PATCH 310/417] Fix paths (#909) --- install/_test_setup.sh | 4 +++- install/geoip-test.sh | 2 +- install/geoip.sh | 8 ++++++-- install/relay-credentials-test.sh | 4 ++-- install/relay-credentials.sh | 4 ++-- 5 files changed, 14 insertions(+), 8 deletions(-) diff --git a/install/_test_setup.sh b/install/_test_setup.sh index 3a4409912c..e00d8d66c2 100644 --- a/install/_test_setup.sh +++ b/install/_test_setup.sh @@ -13,6 +13,8 @@ teardown() { } setup() { + cd .. + # Clone the local repo into a temp dir. FWIW `git clone --local` breaks for # me because it depends on hard-linking, which doesn't work across devices, # and I happen to have my workspace and /tmp on separate devices. @@ -42,7 +44,7 @@ setup() { esac done - cd "$_SANDBOX" + cd "$_SANDBOX/install" trap teardown EXIT } diff --git a/install/geoip-test.sh b/install/geoip-test.sh index f6c55cc69e..3d61c11e3d 100755 --- a/install/geoip-test.sh +++ b/install/geoip-test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash source "$(dirname $0)/_test_setup.sh" -mmdb="geoip/GeoLite2-City.mmdb" +mmdb="../geoip/GeoLite2-City.mmdb" # Starts with no mmdb, ends up with empty. test ! -f $mmdb diff --git a/install/geoip.sh b/install/geoip.sh index 9a1317be45..bc5d84b64a 100644 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -1,8 +1,10 @@ echo "${_group}Setting up GeoIP integration ..." install_geoip() { - local mmdb='geoip/GeoLite2-City.mmdb' - local conf='geoip/GeoIP.conf' + cd ../geoip + + local mmdb='GeoLite2-City.mmdb' + local conf='GeoIP.conf' local result='Done' echo "Setting up IP address geolocation ..." @@ -27,6 +29,8 @@ install_geoip() { echo "$result updating IP address geolocation database." fi echo "$result setting up IP address geolocation." + + cd ../install } install_geoip diff --git a/install/relay-credentials-test.sh b/install/relay-credentials-test.sh index c025410f3d..ea740f3744 100755 --- a/install/relay-credentials-test.sh +++ b/install/relay-credentials-test.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash source "$(dirname $0)/_test_setup.sh" -cfg="relay/config.yml" -creds="relay/credentials.json" +cfg="../relay/config.yml" +creds="../relay/credentials.json" # Relay files don't exist in a clean clone. test ! -f $cfg diff --git a/install/relay-credentials.sh b/install/relay-credentials.sh index 91fe34df72..2d62e2bf53 100644 --- a/install/relay-credentials.sh +++ b/install/relay-credentials.sh @@ -1,7 +1,7 @@ echo "${_group}Generating Relay credentials ..." -RELAY_CONFIG_YML="relay/config.yml" -RELAY_CREDENTIALS_JSON="relay/credentials.json" +RELAY_CONFIG_YML="../relay/config.yml" +RELAY_CREDENTIALS_JSON="../relay/credentials.json" ensure_file_from_example $RELAY_CONFIG_YML From e5e8baca2eac0b3ec87909b8fadd5b4b3aa86941 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 30 Mar 2021 13:58:15 -0400 Subject: [PATCH 311/417] Remove dead file (#912) --- install/avoid-git-bash.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 install/avoid-git-bash.sh diff --git a/install/avoid-git-bash.sh b/install/avoid-git-bash.sh deleted file mode 100644 index e69de29bb2..0000000000 From 2ab05909248f563147ab1cd4ed7488475debb4d9 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 30 Mar 2021 22:02:40 +0300 Subject: [PATCH 312/417] ci(test): Separate unit tests so they don't rely on integration tests (#910) --- .github/workflows/test.yml | 20 ++++++++++++-------- install/_test_setup.sh | 2 +- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 378accc41e..b01e3516d9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -14,7 +14,18 @@ defaults: run: shell: bash jobs: - test: + unit-test: + runs-on: ubuntu-18.04 + name: "unit tests" + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Unit Tests + working-directory: install + run: find ./ -type f -name "*-test.sh" -exec "./{}" \; + + integration-test: runs-on: ubuntu-18.04 name: "test" steps: @@ -39,13 +50,6 @@ jobs: ./install.sh --minimize-downtime ./test.sh - - name: Unit Tests - working-directory: install - run: | - ./create-docker-volumes-test.sh - ./relay-credentials-test.sh - ./geoip-test.sh - - name: Inspect failure if: failure() run: | diff --git a/install/_test_setup.sh b/install/_test_setup.sh index e00d8d66c2..6fdf29eecb 100644 --- a/install/_test_setup.sh +++ b/install/_test_setup.sh @@ -18,7 +18,7 @@ setup() { # Clone the local repo into a temp dir. FWIW `git clone --local` breaks for # me because it depends on hard-linking, which doesn't work across devices, # and I happen to have my workspace and /tmp on separate devices. - git clone --depth=1 "file://$(pwd)" "$_SANDBOX" + git -c advice.detachedHead=false clone --depth=1 "file://$(pwd)" "$_SANDBOX" # Now propagate any local changes from the working copy to the sandbox. This # provides a pretty nice dev experience: edit the files in the working copy, From 315cbaa96d89ad02faf67005e72ae92eee05bd19 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 8 Apr 2021 21:51:24 +0000 Subject: [PATCH 313/417] release: 21.3.1 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..352d7c242c 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.3.1 +SNUBA_IMAGE=getsentry/snuba:21.3.1 +RELAY_IMAGE=getsentry/relay:21.3.1 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index 9a133dcd4e..f18f128a93 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-03-16 +Change Date: 2024-04-08 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 1d618be3a2..e2357442ad 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.3.1 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 2223723fa178dbfcd7807d804739e0412328ed13 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 8 Apr 2021 21:59:17 +0000 Subject: [PATCH 314/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 352d7c242c..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.3.1 -SNUBA_IMAGE=getsentry/snuba:21.3.1 -RELAY_IMAGE=getsentry/relay:21.3.1 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index e2357442ad..1d618be3a2 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.3.1 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From aecc75b1c91370469d501f51b139b7b4997c0995 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Mon, 12 Apr 2021 10:23:53 -0700 Subject: [PATCH 315/417] feat: Update storage target for Snuba consumer and replacer. (#920) This change ensures that the Snuba consumer and replacer start to fill in the new table. It should be applied once we have backfilled data and are ready to cut over to the new storage. Depends on https://github.com/getsentry/snuba/pull/1801 --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index e4c4de1e0f..f0c6441d80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -141,7 +141,7 @@ services: # Kafka consumer responsible for feeding events into Clickhouse snuba-consumer: <<: *snuba_defaults - command: consumer --storage events --auto-offset-reset=latest --max-batch-time-ms 750 + command: consumer --storage errors --auto-offset-reset=latest --max-batch-time-ms 750 # Kafka consumer responsible for feeding outcomes into Clickhouse # Use --auto-offset-reset=earliest to recover up to 7 days of TSDB data # since we did not do a proper migration @@ -158,7 +158,7 @@ services: command: consumer --storage transactions --consumer-group transactions_group --auto-offset-reset=latest --max-batch-time-ms 750 --commit-log-topic=snuba-commit-log snuba-replacer: <<: *snuba_defaults - command: replacer --storage events --auto-offset-reset=latest --max-batch-size 3 + command: replacer --storage errors --auto-offset-reset=latest --max-batch-size 3 snuba-subscription-consumer-events: <<: *snuba_defaults command: subscriptions --auto-offset-reset=latest --consumer-group=snuba-events-subscriptions-consumers --topic=events --result-topic=events-subscription-results --dataset=events --commit-log-topic=snuba-commit-log --commit-log-group=snuba-consumers --delay-seconds=60 --schedule-ttl=60 From dd5a7d430e6341315a98dd0ad8da8988d65bab12 Mon Sep 17 00:00:00 2001 From: Stephen Cefali Date: Mon, 12 Apr 2021 11:52:00 -0700 Subject: [PATCH 316/417] docs: explain changing version on Linux (#922) Explains how to set the version on Linux where `sudo` is required. Based on this Stack overflow answer: https://stackoverflow.com/questions/8633461/how-to-keep-environment-variables-when-using-sudo/8636711#8636711 --- README.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/README.md b/README.md index 1d618be3a2..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -40,6 +40,24 @@ SENTRY_IMAGE=getsentry/sentry:83b1380 ./install.sh Note that this may not work for all commit SHAs as this repository evolves with Sentry and its satellite projects. It is highly recommended to check out a version of this repository that is close to the timestamp of the Sentry commit you are installing. +### Using Linux + +If you are using Linux and you need to use `sudo` when running `./install.sh`, modifying the version of Sentry is slightly different. First, run the following: +```shell +sudo visudo +``` +Then add the following line: +```shell +Defaults env_keep += "SENTRY_IMAGE" +``` +Save the file then in your terminal run the following + +```shell +export SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 +sudo ./install.sh +``` +Where you replace `83b1380` with the sha you want to use. + ## Event Retention Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. From 257bccc96cac7c69595fd71efdac8dab10045723 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 12 Apr 2021 23:43:09 +0300 Subject: [PATCH 317/417] fix: test.sh should use the variable $_group (#921) --- install/_lib.sh | 2 +- test.sh | 28 +++++++++++++++------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/install/_lib.sh b/install/_lib.sh index 0e0ad5b48f..9a852f591b 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -5,7 +5,7 @@ test "${DEBUG:-}" && set -x log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") -# Work from the onpremise root, no matter which script is called from where. +# Work from /install/ for install.sh, project root otherwise if [[ "$(basename $0)" = "install.sh" ]]; then cd "$(dirname $0)/install/" else diff --git a/test.sh b/test.sh index 4c6ca2fead..24c462d9ab 100755 --- a/test.sh +++ b/test.sh @@ -1,7 +1,9 @@ #!/usr/bin/env bash set -e -echo "::group::Setting up variables and helpers ..." +source "$(dirname $0)/install/_lib.sh" + +echo "${_group}Setting up variables and helpers ..." export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' @@ -32,17 +34,17 @@ cleanup () { echo "Done." } trap_with_arg cleanup ERR INT TERM EXIT -echo "::endgroup::" +echo "${_endgroup}" -echo "::group::Starting Sentry for tests ..." +echo "${_group}Starting Sentry for tests ..." # Disable beacon for e2e tests echo 'SENTRY_BEACON=False' >> sentry/sentry.conf.py -docker-compose run --rm web createuser --superuser --email $TEST_USER --password $TEST_PASS || true -docker-compose up -d +$dcr web createuser --superuser --email $TEST_USER --password $TEST_PASS || true +$dc up -d printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' -echo "::endgroup::" +echo "${_endgroup}" -echo "::group::Running tests ..." +echo "${_group}Running tests ..." get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } @@ -75,9 +77,9 @@ do echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done -echo "::endgroup::" +echo "${_endgroup}" -echo "::group::Running moar tests !!!" +echo "${_group}Running moar tests !!!" # Set up initial/required settings (InstallWizard request) sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null @@ -112,8 +114,8 @@ do echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done -echo "::endgroup::" +echo "${_endgroup}" -echo "::group::Ensure cleanup crons are working ..." -docker-compose ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+" -echo "::endgroup::" +echo "${_group}Ensure cleanup crons are working ..." +$dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+" +echo "${_endgroup}" From 35a45ea7ae64b05ccf6a0c2f5bcbbaf204766c61 Mon Sep 17 00:00:00 2001 From: Lyn Nagara Date: Wed, 14 Apr 2021 01:50:15 -0700 Subject: [PATCH 318/417] fix: Ensure snuba cleanup job runs on the correct storage (#923) Follow up to #920. --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index f0c6441d80..71fd01c278 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -172,7 +172,7 @@ services: context: ./cron args: BASE_IMAGE: "$SNUBA_IMAGE" - command: '"*/5 * * * * gosu snuba snuba cleanup --dry-run False"' + command: '"*/5 * * * * gosu snuba snuba cleanup --storage errors --dry-run False"' symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" From e0dde9c57b9c04791935c5455c4ddcb6024e8c1f Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 15 Apr 2021 21:20:39 +0300 Subject: [PATCH 319/417] fix: Add missing snuba transactions clean up service (#924) Follow up to #920 and #923. --- docker-compose.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 71fd01c278..33f39a6d81 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -173,6 +173,14 @@ services: args: BASE_IMAGE: "$SNUBA_IMAGE" command: '"*/5 * * * * gosu snuba snuba cleanup --storage errors --dry-run False"' + snuba-transactions-cleanup: + <<: *snuba_defaults + image: snuba-cleanup-onpremise-local + build: + context: ./cron + args: + BASE_IMAGE: "$SNUBA_IMAGE" + command: '"*/5 * * * * gosu snuba snuba cleanup --storage transactions --dry-run False"' symbolicator: <<: *restart_policy image: "$SYMBOLICATOR_IMAGE" From faf108863a504c4e328e4f8d7562f819523f45ef Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 15 Apr 2021 19:20:03 +0000 Subject: [PATCH 320/417] release: 21.4.0 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..605c069d57 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.4.0 +SNUBA_IMAGE=getsentry/snuba:21.4.0 +RELAY_IMAGE=getsentry/relay:21.4.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index f18f128a93..3ee60110bd 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-04-08 +Change Date: 2024-04-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 53f99bd4b9..25c7880491 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.4.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 662a773e57dbae037ef095af588617f0c23cf4af Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 15 Apr 2021 19:27:57 +0000 Subject: [PATCH 321/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 605c069d57..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.4.0 -SNUBA_IMAGE=getsentry/snuba:21.4.0 -RELAY_IMAGE=getsentry/relay:21.4.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index 25c7880491..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.4.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 37efaf288223c9dde2f959eab20fd8604f5b8f23 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 19 Apr 2021 14:48:54 +0200 Subject: [PATCH 322/417] ref(relay): Remove comment from sample config (#926) This comment has unprofessional tone and does not serve any purpose. Since this is copied to every onpremise installation and I keep copying this code to public issue comments, it's better removed. --- relay/config.example.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/relay/config.example.yml b/relay/config.example.yml index 8538bd7d46..52e6630671 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -8,6 +8,6 @@ processing: enabled: true kafka_config: - {name: "bootstrap.servers", value: "kafka:9092"} - - {name: "message.max.bytes", value: 50000000} #50MB or bust + - {name: "message.max.bytes", value: 50000000} # 50MB redis: redis://redis:6379 geoip_path: "/geoip/GeoLite2-City.mmdb" From e7a11016838a1a8ea85db2f8869f6f1eafb90368 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Wed, 21 Apr 2021 22:32:40 +0000 Subject: [PATCH 323/417] release: 21.4.1 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..dac8b2ca95 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.4.1 +SNUBA_IMAGE=getsentry/snuba:21.4.1 +RELAY_IMAGE=getsentry/relay:21.4.1 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index 3ee60110bd..6f2b16024e 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-04-15 +Change Date: 2024-04-21 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 53f99bd4b9..aaec3e1411 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.4.1 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From f3d0e3d1935fe54ebd836351fa91fdb3fef7975e Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Wed, 21 Apr 2021 22:39:58 +0000 Subject: [PATCH 324/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index dac8b2ca95..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.4.1 -SNUBA_IMAGE=getsentry/snuba:21.4.1 -RELAY_IMAGE=getsentry/relay:21.4.1 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index aaec3e1411..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.4.1 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 60e947d4f714469d56d3dbd5b4d7d1aaecc0af7e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 22 Apr 2021 20:02:11 +0300 Subject: [PATCH 325/417] fix: Fix .env path for test scripts (#928) --- install/_lib.sh | 2 +- test.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/install/_lib.sh b/install/_lib.sh index 9a852f591b..2d7517fdc6 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -6,7 +6,7 @@ log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") # Work from /install/ for install.sh, project root otherwise -if [[ "$(basename $0)" = "install.sh" ]]; then +if [[ "$(basename $0)" = "install.sh" || "$(basename $0)" = "test.sh" ]]; then cd "$(dirname $0)/install/" else cd "$(dirname $0)" # assume we're a *-test.sh script diff --git a/test.sh b/test.sh index 24c462d9ab..26a9e99ac9 100755 --- a/test.sh +++ b/test.sh @@ -38,7 +38,7 @@ echo "${_endgroup}" echo "${_group}Starting Sentry for tests ..." # Disable beacon for e2e tests -echo 'SENTRY_BEACON=False' >> sentry/sentry.conf.py +echo 'SENTRY_BEACON=False' >> $SENTRY_CONFIG_PY $dcr web createuser --superuser --email $TEST_USER --password $TEST_PASS || true $dc up -d printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' From a95b9fa011ebf3d8fb0fcd6e20d35527c72911cf Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 26 Apr 2021 13:16:29 +0300 Subject: [PATCH 326/417] fix: Make relay depend on web for DNS resolution (#934) This is a stop-gap solution to #918 until we figure out the negative DNS caching issue inside `relay`. This may also be due to Docker Compose making some assumptions/optimizations/limiting regarding cross-container access unless they are explicitly linked via the `depends_on` key. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 33f39a6d81..c981be9b4f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -259,6 +259,7 @@ services: depends_on: - kafka - redis + - web volumes: sentry-data: external: true From f27eaef55781a43cf00bff4bcdc64bea8d77c707 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 6 May 2021 23:18:46 +0300 Subject: [PATCH 327/417] meta(gha): Deploy action issue-status-helper.yml (#943) --- .github/workflows/issue-status-helper.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 .github/workflows/issue-status-helper.yml diff --git a/.github/workflows/issue-status-helper.yml b/.github/workflows/issue-status-helper.yml new file mode 100644 index 0000000000..696624baa3 --- /dev/null +++ b/.github/workflows/issue-status-helper.yml @@ -0,0 +1,16 @@ +name: Issue Status Helper +on: + issues: + types: [labeled] +jobs: + routed: + runs-on: ubuntu-latest + if: "startsWith(github.event.label.name, 'Status: ')" + steps: + - name: "Ensure a single 'Status: *' label" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + run: | + labels_to_remove=$(gh api "/repos/$GH_REPO/labels" -q '[.[].name | select(startswith("Status: ") and . != "${{ github.event.label.name }}")] | join(",")') + gh issue edit ${{ github.event.issue.number }} --remove-label "$labels_to_remove" --add-label "${{ github.event.label.name }}" \ No newline at end of file From 1068326998cd73a8e69c3d938fc6c0aa04f10b61 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 7 May 2021 23:09:39 +0300 Subject: [PATCH 328/417] meta(gha): Deploy action issue-routing-helper.yml (#945) --- .github/workflows/issue-routing-helper.yml | 39 ++++++++++++++++++++++ 1 file changed, 39 insertions(+) create mode 100644 .github/workflows/issue-routing-helper.yml diff --git a/.github/workflows/issue-routing-helper.yml b/.github/workflows/issue-routing-helper.yml new file mode 100644 index 0000000000..0648d907d0 --- /dev/null +++ b/.github/workflows/issue-routing-helper.yml @@ -0,0 +1,39 @@ +name: Issue Routing Helper +on: + issues: + types: [labeled] +env: + # Use GH_RELEASE_PAT as github-actions bot is not allowed to ping teams + GH_TOKEN: ${{ secrets.GH_RELEASE_PAT }} + GH_REPO: ${{ github.repository }} +jobs: + route: + runs-on: ubuntu-latest + if: "startsWith(github.event.label.name, 'Team: ')" + steps: + - name: "Ensure a single 'Team: *' label with 'Status: Untriaged'" + run: | + labels_to_remove=$(gh api --paginate "/repos/$GH_REPO/labels" -q '[.[].name | select((startswith("Team: ") or startswith("Status: ")) and . != "${{ github.event.label.name }}" and . != "Status: Untriaged")] | join(",")') + gh issue edit ${{ github.event.issue.number }} --remove-label "$labels_to_remove" --add-label '${{ github.event.label.name }},Status: Untriaged' + - name: "Mention/ping assigned team for triage" + run: | + # Get team label mention name: + team_label='${{ github.event.label.name }}' + team_name="${team_label:6}" # Strip the first 6 chars, which is the 'Team: ' part + team_slug="${team_name// /-}" # Replace spaces with hyphens for url/slug friendliness + mention_slug=$(gh api "/orgs/getsentry/teams/$team_slug" -q .slug || true) + + if [[ -z "$mention_slug" ]]; then + echo "Couldn't find team mention from slug, trying the label description" + team_slug=$(gh api "/repos/$GH_REPO/labels/$team_label" -q '.description') + mention_slug=$(gh api "/orgs/getsentry/teams/$team_slug" -q .slug || true) + fi + + if [[ -n "$mention_slug" ]]; then + echo "Routing to @getsentry/$mention_slug for [triage](https://develop.sentry.dev/processing-tickets/#3-triage). ⏲️" > comment_body + else + echo "[Failed]($GITHUB_SERVER_URL/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID) to route to \`${{ github.event.label.name }}\`. 😕" > comment_body + echo "" >> comment_body + echo "Defaulting to @getsentry/open-source for [triage](https://develop.sentry.dev/processing-tickets/#3-triage). ⏲️" >> comment_body + fi + gh issue comment ${{ github.event.issue.number }} --body-file comment_body From 67c2310e7f2bd185cf941d7168b3e9fa3b28504e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 12 May 2021 22:03:34 +0300 Subject: [PATCH 329/417] meta(gha): Deploy action issue-status-helper.yml (#949) --- .github/workflows/issue-status-helper.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/issue-status-helper.yml b/.github/workflows/issue-status-helper.yml index 696624baa3..b5164750e8 100644 --- a/.github/workflows/issue-status-helper.yml +++ b/.github/workflows/issue-status-helper.yml @@ -3,7 +3,7 @@ on: issues: types: [labeled] jobs: - routed: + ensure_one_status: runs-on: ubuntu-latest if: "startsWith(github.event.label.name, 'Status: ')" steps: @@ -12,5 +12,5 @@ jobs: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} GH_REPO: ${{ github.repository }} run: | - labels_to_remove=$(gh api "/repos/$GH_REPO/labels" -q '[.[].name | select(startswith("Status: ") and . != "${{ github.event.label.name }}")] | join(",")') + labels_to_remove=$(gh api --paginate "/repos/$GH_REPO/labels" -q '[.[].name | select(startswith("Status: ") and . != "${{ github.event.label.name }}")] | join(",")') gh issue edit ${{ github.event.issue.number }} --remove-label "$labels_to_remove" --add-label "${{ github.event.label.name }}" \ No newline at end of file From 168f3b957fde0c7cc3640efb62d3fc1271b305bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?S=C3=A9bastien=20PIERRE?= Date: Thu, 13 May 2021 12:27:26 +0200 Subject: [PATCH 330/417] feat: Add basic healthchecks for Zookeeper & Kafka (#948) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add basic healthchecks on Zookeeper & Kafka containers to have a view on container status. These checks are quite basic because I have no knowledge at all on these components. Co-authored-by: Sébastien Pierre --- docker-compose.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index c981be9b4f..5415f5d1e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,10 +83,16 @@ services: CONFLUENT_SUPPORT_METRICS_ENABLE: "false" ZOOKEEPER_LOG4J_ROOT_LOGLEVEL: "WARN" ZOOKEEPER_TOOLS_LOG4J_LOGLEVEL: "WARN" + KAFKA_OPTS: "-Dzookeeper.4lw.commands.whitelist=ruok" volumes: - "sentry-zookeeper:/var/lib/zookeeper/data" - "sentry-zookeeper-log:/var/lib/zookeeper/log" - "sentry-secrets:/etc/zookeeper/secrets" + healthcheck: + test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] + interval: 10s + timeout: 5s + retries: 6 kafka: <<: *restart_policy depends_on: @@ -108,6 +114,11 @@ services: - "sentry-kafka:/var/lib/kafka/data" - "sentry-kafka-log:/var/lib/kafka/log" - "sentry-secrets:/etc/kafka/secrets" + healthcheck: + test: ["CMD-SHELL", 'nc -z localhost 9092'] + interval: 10s + timeout: 5s + retries: 6 clickhouse: <<: *restart_policy image: "yandex/clickhouse-server:20.3.9.70" From 84312dc4777f66e64dec1f86dd1105902d58ccfd Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 13 May 2021 21:56:48 +0300 Subject: [PATCH 331/417] fix: Fix incorrect zookeeper mount path (#952) Fixes #951, a regression introduced with #908. --- install/set-up-zookeeper.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/install/set-up-zookeeper.sh b/install/set-up-zookeeper.sh index 00b633da15..a4a56fec47 100644 --- a/install/set-up-zookeeper.sh +++ b/install/set-up-zookeeper.sh @@ -5,10 +5,12 @@ if [[ "$ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS" -eq 1 ]]; then ZOOKEEPER_LOG_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/log/version-2/* | wc -l | tr -d '[:space:]'') ZOOKEEPER_SNAPSHOT_FILE_COUNT=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2/* | wc -l | tr -d '[:space:]'') # This is a workaround for a ZK upgrade bug: https://issues.apache.org/jira/browse/ZOOKEEPER-3056 + cd .. if [[ "$ZOOKEEPER_LOG_FILE_COUNT" -gt 0 ]] && [[ "$ZOOKEEPER_SNAPSHOT_FILE_COUNT" -eq 0 ]]; then $dcr -v $(pwd)/zookeeper:/temp zookeeper bash -c 'cp /temp/snapshot.0 /var/lib/zookeeper/data/version-2/snapshot.0' $dc run -d -e ZOOKEEPER_SNAPSHOT_TRUST_EMPTY=true zookeeper fi + cd install fi echo "${_endgroup}" From 864e184e3b0712a735b898d9e9853fd17a44a474 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Mon, 17 May 2021 21:10:58 +0000 Subject: [PATCH 332/417] release: 21.5.0 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..65ab43b361 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.5.0 +SNUBA_IMAGE=getsentry/snuba:21.5.0 +RELAY_IMAGE=getsentry/relay:21.5.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index 6f2b16024e..37c1385185 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-04-21 +Change Date: 2024-05-17 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 53f99bd4b9..b346b211b1 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.5.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From a2c315cf513c6cb6ddff41bc87226ad56134f19e Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Mon, 17 May 2021 21:18:55 +0000 Subject: [PATCH 333/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 65ab43b361..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.5.0 -SNUBA_IMAGE=getsentry/snuba:21.5.0 -RELAY_IMAGE=getsentry/relay:21.5.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index b346b211b1..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.5.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From ee240f041c976df9e4701f80a0f6f2dd9b3fdd7b Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Wed, 19 May 2021 11:14:39 +0000 Subject: [PATCH 334/417] release: 21.5.1 --- .env | 8 ++++---- LICENSE | 2 +- README.md | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.env b/.env index 3848c8c0eb..8014f1a4b1 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.5.1 +SNUBA_IMAGE=getsentry/snuba:21.5.1 +RELAY_IMAGE=getsentry/relay:21.5.1 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 diff --git a/LICENSE b/LICENSE index 37c1385185..67c62a8f95 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-05-17 +Change Date: 2024-05-19 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 53f99bd4b9..067f723304 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.5.1 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 93bf1b6db56a44655c8e4178d34f885ee8bd0bc3 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Wed, 19 May 2021 11:22:39 +0000 Subject: [PATCH 335/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 8014f1a4b1..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.5.1 -SNUBA_IMAGE=getsentry/snuba:21.5.1 -RELAY_IMAGE=getsentry/relay:21.5.1 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.3 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly diff --git a/README.md b/README.md index 067f723304..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.5.1 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From d5e4e52d8181b0982ecfa61c593c07aadd27e025 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 21 May 2021 23:13:23 +0300 Subject: [PATCH 336/417] meta(gha): Deploy action issue-routing-helper.yml (#958) --- .github/workflows/issue-routing-helper.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/issue-routing-helper.yml b/.github/workflows/issue-routing-helper.yml index 0648d907d0..22e381c32e 100644 --- a/.github/workflows/issue-routing-helper.yml +++ b/.github/workflows/issue-routing-helper.yml @@ -9,7 +9,12 @@ env: jobs: route: runs-on: ubuntu-latest - if: "startsWith(github.event.label.name, 'Team: ')" + if: >- + startsWith(github.event.label.name, 'Team: ') + && + !contains(github.event.issue.labels.*.name, 'Status: Backlog') + && + !contains(github.event.issue.labels.*.name, 'Status: In Progress') steps: - name: "Ensure a single 'Team: *' label with 'Status: Untriaged'" run: | From 8dc84600c5ceb7063813d18657011fa53a659d3f Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Mon, 24 May 2021 17:51:36 -0700 Subject: [PATCH 337/417] feat(cdc): Prepare the self hosted environment for the Change Data Capture pipeline (#938) We will use Change Data Capture to stream WAL updates from postgres into clickhouse so that features like issue search will be able to join event data and metadata (from postgres) through Snuba. This requires the followings: A logical replicaiton plugin to be installed in postgres (https://github.com/getsentry/wal2json) A service to run that streams from the replication log to Kafka (https://github.com/getsentry/cdc) Datasets in Snuba. This PR is preparing postgres to stream updates via the replication log. The idea is to download the the replication log plugin binary during install.sh mount a volume with the binary when starting postgres providing a new entrypoint to postgres that ensures everything is correctly configured. There is a difference between how this is set up and how we do the same in the development environment. In the development environment we download the library from the entrypoint itself and store it in a persistent volume, so we do not have to download it every time. Unfortunately this does not work here as the postgres image is postgres:9.6 while it is postgres:9.6-alpine. This one does not come with either wget or curl. I don't think installing that in the entrypoint would be a good idea, so the download happens in install.sh. I actually think this way is safer so we never depend on connectivity for postgres to start properly. --- .env | 1 + .gitignore | 3 +++ docker-compose.yml | 6 +++++ install.sh | 1 + install/install-wal2json.sh | 34 ++++++++++++++++++++++++ postgres/init_hba.sh | 7 +++++ postgres/postgres-entrypoint.sh | 46 +++++++++++++++++++++++++++++++++ scripts/bump-version.sh | 3 +++ 8 files changed, 101 insertions(+) create mode 100644 install/install-wal2json.sh create mode 100755 postgres/init_hba.sh create mode 100755 postgres/postgres-entrypoint.sh diff --git a/.env b/.env index 3848c8c0eb..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -7,3 +7,4 @@ SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +WAL2JSON_VERSION=latest diff --git a/.gitignore b/.gitignore index 707622f425..8a169049fa 100644 --- a/.gitignore +++ b/.gitignore @@ -84,3 +84,6 @@ symbolicator/config.yml geoip/GeoIP.conf geoip/*.mmdb geoip/.geoipupdate.lock + +# wal2json download +postgres/wal2json diff --git a/docker-compose.yml b/docker-compose.yml index 5415f5d1e6..9a78d96930 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -71,10 +71,16 @@ services: postgres: <<: *restart_policy image: "postgres:9.6" + command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1"] environment: POSTGRES_HOST_AUTH_METHOD: "trust" + entrypoint: /opt/sentry/postgres-entrypoint.sh volumes: - "sentry-postgres:/var/lib/postgresql/data" + - type: bind + read_only: true + source: ./postgres/ + target: /opt/sentry/ zookeeper: <<: *restart_policy image: "confluentinc/cp-zookeeper:5.5.0" diff --git a/install.sh b/install.sh index 171851b59d..3886463061 100755 --- a/install.sh +++ b/install.sh @@ -18,6 +18,7 @@ source update-docker-images.sh source build-docker-images.sh source turn-things-off.sh source set-up-zookeeper.sh +source install-wal2json.sh source bootstrap-snuba.sh source create-kafka-topics.sh source upgrade-postgres.sh diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh new file mode 100644 index 0000000000..5ed58005e0 --- /dev/null +++ b/install/install-wal2json.sh @@ -0,0 +1,34 @@ +echo "${_group}Downloading and installing wal2json ..." + +FILE_TO_USE="../postgres/wal2json/wal2json.so" +ARCH=$(uname -m) +FILE_NAME="wal2json-Linux-$ARCH-glibc.so" + +DOCKER_CURL="docker run --rm curlimages/curl" + +if [[ $WAL2JSON_VERSION == "latest" ]]; then + VERSION=$( + $DOCKER_CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | + grep '"tag_name":' | + sed -E 's/.*"([^"]+)".*/\1/' + ) + + if [[ ! $VERSION ]]; then + echo "Cannot find wal2json latest version" + exit 1 + fi +else + VERSION=$WAL2JSON_VERSION +fi + +mkdir -p ../postgres/wal2json +if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then + mkdir -p "../postgres/wal2json/$VERSION" + $DOCKER_CURL -L \ + "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ + > "../postgres/wal2json/$VERSION/$FILE_NAME" + + cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" +fi + +echo "${_endgroup}" diff --git a/postgres/init_hba.sh b/postgres/init_hba.sh new file mode 100755 index 0000000000..f4b332abfa --- /dev/null +++ b/postgres/init_hba.sh @@ -0,0 +1,7 @@ +#!/bin/bash +# Initializes the pg_hba file with access permissions to the replication +# slots. + +set -e + +{ echo "host replication all all trust"; } >> "$PGDATA/pg_hba.conf" diff --git a/postgres/postgres-entrypoint.sh b/postgres/postgres-entrypoint.sh new file mode 100755 index 0000000000..0b0d98a964 --- /dev/null +++ b/postgres/postgres-entrypoint.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# This script replaces the default docker entrypoint for postgres in the +# development environment. +# Its job is to ensure postgres is properly configured to support the +# Change Data Capture pipeline (by setting access permissions and installing +# the replication plugin we use for CDC). Unfortunately the default +# Postgres image does not allow this level of configurability so we need +# to do it this way in order not to have to publish and maintain our own +# Postgres image. +# +# This then, at the end, transfers control to the default entrypoint. + +set -e + +prep_init_db() { + cp /opt/sentry/init_hba.sh /docker-entrypoint-initdb.d/init_hba.sh +} + +cdc_setup_hba_conf() { + # Ensure pg-hba is properly configured to allow connections + # to the replication slots. + + PG_HBA="$PGDATA/pg_hba.conf" + if [ ! -f "$PG_HBA" ]; then + echo "DB not initialized. Postgres will take care of pg_hba" + elif [ "$(grep -c -E "^host\s+replication" "$PGDATA"/pg_hba.conf)" != 0 ]; then + echo "Replication config already present in pg_hba. Not changing anything." + else + # Execute the same script we run on DB initialization + /opt/sentry/init_hba.sh + fi +} + +bind_wal2json() { + # Copy the file in the right place + cp /opt/sentry/wal2json/wal2json.so `pg_config --pkglibdir`/wal2json.so +} + +echo "Setting up Change Data Capture" + +prep_init_db +if [ "$1" = 'postgres' ]; then + cdc_setup_hba_conf + bind_wal2json +fi +exec /docker-entrypoint.sh "$@" diff --git a/scripts/bump-version.sh b/scripts/bump-version.sh index 4c8bb5a5d3..b5d4586c42 100755 --- a/scripts/bump-version.sh +++ b/scripts/bump-version.sh @@ -8,11 +8,14 @@ OLD_VERSION="$1" NEW_VERSION="$2" SYMBOLICATOR_VERSION=${SYMBOLICATOR_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/symbolicator/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} +WAL2JSON_VERSION=${WAL2JSON_VERSION:-$(curl -s "https://api.github.com/repos/getsentry/wal2json/releases/latest" | grep -Po '"tag_name": "\K.*?(?=")')} sed -i -e "s/^SYMBOLICATOR_IMAGE=\([^:]\+\):.\+\$/SYMBOLICATOR_IMAGE=\1:$SYMBOLICATOR_VERSION/" .env +sed -i -e "s/^WAL2JSON_VERSION=\([^:]\+\):.\+\$/WAL2JSON_VERSION=\1:$WAL2JSON_VERSION/" .env sed -i -e "s/^\(SENTRY\|SNUBA\|RELAY\)_IMAGE=\([^:]\+\):.\+\$/\1_IMAGE=\2:$NEW_VERSION/" .env sed -i -e "s/^\# Self-Hosted Sentry .*/# Self-Hosted Sentry $NEW_VERSION/" README.md sed -i -e "s/\(Change Date:\s*\)[-0-9]\+\$/\\1$(date +'%Y-%m-%d' -d '3 years')/" LICENSE echo "New version: $NEW_VERSION" echo "New Symbolicator version: $SYMBOLICATOR_VERSION" +echo "New wal2json version: $WAL2JSON_VERSION" From e20e6a3e5483ddfda6d3d9c07cb9ea53d3d22731 Mon Sep 17 00:00:00 2001 From: William Desportes Date: Sun, 30 May 2021 13:33:53 +0200 Subject: [PATCH 338/417] ci: Add .gitattributes file to exclude unnecessary files from repo zips (#973) Ref: #950 --- .gitattributes | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000000..3cc1aa5da8 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,6 @@ +/.gitattributes export-ignore +/.gitignore export-ignore +/.github export-ignore +/.editorconfig export-ignore +/.craft.yml export-ignore +/test.sh export-ignore From 97e4e375cd32419de499a9edd82a4920496c0992 Mon Sep 17 00:00:00 2001 From: Jan Michael Auer Date: Mon, 31 May 2021 13:44:51 +0200 Subject: [PATCH 339/417] fix(test): Export test script (#978) --- .gitattributes | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitattributes b/.gitattributes index 3cc1aa5da8..0beb4bb0ad 100644 --- a/.gitattributes +++ b/.gitattributes @@ -3,4 +3,3 @@ /.github export-ignore /.editorconfig export-ignore /.craft.yml export-ignore -/test.sh export-ignore From f4c309624538ca0ebce1ca5b0ab714f0b22d9921 Mon Sep 17 00:00:00 2001 From: William Desportes Date: Wed, 2 Jun 2021 20:53:07 +0200 Subject: [PATCH 340/417] feat: Add healthchecks for redis, memcached and postgres (#975) Ref: #950 --- docker-compose.yml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 9a78d96930..957b659686 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,9 +59,22 @@ services: memcached: <<: *restart_policy image: "memcached:1.5-alpine" + healthcheck: + # From: https://stackoverflow.com/a/31877626/5155484 + test: echo stats | nc 127.0.0.1 11211 + interval: 2s + timeout: 3s + retries: 30 + start_period: 3s redis: <<: *restart_policy image: "redis:5.0-alpine" + healthcheck: + test: redis-cli ping + interval: 2s + timeout: 3s + retries: 30 + start_period: 3s volumes: - "sentry-redis:/data" ulimits: @@ -71,6 +84,13 @@ services: postgres: <<: *restart_policy image: "postgres:9.6" + healthcheck: + # Using default user "postgres" from sentry/sentry.conf.example.py or value of POSTGRES_USER if provided + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] + interval: 2s + timeout: 3s + retries: 30 + start_period: 10s command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1"] environment: POSTGRES_HOST_AUTH_METHOD: "trust" From 659e03e9cf7f664d843815a3d57498ba7073c272 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 3 Jun 2021 16:22:01 +0300 Subject: [PATCH 341/417] ref(craft): Modernize and rearrange Craft config (#980) --- .craft.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.craft.yml b/.craft.yml index fcc29e73df..5ad85683e3 100644 --- a/.craft.yml +++ b/.craft.yml @@ -1,12 +1,6 @@ -minVersion: "0.14.0" -github: - owner: getsentry - repo: onpremise -releaseBranchPrefix: releases +minVersion: "0.23.1" changelogPolicy: none artifactProvider: name: none -statusProvider: - name: github targets: - name: github From 8d92667f4e62329788818a43db0d610db92d8575 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 7 Jun 2021 20:13:37 +0300 Subject: [PATCH 342/417] meta: Add some changelog (#984) @billyvg has a breaking change coming up soon so let's use this as an opportunity to add a changelog. This changelog will only capture important announcements (such as this breaking change over at Sentry) and changes in this repo for now. --- .craft.yml | 2 +- CHANGELOG.md | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 CHANGELOG.md diff --git a/.craft.yml b/.craft.yml index 5ad85683e3..4524afff10 100644 --- a/.craft.yml +++ b/.craft.yml @@ -1,5 +1,5 @@ minVersion: "0.23.1" -changelogPolicy: none +changelogPolicy: auto artifactProvider: name: none targets: diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000000..3a659c05b0 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,5 @@ +# Changelog + +## Unreleased + +- feat: Add healthchecks for redis, memcached and postgres (#975) From 0cfaa73b0586018c667b0c8cafa2c605a7bf29f5 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 9 Jun 2021 10:59:50 -0600 Subject: [PATCH 343/417] Make a reset button (#988) --- reset.sh | 62 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 reset.sh diff --git a/reset.sh b/reset.sh new file mode 100755 index 0000000000..fe5df7d9cb --- /dev/null +++ b/reset.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash + +# The purpose of this script is to make it easy to reset a local onpremise +# install to a clean state, optionally targeting a particular version. + +set -euo pipefail + +if [ -n "${DEBUG:-}" ]; then + set -x +fi + +cd "$(dirname $0)" + + +function confirm () { + read -p "$1 [y/n] " confirmation + if [ "$confirmation" != "y" ]; then + echo "Canceled. 😅" + exit + fi +} + + +# If we have a version given, validate it. +# ---------------------------------------- +# Note that arbitrary git refs won't work, because the *_IMAGE variables in +# .env will almost certainly point to :latest. Tagged releases are generally +# the only refs where these component versions are pinned, so enforce that +# we're targeting a valid tag here. Do this early in order to fail fast. + +version="${1:-}" +if [ -n "$version" ]; then + set +e + git rev-parse --verify --quiet "refs/tags/$version" > /dev/null + if [ $? -gt 0 ]; then + echo "Bad version: $version" + exit + fi + set -e +fi + +# Make sure they mean it. +confirm "☠️ Warning! 😳 This is highly destructive! 😱 Are you sure you wish to proceed?" +echo "Okay ... good luck! 😰" + +# Hit the reset button. +docker compose down --volumes --remove-orphans --rmi local + +# Remove any remaining (likely external) volumes with name matching 'sentry-.*'. +for volume in $(docker volume list --format '{{ .Name }}' | grep '^sentry-'); do + docker volume remove $volume > /dev/null \ + && echo "Removed volume: $volume" \ + || echo "Skipped volume: $volume" +done + +# If we have a version given, switch to it. +if [ -n "$version" ]; then + git checkout "$version" +fi + +# Install. +exec ./install.sh From d05d9a62290a3657ba526a22b2bb31f96438b3cc Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Tue, 15 Jun 2021 18:05:28 +0000 Subject: [PATCH 344/417] release: 21.6.0 --- .env | 8 ++++---- CHANGELOG.md | 3 ++- LICENSE | 2 +- README.md | 2 +- 4 files changed, 8 insertions(+), 7 deletions(-) diff --git a/.env b/.env index d49dae9c4f..fdd22a8341 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.6.0 +SNUBA_IMAGE=getsentry/snuba:21.6.0 +RELAY_IMAGE=getsentry/relay:21.6.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 3a659c05b0..b03f905d97 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,6 @@ # Changelog -## Unreleased +## 21.6.0 - feat: Add healthchecks for redis, memcached and postgres (#975) + diff --git a/LICENSE b/LICENSE index 67c62a8f95..d06ca6e2a7 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-05-19 +Change Date: 2024-06-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 53f99bd4b9..4e58e3a189 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.6.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 91f40486ca4095b33147e97d7bd1f58c3640b064 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 15 Jun 2021 22:21:26 +0300 Subject: [PATCH 345/417] fix(ci): Run bulids on `release` branches Follow up to #980 --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b01e3516d9..5c239dbd12 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -6,7 +6,7 @@ on: push: branches: - "master" - - "releases/**" + - "release/**" pull_request: env: DOCKER_COMPOSE_VERSION: 1.24.1 From b74416d3f0c7b71bf95c70da94ba6e56fd715874 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Tue, 15 Jun 2021 19:28:39 +0000 Subject: [PATCH 346/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index fdd22a8341..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.6.0 -SNUBA_IMAGE=getsentry/snuba:21.6.0 -RELAY_IMAGE=getsentry/relay:21.6.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest diff --git a/README.md b/README.md index 4e58e3a189..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.6.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From aea6adc5b870da83c9188f672ead1a4d459683fc Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Tue, 15 Jun 2021 19:48:13 +0000 Subject: [PATCH 347/417] release: 21.6.1 --- .env | 8 ++++---- CHANGELOG.md | 4 ++++ README.md | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.env b/.env index d49dae9c4f..086448270a 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.6.1 +SNUBA_IMAGE=getsentry/snuba:21.6.1 +RELAY_IMAGE=getsentry/relay:21.6.1 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest diff --git a/CHANGELOG.md b/CHANGELOG.md index b03f905d97..c43c383dbb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 21.6.1 + +- No documented changes. + ## 21.6.0 - feat: Add healthchecks for redis, memcached and postgres (#975) diff --git a/README.md b/README.md index 53f99bd4b9..c74466a9d4 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.6.1 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 1fae7759c01092a126a62ad7b70170559fa21df2 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Tue, 15 Jun 2021 19:56:17 +0000 Subject: [PATCH 348/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 086448270a..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.6.1 -SNUBA_IMAGE=getsentry/snuba:21.6.1 -RELAY_IMAGE=getsentry/relay:21.6.1 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest diff --git a/README.md b/README.md index c74466a9d4..53f99bd4b9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.6.1 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 198b2751f5ad9828d26cf039083747ba42084b51 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 15 Jun 2021 23:38:22 +0300 Subject: [PATCH 349/417] docs: Fold README into self-hosted docs (#993) Closes #983 and closes #790. Refs getsentry/develop#355. --- README.md | 58 ++++++++++++++----------------------------------------- 1 file changed, 15 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 53f99bd4b9..3c2c02b956 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements - * Docker 19.03.6+ - * Compose 1.24.1+ - * 4 CPU Cores - * 8 GB RAM - * 20 GB Free Disk Space +* Docker 19.03.6+ +* Compose 1.24.1+ +* 4 CPU Cores +* 8 GB RAM +* 20 GB Free Disk Space ## Setup @@ -16,19 +16,15 @@ To get started with all the defaults, simply clone the repo and run `./install.s During the install, a prompt will ask if you want to create a user account. If you require that the install not be blocked by the prompt, run `./install.sh --no-user-prompt`. -There may need to be modifications to the included example config files (`sentry/config.example.yml` and `sentry/sentry.conf.example.py`) to accommodate your needs or your environment (such as adding GitHub credentials). If you want to perform these, do them before you run the install script and copy them without the `.example` extensions in the name (such as `sentry/sentry.conf.py`) before running the `install.sh` script. +Please visit [our documentation](https://develop.sentry.dev/self-hosted/) for everything else. -The recommended way to customize your configuration is using the files below, in that order: +## Tips & Tricks - * `config.yml` - * `sentry.conf.py` - * `.env` w/ environment variables +### Event Retention -We currently support a very minimal set of environment variables to promote other means of configuration. - -If you have any issues or questions, our [Community Forum](https://forum.sentry.io/c/on-premise) is at your service! Everytime you run the install script, it will generate a log file, `sentry_install_log-.txt` with the output. Sharing these logs would help people diagnose any issues you might be having. +Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. -## Versioning +### Installing a specific SHA If you want to install a specific release of Sentry, use the tags/releases on this repo. @@ -43,49 +39,25 @@ Note that this may not work for all commit SHAs as this repository evolves with ### Using Linux If you are using Linux and you need to use `sudo` when running `./install.sh`, modifying the version of Sentry is slightly different. First, run the following: + ```shell sudo visudo ``` + Then add the following line: + ```shell Defaults env_keep += "SENTRY_IMAGE" ``` + Save the file then in your terminal run the following ```shell export SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 sudo ./install.sh ``` -Where you replace `83b1380` with the sha you want to use. - -## Event Retention - -Sentry comes with a cleanup cron job that prunes events older than `90 days` by default. If you want to change that, you can change the `SENTRY_EVENT_RETENTION_DAYS` environment variable in `.env` or simply override it in your environment. If you do not want the cleanup cron, you can remove the `sentry-cleanup` service from the `docker-compose.yml`file. - -## Securing Sentry with SSL/TLS - -If you'd like to protect your Sentry install with SSL/TLS, there are -fantastic SSL/TLS proxies like [HAProxy](http://www.haproxy.org/) -and [Nginx](http://nginx.org/). Our recommendation is running an external Nginx instance or your choice of load balancer that does the TLS termination and more. Read more over at our [productionalizing self-hosted docs](https://develop.sentry.dev/self-hosted/#productionalizing). - -## Updating Sentry - -_You need to be on at least Sentry 9.1.2 to be able to upgrade automatically to the latest version. If you are not, upgrade to 9.1.2 first by checking out the [9.1.2 tag](https://github.com/getsentry/onpremise/tree/9.1.2) on this repo._ - -We recommend (and sometimes require) you to upgrade Sentry one version at a time. That means if you are running 20.6.0, instead of going directly to 20.8.0, first go through 20.7.0. Skipping versions would work most of the time, but there will be times that we require you to stop at specific versions to ensure essential data migrations along the way. - -Pull the version of the repository that you wish to upgrade to by checking out the tagged release of this repo. Make sure to check for any difference between the example config files and your current config files in use. There might be new configuration that has to be added to your adjusted files such as feature flags or server configuration. - -The included `install.sh` script is meant to be idempotent and to bring you to the latest version. What this means is you can and should run `install.sh` to upgrade to the latest version available. Remember that the output of the script will be stored in a log file, `sentry_install_log-.txt`, which you may share for diagnosis if anything goes wrong. - -For more information regarding updating your Sentry installation, please visit [our documentation](https://develop.sentry.dev/self-hosted/#upgrading). - -## Resources - - * [Documentation](https://develop.sentry.dev/self-hosted/) - * [Bug Tracker](https://github.com/getsentry/onpremise/issues) - * [Community Forums](https://forum.sentry.io/c/on-premise) +Where you replace `83b1380` with the sha you want to use. [build-status-image]: https://github.com/getsentry/onpremise/workflows/test/badge.svg [build-status-url]: https://git.io/JUYkh From 1d49629229213371b9d019d2140ed69eeabd3b6f Mon Sep 17 00:00:00 2001 From: Marek Matys <57749215+thermaq@users.noreply.github.com> Date: Thu, 17 Jun 2021 13:34:32 +0200 Subject: [PATCH 350/417] Fixed typo (#996) --- reset.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reset.sh b/reset.sh index fe5df7d9cb..0e4db0e93e 100755 --- a/reset.sh +++ b/reset.sh @@ -44,7 +44,7 @@ confirm "☠️ Warning! 😳 This is highly destructive! 😱 Are you sure you echo "Okay ... good luck! 😰" # Hit the reset button. -docker compose down --volumes --remove-orphans --rmi local +docker-compose down --volumes --remove-orphans --rmi local # Remove any remaining (likely external) volumes with name matching 'sentry-.*'. for volume in $(docker volume list --format '{{ .Name }}' | grep '^sentry-'); do From f5411170ac0fb7f5ce8e7f5a1b21b16cbb176937 Mon Sep 17 00:00:00 2001 From: fmartinou Date: Fri, 25 Jun 2021 13:55:34 +0200 Subject: [PATCH 351/417] Update middlewares to latest stable versions (#1002) This PR is a try to update most middlewares used by Sentry to latest stable versions. [As mentioned in the forum](https://forum.sentry.io/t/middleware-version-compatibility/14353/2) I didn't update Postgresql & Clickhouse due to known issues. I also : - changed versions to immutable tags (MAJOR.MINOR.PATCH semver versions when possible). - changed nginx to the Alpine variant --- docker-compose.yml | 12 ++++++------ install/wrap-up.sh | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 957b659686..d0d1f94d31 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -58,7 +58,7 @@ services: - "sentry-smtp-log:/var/log/exim4" memcached: <<: *restart_policy - image: "memcached:1.5-alpine" + image: "memcached:1.6.9-alpine" healthcheck: # From: https://stackoverflow.com/a/31877626/5155484 test: echo stats | nc 127.0.0.1 11211 @@ -68,7 +68,7 @@ services: start_period: 3s redis: <<: *restart_policy - image: "redis:5.0-alpine" + image: "redis:6.2.4-alpine" healthcheck: test: redis-cli ping interval: 2s @@ -103,7 +103,7 @@ services: target: /opt/sentry/ zookeeper: <<: *restart_policy - image: "confluentinc/cp-zookeeper:5.5.0" + image: "confluentinc/cp-zookeeper:6.2.0" environment: ZOOKEEPER_CLIENT_PORT: "2181" CONFLUENT_SUPPORT_METRICS_ENABLE: "false" @@ -123,7 +123,7 @@ services: <<: *restart_policy depends_on: - zookeeper - image: "confluentinc/cp-kafka:5.5.0" + image: "confluentinc/cp-kafka:6.2.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" @@ -165,7 +165,7 @@ services: # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 geoipupdate: - image: "maxmindinc/geoipupdate:latest" + image: "maxmindinc/geoipupdate:v4.7.1" # Override the entrypoint in order to avoid using envvars for config. # Futz with settings so we can keep mmdb and conf in same dir on host # (image looks for them in separate dirs by default). @@ -272,7 +272,7 @@ services: <<: *restart_policy ports: - "$SENTRY_BIND:80/tcp" - image: "nginx:1.16" + image: "nginx:1.21.0-alpine" volumes: - type: bind read_only: true diff --git a/install/wrap-up.sh b/install/wrap-up.sh index 2671a3a513..bbe08fac17 100644 --- a/install/wrap-up.sh +++ b/install/wrap-up.sh @@ -3,7 +3,7 @@ if [[ "$MINIMIZE_DOWNTIME" ]]; then # Start the whole setup, except nginx and relay. $dc up -d --remove-orphans $($dc config --services | grep -v -E '^(nginx|relay)$') - $dc exec -T nginx service nginx reload + $dc exec -T nginx nginx -s reload docker run --rm --network="${COMPOSE_PROJECT_NAME}_default" alpine ash \ -c 'while [[ "$(wget -T 1 -q -O- http://web:9000/_health/)" != "ok" ]]; do sleep 0.5; done' From e25e36b55c6b97c555fbd3d4cb15d88a2974610e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20Jane=C4=8Dek?= Date: Mon, 28 Jun 2021 11:54:35 +0200 Subject: [PATCH 352/417] fix: raise healthcheck interval for redis, memcached and postgres (#1007) The 2s interval caused constantly high CPU usage. 30s interval with 3 retries is the Docker default and doesn't hurt the system that much. Fixes #1000 --- docker-compose.yml | 49 ++++++++++++++++++++++++++-------------------- 1 file changed, 28 insertions(+), 21 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d0d1f94d31..1a80780426 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,16 @@ version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped +x-healthcheck-defaults: &healthcheck_defaults + # Avoid setting the interval too small, as docker uses much more CPU than one would expect. + # Related issues: + # https://github.com/moby/moby/issues/39102 + # https://github.com/moby/moby/issues/39388 + # https://github.com/getsentry/onpremise/issues/1000 + interval: 30s + timeout: 5s + retries: 3 + start_period: 10s x-sentry-defaults: &sentry_defaults <<: *restart_policy image: "$SENTRY_IMAGE" @@ -60,21 +70,15 @@ services: <<: *restart_policy image: "memcached:1.6.9-alpine" healthcheck: + <<: *healthcheck_defaults # From: https://stackoverflow.com/a/31877626/5155484 test: echo stats | nc 127.0.0.1 11211 - interval: 2s - timeout: 3s - retries: 30 - start_period: 3s redis: <<: *restart_policy image: "redis:6.2.4-alpine" healthcheck: + <<: *healthcheck_defaults test: redis-cli ping - interval: 2s - timeout: 3s - retries: 30 - start_period: 3s volumes: - "sentry-redis:/data" ulimits: @@ -85,13 +89,19 @@ services: <<: *restart_policy image: "postgres:9.6" healthcheck: + <<: *healthcheck_defaults # Using default user "postgres" from sentry/sentry.conf.example.py or value of POSTGRES_USER if provided test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres}"] - interval: 2s - timeout: 3s - retries: 30 - start_period: 10s - command: ["postgres", "-c", "wal_level=logical", "-c", "max_replication_slots=1", "-c", "max_wal_senders=1"] + command: + [ + "postgres", + "-c", + "wal_level=logical", + "-c", + "max_replication_slots=1", + "-c", + "max_wal_senders=1", + ] environment: POSTGRES_HOST_AUTH_METHOD: "trust" entrypoint: /opt/sentry/postgres-entrypoint.sh @@ -115,10 +125,9 @@ services: - "sentry-zookeeper-log:/var/lib/zookeeper/log" - "sentry-secrets:/etc/zookeeper/secrets" healthcheck: - test: ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] - interval: 10s - timeout: 5s - retries: 6 + <<: *healthcheck_defaults + test: + ["CMD-SHELL", 'echo "ruok" | nc -w 2 -q 2 localhost 2181 | grep imok'] kafka: <<: *restart_policy depends_on: @@ -141,10 +150,8 @@ services: - "sentry-kafka-log:/var/lib/kafka/log" - "sentry-secrets:/etc/kafka/secrets" healthcheck: - test: ["CMD-SHELL", 'nc -z localhost 9092'] - interval: 10s - timeout: 5s - retries: 6 + <<: *healthcheck_defaults + test: ["CMD-SHELL", "nc -z localhost 9092"] clickhouse: <<: *restart_policy image: "yandex/clickhouse-server:20.3.9.70" From ef95b159fb60314f20ed4e055d063b4cede4374c Mon Sep 17 00:00:00 2001 From: Billy Vong Date: Tue, 29 Jun 2021 03:44:20 -0700 Subject: [PATCH 353/417] docs(): Add notice about breaking change for custom plugins (#985) This adds a breaking change notice to our changelog regarding custom plugins: The frontend bundle will be loaded asynchronously. This is a breaking change that can affect custom plugins that access certain globals in the django template. Please see https://forum.sentry.io/t/breaking-frontend-changes-for-custom-plugins/14184 for more information --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c43c383dbb..61482758ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- BREAKING CHANGE: The frontend bundle will be loaded asynchronously (via [#25744](https://github.com/getsentry/sentry/pull/25744)). This is a breaking change that can affect custom plugins that access certain globals in the django template. Please see https://forum.sentry.io/t/breaking-frontend-changes-for-custom-plugins/14184 for more information. + ## 21.6.1 - No documented changes. From 2888bf59592ebef5372a675601ea408a57e3c431 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 29 Jun 2021 14:52:55 -0600 Subject: [PATCH 354/417] Use the new GH issue forms (#1011) --- .github/ISSUE_TEMPLATE/bug.yml | 46 ++++++++++ .github/ISSUE_TEMPLATE/bug_report.md | 29 ------- .github/ISSUE_TEMPLATE/feature.yml | 28 ++++++ .github/ISSUE_TEMPLATE/feature_request.md | 18 ---- .github/workflows/validate-new-issue.yml | 100 ---------------------- 5 files changed, 74 insertions(+), 147 deletions(-) create mode 100644 .github/ISSUE_TEMPLATE/bug.yml delete mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature.yml delete mode 100644 .github/ISSUE_TEMPLATE/feature_request.md delete mode 100644 .github/workflows/validate-new-issue.yml diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml new file mode 100644 index 0000000000..3995c19baa --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -0,0 +1,46 @@ +name: 🐞 Bug Report +description: Tell us about something that's not working the way we (probably) intend. +body: + - type: input + id: version + attributes: + label: Version + placeholder: 21.7.0 ← should look like this (check the footer) + description: What version of self-hosted Sentry are you running? + validations: + required: false + - type: textarea + id: repro + attributes: + label: Steps to Reproduce + description: How can we see what you're seeing? Specific is terrific. + value: |- + 1. foo + 2. bar + 3. baz + validations: + required: true + - type: textarea + id: expected + attributes: + label: Expected Result + validations: + required: true + - type: textarea + id: actual + attributes: + label: Actual Result + description: Logs? Screenshots? Yes, please. + value: |- + e.g.: + - latest install logs: `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` + - `docker-compose logs` output + validations: + required: true + - type: markdown + attributes: + value: |- + ## Thanks 🙏 + Check our [triage docs](https://open.sentry.io/triage/) for what to expect next. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 359d8126a5..0000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: 🐞 Bug Report -about: Report a bug in Self-Hosted Sentry ---- - -### Version Information - -Version: *VERSION HERE* - -### Steps to Reproduce - -1. What -2. you -3. did. - -### Expected Result - -What you thought would happen. - -### Actual Result - -What actually happened. Maybe a screenshot/recording? - -### Logs - -What you saw along the way, e.g.: - -- latest install logs: `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` -- `docker-compose logs` output diff --git a/.github/ISSUE_TEMPLATE/feature.yml b/.github/ISSUE_TEMPLATE/feature.yml new file mode 100644 index 0000000000..49cfa3e1c5 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature.yml @@ -0,0 +1,28 @@ +name: 💡 Feature Request +description: Tell us about a problem our software could solve but doesn't. +body: + - type: textarea + id: problem + attributes: + label: Problem Statement + description: What problem could `onpremise` solve that it doesn't? + placeholder: |- + I want to make whirled peas, but `onpremise` doesn't blend. + validations: + required: true + - type: textarea + id: expected + attributes: + label: Solution Brainstorm + description: We know you have bright ideas to share ... share away, friend. + placeholder: |- + Add a blender to `onpremise`. + validations: + required: false + - type: markdown + attributes: + value: |- + ## Thanks 🙏 + Check our [triage docs](https://open.sentry.io/triage/) for what to expect next. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index 32e32f231f..0000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: 🧠 Feature Request -about: Suggest an idea for improving Self-Hosted Sentry ---- - -### Summary - -One paragraph description of the feature. - -### Motivation - -Why should this be worked on? What problems or use cases does it solve or -improve? - -### Additional Context - -Any other context or screenshots or API request payload/responses that -pertain to the feature. diff --git a/.github/workflows/validate-new-issue.yml b/.github/workflows/validate-new-issue.yml deleted file mode 100644 index 328e2e236c..0000000000 --- a/.github/workflows/validate-new-issue.yml +++ /dev/null @@ -1,100 +0,0 @@ -name: Validate new issue -on: - issues: - types: ["opened"] -jobs: - validate-new-issue: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: "Validate new issue" - shell: bash - env: - GITHUB_TOKEN: ${{ github.token }} - run: | - issue_number=${{ github.event.issue.number }} - echo "Validating issue #${issue_number}." - - # Trust users who belong to the getsentry org. - if gh api "https://api.github.com/orgs/getsentry/members/${{ github.actor }}" >/dev/null 2>&1; then - echo "Skipping validation, because ${{ github.actor }} is a member of the getsentry org." - exit 0 - else - echo "${{ github.actor }} is not a member of the getsentry org. 🧐" - fi - - # Helper - function gh-issue-label() { - gh api "/repos/:owner/:repo/issues/${1}/labels" \ - -X POST \ - --input <(echo "{\"labels\":[\"$2\"]}") - } - - # Prep reasons for error message comment. - REASON="your issue does not properly use one of this repo's available issue templates" - REASON_EXACT_MATCH="you created an issue from a template without filling in anything" - REASON_EMPTY="you created an empty issue" - - BASE_CASE_TITLE="validation bot is confused" - - # Definition of valid: - # - is a report about buggy validation 😅 or ... - # - not empty (ignoring whitespace) - # - matches a template - # - at least one of the headings are also in this issue - # - extra headings in the issue are fine - # - order doesn't matter - # - case-sensitive tho - # - not an *exact* match for a template (ignoring whitespace) - - jq -r .issue.title "$GITHUB_EVENT_PATH" > issue-title - if diff issue-title <(echo "$BASE_CASE_TITLE") > /dev/null; then - echo "Infinite recursion avoided." - exit 0 - fi - - function extract-headings { { sed 's/\r$//' "$1" | grep '^#' || echo -n ''; } | sort; } - jq -r .issue.body "$GITHUB_EVENT_PATH" > issue - if ! grep -q '[^[:space:]]' issue; then - REASON="${REASON_EMPTY}" - else - extract-headings <(cat issue) > headings-in-issue - for template in $(ls .github/ISSUE_TEMPLATE/*.md 2> /dev/null); do - # Strip front matter. https://stackoverflow.com/a/29292490/14946704 - sed -i'' '1{/^---$/!q;};1,/^---$/d' "$template" - extract-headings "$template" > headings-in-template - echo -n "$(basename $template)? " - if [ ! -s headings-in-template ]; then - echo "No headers in template. 🤷" - elif [ "$(comm -12 headings-in-template headings-in-issue)" ]; then - echo "Match! 👍 💃" - if diff -Bw "$template" issue > /dev/null; then - echo "... like, an /exact/ match. 😖" - REASON="${REASON_EXACT_MATCH}" - break - else - gh-issue-label "${issue_number}" "Status: Unrouted" - exit 0 - fi - else - echo "No match. 👎" - fi - done - fi - - # Failed validation! Close the issue with a comment and a label. - cat << EOF > comment - Sorry, friend. As far as this ol' bot can tell, ${REASON}. Please [try again](https://github.com/${{ github.repository }}/issues/new/choose), if you like. (And if I'm confused, please [let us know](https://github.com/getsentry/.github/issues/new?title=$(echo "$BASE_CASE_TITLE" | tr ' ' '+')&body=${{ github.event.issue.html_url }}). 😬) - - ---- - - [![Did you see the memo about this?](https://user-images.githubusercontent.com/134455/104515469-e04a9c80-55c0-11eb-8e15-ffe9c0b8dd7f.gif)](https://www.youtube.com/watch?v=Fy3rjQGc6lA) - - ([log](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})) - EOF - - echo -n "Commented: " - gh issue comment "${issue_number}" --body "$(cat comment)" - gh-issue-label "${issue_number}" "Status: Invalid" - gh issue close "${issue_number}" - echo "Closed with: \"${REASON}.\"" From b30e7ef94b3e5a4cbe04ff5c9c3d92aff75f93f7 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 30 Jun 2021 09:53:28 +0300 Subject: [PATCH 355/417] fix(templates): Bug issue template should use placeholder The bug issue template used `value` instead of `placeholder` for certain fields, leading to issue reports like #1012. This commit fixes that. --- .github/ISSUE_TEMPLATE/bug.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index 3995c19baa..e1b6936472 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -14,7 +14,7 @@ body: attributes: label: Steps to Reproduce description: How can we see what you're seeing? Specific is terrific. - value: |- + placeholder: |- 1. foo 2. bar 3. baz @@ -31,7 +31,7 @@ body: attributes: label: Actual Result description: Logs? Screenshots? Yes, please. - value: |- + placeholder: |- e.g.: - latest install logs: `ls -1 sentry_install_log-*.txt | tail -1 | xargs cat` - `docker-compose logs` output From a583fdc37bcecc1e95ceaf4ad308b84abb1fa718 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 1 Jul 2021 07:22:03 -0600 Subject: [PATCH 356/417] Require version on bug form (#1017) --- .github/ISSUE_TEMPLATE/bug.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/ISSUE_TEMPLATE/bug.yml b/.github/ISSUE_TEMPLATE/bug.yml index e1b6936472..b1ec358625 100644 --- a/.github/ISSUE_TEMPLATE/bug.yml +++ b/.github/ISSUE_TEMPLATE/bug.yml @@ -8,7 +8,7 @@ body: placeholder: 21.7.0 ← should look like this (check the footer) description: What version of self-hosted Sentry are you running? validations: - required: false + required: true - type: textarea id: repro attributes: From 3a412d7d945ef6da52a15e8c1cd5663ceae3eff5 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 2 Jul 2021 15:11:32 +0300 Subject: [PATCH 357/417] fix(kafka): Get Confluent images back to 5.5.0 (#1021) Fixes #1009 by partially reverting #1002. We need to make a 21.6.2 release soon and I didn't have time to dig into why Kafka upgrades were failing so reverting for safety for now. --- docker-compose.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 1a80780426..046702711a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -113,7 +113,7 @@ services: target: /opt/sentry/ zookeeper: <<: *restart_policy - image: "confluentinc/cp-zookeeper:6.2.0" + image: "confluentinc/cp-zookeeper:5.5.0" environment: ZOOKEEPER_CLIENT_PORT: "2181" CONFLUENT_SUPPORT_METRICS_ENABLE: "false" @@ -132,7 +132,7 @@ services: <<: *restart_policy depends_on: - zookeeper - image: "confluentinc/cp-kafka:6.2.0" + image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" From a2a99b910df95b7921b877a37c368f9221af69b4 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Fri, 2 Jul 2021 21:04:36 +0000 Subject: [PATCH 358/417] release: 21.6.2 --- .env | 8 ++++---- CHANGELOG.md | 2 +- LICENSE | 2 +- README.md | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index d49dae9c4f..d579a9e4aa 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.6.2 +SNUBA_IMAGE=getsentry/snuba:21.6.2 +RELAY_IMAGE=getsentry/relay:21.6.2 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 61482758ed..c87135246b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 21.6.2 - BREAKING CHANGE: The frontend bundle will be loaded asynchronously (via [#25744](https://github.com/getsentry/sentry/pull/25744)). This is a breaking change that can affect custom plugins that access certain globals in the django template. Please see https://forum.sentry.io/t/breaking-frontend-changes-for-custom-plugins/14184 for more information. diff --git a/LICENSE b/LICENSE index d06ca6e2a7..d6feac6a35 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-06-15 +Change Date: 2024-07-02 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 3c2c02b956..388bc425c9 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.6.2 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 66e1820087efc2a5a75ee43c35e35d3fa218d9fc Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Fri, 2 Jul 2021 21:12:11 +0000 Subject: [PATCH 359/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index d579a9e4aa..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.6.2 -SNUBA_IMAGE=getsentry/snuba:21.6.2 -RELAY_IMAGE=getsentry/relay:21.6.2 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest diff --git a/README.md b/README.md index 388bc425c9..3c2c02b956 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.6.2 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From d24a7b3ac6e925026f17a6beb1fd8903ed0987c5 Mon Sep 17 00:00:00 2001 From: Filippo Pacifici Date: Tue, 6 Jul 2021 02:31:12 -0700 Subject: [PATCH 360/417] fix(install) Make wal2json copy more robust (#1024) There is a potential conrner case where we may end up with the wal2json library in the `postgres/wal2json/VERSION/file` but not in `postgres/wal2json/wal2json.so`. Not sure exactly how likely this could be, but thechnically it is possible that the download succeeds and `cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE"` does not. The next attempt the copy would not be attempted. This fix ensures the copy always happens --- install/install-wal2json.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 5ed58005e0..2ff9c17741 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -27,8 +27,8 @@ if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then $DOCKER_CURL -L \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ > "../postgres/wal2json/$VERSION/$FILE_NAME" - - cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" -fi +fi +cp "../postgres/wal2json/$VERSION/$FILE_NAME" "$FILE_TO_USE" + echo "${_endgroup}" From bd8fbf960d403425c1617c14bf8f2eabb8ec50b8 Mon Sep 17 00:00:00 2001 From: Florian Kaiser Date: Wed, 7 Jul 2021 16:10:54 +0200 Subject: [PATCH 361/417] fix(wal2json): Respect http_proxy/https_proxy/no_proxy when installing (#1026) `http_proxy`, `https_proxy` and `no_proxy` environment variables should be forwarded to the curl container. --- install/install-wal2json.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 2ff9c17741..47e390ea6f 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -4,11 +4,15 @@ FILE_TO_USE="../postgres/wal2json/wal2json.so" ARCH=$(uname -m) FILE_NAME="wal2json-Linux-$ARCH-glibc.so" -DOCKER_CURL="docker run --rm curlimages/curl" +docker_curl() { + # The environment variables can be specified in lower case or upper case. + # The lower case version has precedence. http_proxy is an exception as it is only available in lower case. + docker run --rm -e http_proxy -e https_proxy -e HTTPS_PROXY -e no_proxy -e NO_PROXY curlimages/curl "$@" +} if [[ $WAL2JSON_VERSION == "latest" ]]; then VERSION=$( - $DOCKER_CURL https://api.github.com/repos/getsentry/wal2json/releases/latest | + docker_curl https://api.github.com/repos/getsentry/wal2json/releases/latest | grep '"tag_name":' | sed -E 's/.*"([^"]+)".*/\1/' ) @@ -24,7 +28,7 @@ fi mkdir -p ../postgres/wal2json if [ ! -f "../postgres/wal2json/$VERSION/$FILE_NAME" ]; then mkdir -p "../postgres/wal2json/$VERSION" - $DOCKER_CURL -L \ + docker_curl -L \ "https://github.com/getsentry/wal2json/releases/download/$VERSION/$FILE_NAME" \ > "../postgres/wal2json/$VERSION/$FILE_NAME" fi From 4da10694c2894edbd171704ef35b9300419ca649 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 8 Jul 2021 08:37:42 +0000 Subject: [PATCH 362/417] release: 21.6.3 --- .env | 8 ++++---- CHANGELOG.md | 4 ++++ LICENSE | 2 +- README.md | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.env b/.env index d49dae9c4f..f2e51309f7 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.6.3 +SNUBA_IMAGE=getsentry/snuba:21.6.3 +RELAY_IMAGE=getsentry/relay:21.6.3 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest diff --git a/CHANGELOG.md b/CHANGELOG.md index c87135246b..4929c52564 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 21.6.3 + +- No documented changes. + ## 21.6.2 - BREAKING CHANGE: The frontend bundle will be loaded asynchronously (via [#25744](https://github.com/getsentry/sentry/pull/25744)). This is a breaking change that can affect custom plugins that access certain globals in the django template. Please see https://forum.sentry.io/t/breaking-frontend-changes-for-custom-plugins/14184 for more information. diff --git a/LICENSE b/LICENSE index d6feac6a35..800483902f 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-07-02 +Change Date: 2024-07-08 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 3c2c02b956..79f75525a6 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.6.3 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 4282249638a10b21ac6b4092326e6cf30f06a29d Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 8 Jul 2021 08:45:06 +0000 Subject: [PATCH 363/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index f2e51309f7..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.6.3 -SNUBA_IMAGE=getsentry/snuba:21.6.3 -RELAY_IMAGE=getsentry/relay:21.6.3 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest diff --git a/README.md b/README.md index 79f75525a6..3c2c02b956 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.6.3 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From fae2293eca35a70481aee6a2798a105d13a7b06a Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 15 Jul 2021 18:04:09 +0000 Subject: [PATCH 364/417] release: 21.7.0 --- .env | 8 ++++---- CHANGELOG.md | 4 ++++ LICENSE | 2 +- README.md | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.env b/.env index d49dae9c4f..0e931e8508 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.7.0 +SNUBA_IMAGE=getsentry/snuba:21.7.0 +RELAY_IMAGE=getsentry/relay:21.7.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest diff --git a/CHANGELOG.md b/CHANGELOG.md index 4929c52564..f3f25f354d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## 21.7.0 + +- No documented changes. + ## 21.6.3 - No documented changes. diff --git a/LICENSE b/LICENSE index 800483902f..5bc19b3201 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-07-08 +Change Date: 2024-07-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 3c2c02b956..de350db17a 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.7.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 64aec777ceb47a151fefe428df5d351fb434d14d Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Thu, 15 Jul 2021 18:15:46 +0000 Subject: [PATCH 365/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 0e931e8508..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.7.0 -SNUBA_IMAGE=getsentry/snuba:21.7.0 -RELAY_IMAGE=getsentry/relay:21.7.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest diff --git a/README.md b/README.md index de350db17a..3c2c02b956 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.7.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 34812ce837b29f6ff204638b7fd24caeaddb411c Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 16 Jul 2021 12:56:56 -0600 Subject: [PATCH 366/417] Port test.sh to macOS (#1031) --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 26a9e99ac9..37d7af56ed 100755 --- a/test.sh +++ b/test.sh @@ -85,7 +85,7 @@ sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.u SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" && $4 ~ "^http" { print $4; exit; }') # We ignore the protocol and the host as we already know those -DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\?://\([0-9a-z]\+\)@[^/]\+/\([0-9]\+\)$|\1\n\2|p'`) +DSN_PIECES=(`echo $SENTRY_DSN | sed -ne 's|^https\{0,1\}://\([0-9a-z]\{1,\}\)@[^/]\{1,\}/\([0-9]\{1,\}\)$|\1 \2|p' | tr ' ' '\n'`) SENTRY_KEY=${DSN_PIECES[0]} PROJECT_ID=${DSN_PIECES[1]} From bd6f573aa4e5542276cc77acae63caa9e3f53085 Mon Sep 17 00:00:00 2001 From: ZHOU Cheng Date: Thu, 22 Jul 2021 03:50:16 +0800 Subject: [PATCH 367/417] Add mail.use-ssl and mail.list-namespace option in example (#1040) --- sentry/config.example.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 298929958f..951916c5e4 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -13,9 +13,15 @@ mail.host: 'smtp' # mail.username: '' # mail.password: '' # mail.use-tls: false +# mail.use-ssl: false # The email address to send on behalf of # mail.from: 'root@localhost' +# The mailing list namespace for emails sent by this Sentry server. +# This should be a domain you own (often the same domain as the domain +# part of the `mail.from` configuration parameter value) or `localhost`. +# mail.list-namespace: 'localhost' + # If you'd like to configure email replies, enable this. # mail.enable-replies: true From 17b675c83393cdb6d160f24a88b90b8eebc20193 Mon Sep 17 00:00:00 2001 From: Kyle Filz Date: Fri, 30 Jul 2021 10:39:47 -0500 Subject: [PATCH 368/417] feat: Support custom CA roots (#1015) Mount a certificate folder to local ca storage in containers, and add update command to cron image's entrypoint. Result of poking and prodding from getsentry/sentry#26851 --- .github/workflows/test.yml | 13 +++-- .gitignore | 7 +++ CHANGELOG.md | 4 ++ .../custom-ca-roots/docker-compose.test.yml | 12 +++++ .../custom-ca-roots/nginx/nginx.conf | 32 +++++++++++++ _integration-test/custom-ca-roots/setup.sh | 47 +++++++++++++++++++ _integration-test/custom-ca-roots/teardown.sh | 4 ++ _integration-test/custom-ca-roots/test.py | 15 ++++++ test.sh => _integration-test/run.sh | 11 ++++- cron/entrypoint.sh | 4 ++ docker-compose.yml | 9 ++++ install/_lib.sh | 4 +- sentry/entrypoint.sh | 4 ++ 13 files changed, 157 insertions(+), 9 deletions(-) create mode 100644 _integration-test/custom-ca-roots/docker-compose.test.yml create mode 100644 _integration-test/custom-ca-roots/nginx/nginx.conf create mode 100755 _integration-test/custom-ca-roots/setup.sh create mode 100755 _integration-test/custom-ca-roots/teardown.sh create mode 100644 _integration-test/custom-ca-roots/test.py rename test.sh => _integration-test/run.sh (94%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5c239dbd12..2a33ea1815 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,7 +15,7 @@ defaults: shell: bash jobs: unit-test: - runs-on: ubuntu-18.04 + runs-on: ubuntu-20.04 name: "unit tests" steps: - name: Checkout @@ -26,8 +26,8 @@ jobs: run: find ./ -type f -name "*-test.sh" -exec "./{}" \; integration-test: - runs-on: ubuntu-18.04 - name: "test" + runs-on: ubuntu-20.04 + name: "integration test" steps: - name: Pin docker-compose run: | @@ -42,13 +42,16 @@ jobs: - name: Integration Test run: | echo "Testing initial install" + # Create ./certificates here because install.sh will create it with root:root + # and then run.sh (-> setup.sh) won't be able to write to it. + mkdir certificates ./install.sh - ./test.sh + ./_integration-test/run.sh echo "Testing in-place upgrade" # Also test plugin installation here echo "sentry-auth-oidc" >> sentry/requirements.txt ./install.sh --minimize-downtime - ./test.sh + ./_integration-test/run.sh - name: Inspect failure if: failure() diff --git a/.gitignore b/.gitignore index 8a169049fa..c8967cdfd6 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,10 @@ geoip/.geoipupdate.lock # wal2json download postgres/wal2json + +# custom certificate authorities +certificates + +# integration testing +_integration-test/custom-ca-roots/nginx/* +sentry/test-custom-ca-roots.py diff --git a/CHANGELOG.md b/CHANGELOG.md index f3f25f354d..611babda86 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. + ## 21.7.0 - No documented changes. diff --git a/_integration-test/custom-ca-roots/docker-compose.test.yml b/_integration-test/custom-ca-roots/docker-compose.test.yml new file mode 100644 index 0000000000..2bc40ba1b1 --- /dev/null +++ b/_integration-test/custom-ca-roots/docker-compose.test.yml @@ -0,0 +1,12 @@ +version: '3.4' +services: + fixture-custom-ca-roots: + image: nginx:1.21.0-alpine + restart: unless-stopped + volumes: + - ./_integration-test/custom-ca-roots/nginx:/etc/nginx:ro + networks: + default: + aliases: + - self.test + - fail.test diff --git a/_integration-test/custom-ca-roots/nginx/nginx.conf b/_integration-test/custom-ca-roots/nginx/nginx.conf new file mode 100644 index 0000000000..517aea4102 --- /dev/null +++ b/_integration-test/custom-ca-roots/nginx/nginx.conf @@ -0,0 +1,32 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + server { + listen 443 ssl; + server_name "self.test"; + ssl_certificate "/etc/nginx/self.test.crt"; + ssl_certificate_key "/etc/nginx/self.test.key"; + location / { + add_header Content-Type text/plain; + return 200 'ok'; + } + } + server { + listen 443 ssl; + server_name "fake.test"; + ssl_certificate "/etc/nginx/fake.test.crt"; + ssl_certificate_key "/etc/nginx/fake.test.key"; + location / { + add_header Content-Type text/plain; + return 200 'bad'; + } + } +} diff --git a/_integration-test/custom-ca-roots/setup.sh b/_integration-test/custom-ca-roots/setup.sh new file mode 100755 index 0000000000..a8cb2f1615 --- /dev/null +++ b/_integration-test/custom-ca-roots/setup.sh @@ -0,0 +1,47 @@ +#! /usr/bin/env bash +set -e + +export COMPOSE_FILE="../docker-compose.yml:./custom-ca-roots/docker-compose.test.yml" + +TEST_NGINX_CONF_PATH="./custom-ca-roots/nginx" +CUSTOM_CERTS_PATH="../certificates" + +# generate tightly constrained CA +# NB: `-addext` requires LibreSSL 3.1.0+, or OpenSSL (brew install openssl) +openssl req -x509 -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/ca.key \ +-sha256 -days 1 -out $TEST_NGINX_CONF_PATH/ca.crt -batch \ +-subj "/CN=TEST CA *DO NOT TRUST*" \ +-addext "keyUsage = critical, keyCertSign, cRLSign" \ +-addext "nameConstraints = critical, permitted;DNS:self.test" + +## Lines like the following are debug helpers ... +# openssl x509 -in nginx/ca.crt -text -noout + +mkdir -p $CUSTOM_CERTS_PATH +cp $TEST_NGINX_CONF_PATH/ca.crt $CUSTOM_CERTS_PATH/test-custom-ca-roots.crt + +# generate server certificate +openssl req -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/self.test.key \ +-addext "subjectAltName=DNS:self.test" \ +-out $TEST_NGINX_CONF_PATH/self.test.req -batch -subj "/CN=Self Signed with CA Test Server" + +# openssl req -in nginx/self.test.req -text -noout + +openssl x509 -req -in $TEST_NGINX_CONF_PATH/self.test.req -CA $TEST_NGINX_CONF_PATH/ca.crt -CAkey $TEST_NGINX_CONF_PATH/ca.key \ +-extfile <(printf "subjectAltName=DNS:self.test") \ +-CAcreateserial -out $TEST_NGINX_CONF_PATH/self.test.crt -days 1 -sha256 + +# openssl x509 -in nginx/self.test.crt -text -noout + +# sanity check that signed certificate passes OpenSSL's validation +openssl verify -CAfile $TEST_NGINX_CONF_PATH/ca.crt $TEST_NGINX_CONF_PATH/self.test.crt + +# self signed certificate, for sanity check of not just accepting all certs +openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout $TEST_NGINX_CONF_PATH/fake.test.key \ +-out $TEST_NGINX_CONF_PATH/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server" + +# openssl x509 -in nginx/fake.test.crt -text -noout + +cp ./custom-ca-roots/test.py ../sentry/test-custom-ca-roots.py + +$dc up -d fixture-custom-ca-roots diff --git a/_integration-test/custom-ca-roots/teardown.sh b/_integration-test/custom-ca-roots/teardown.sh new file mode 100755 index 0000000000..059f69b93b --- /dev/null +++ b/_integration-test/custom-ca-roots/teardown.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +$dc rm -s -f -v fixture-custom-ca-roots +rm -f ../certificates/test-custom-ca-roots.crt ../sentry/test-custom-ca-roots.py +unset COMPOSE_FILE diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py new file mode 100644 index 0000000000..0f9b501f83 --- /dev/null +++ b/_integration-test/custom-ca-roots/test.py @@ -0,0 +1,15 @@ +import unittest +import requests + + +class CustomCATests(unittest.TestCase): + def test_valid_self_signed(self): + self.assertEqual(requests.get("https://self.test").text, 'ok') + + def test_invalid_self_signed(self): + with self.assertRaises(requests.exceptions.SSLError): + requests.get("https://fail.test") + + +if __name__ == '__main__': + unittest.main() diff --git a/test.sh b/_integration-test/run.sh similarity index 94% rename from test.sh rename to _integration-test/run.sh index 37d7af56ed..f25a302639 100755 --- a/test.sh +++ b/_integration-test/run.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -e -source "$(dirname $0)/install/_lib.sh" +source "$(dirname $0)/../install/_lib.sh" echo "${_group}Setting up variables and helpers ..." export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" @@ -42,6 +42,7 @@ echo 'SENTRY_BEACON=False' >> $SENTRY_CONFIG_PY $dcr web createuser --superuser --email $TEST_USER --password $TEST_PASS || true $dc up -d printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done' +echo "" echo "${_endgroup}" echo "${_group}Running tests ..." @@ -99,7 +100,7 @@ export -f sentry_api_request get_csrf_token export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH printf "Getting the test event back" timeout 30 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' -echo ""; +echo " got it!"; EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH") declare -a EVENT_TEST_STRINGS=( @@ -119,3 +120,9 @@ echo "${_endgroup}" echo "${_group}Ensure cleanup crons are working ..." $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+" echo "${_endgroup}" + +echo "${_group}Test custom CAs work ..." +source ./custom-ca-roots/setup.sh +$dcr --no-deps web python3 /etc/sentry/test-custom-ca-roots.py +source ./custom-ca-roots/teardown.sh +echo "${_endgroup}" diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh index baa833a77b..383c8b29c7 100755 --- a/cron/entrypoint.sh +++ b/cron/entrypoint.sh @@ -1,5 +1,9 @@ #!/usr/bin/env bash +if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then + update-ca-certificates +fi + # Prior art: # - https://git.io/fjNOg # - https://blog.knoldus.com/running-a-cron-job-in-docker-container/ diff --git a/docker-compose.yml b/docker-compose.yml index 046702711a..86f3ed44be 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -35,6 +35,14 @@ x-sentry-defaults: &sentry_defaults PYTHONUSERBASE: "/data/custom-packages" SENTRY_CONF: "/etc/sentry" SNUBA: "http://snuba-api:1218" + # Force everything to use the system CA bundle + # This is mostly needed to support installing custom CA certs + # This one is used by botocore + DEFAULT_CA_BUNDLE: &ca_bundle "/etc/ssl/certs/ca-certificates.crt" + # This one is used by requests + REQUESTS_CA_BUNDLE: *ca_bundle + # This one is used by grpc/google modules + GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: @@ -42,6 +50,7 @@ x-sentry-defaults: &sentry_defaults - "sentry-data:/data" - "./sentry:/etc/sentry" - "./geoip:/geoip:ro" + - "./certificates:/usr/local/share/ca-certificates:ro" x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: diff --git a/install/_lib.sh b/install/_lib.sh index 2d7517fdc6..0b5417f456 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -6,10 +6,10 @@ log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt" exec &> >(tee -a "$log_file") # Work from /install/ for install.sh, project root otherwise -if [[ "$(basename $0)" = "install.sh" || "$(basename $0)" = "test.sh" ]]; then +if [[ "$(basename $0)" = "install.sh" ]]; then cd "$(dirname $0)/install/" else - cd "$(dirname $0)" # assume we're a *-test.sh script + cd "$(dirname $0)" # assume we're a test script or some such fi _ENV="$(realpath ../.env)" diff --git a/sentry/entrypoint.sh b/sentry/entrypoint.sh index 55c7e4141a..2f2614a798 100755 --- a/sentry/entrypoint.sh +++ b/sentry/entrypoint.sh @@ -1,6 +1,10 @@ #!/bin/bash set -e +if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then + update-ca-certificates +fi + req_file="/etc/sentry/requirements.txt" plugins_dir="/data/custom-packages" checksum_file="$plugins_dir/.checksum" From 39ea4dcec35e9cb73999e8f1686cd39cdb759ca7 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 30 Jul 2021 20:45:49 +0300 Subject: [PATCH 369/417] fix: Bring back test.sh relied upon by other repos --- test.sh | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 test.sh diff --git a/test.sh b/test.sh new file mode 100755 index 0000000000..85945dc3c7 --- /dev/null +++ b/test.sh @@ -0,0 +1,4 @@ +#!/usr/bin/env bash +set -e + +./_integration-test/run.sh \ No newline at end of file From e17faecd1092d49458f4db1cab6f498de65ca3b1 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 4 Aug 2021 22:36:48 +0300 Subject: [PATCH 370/417] fix: Fix `curl` image to version 7.77.0 (#1049) Turns out the latest, `7.78.0` may have issues with DNS resolution from time to time (I experienced this locally). It is also a good practice to fix it to a specific version. --- CHANGELOG.md | 1 + install/install-wal2json.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 611babda86..bd03a88950 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## Unreleased - feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. +- fix: Fix `curl` image to version 7.77.0 ## 21.7.0 diff --git a/install/install-wal2json.sh b/install/install-wal2json.sh index 47e390ea6f..2973c0034d 100644 --- a/install/install-wal2json.sh +++ b/install/install-wal2json.sh @@ -7,7 +7,7 @@ FILE_NAME="wal2json-Linux-$ARCH-glibc.so" docker_curl() { # The environment variables can be specified in lower case or upper case. # The lower case version has precedence. http_proxy is an exception as it is only available in lower case. - docker run --rm -e http_proxy -e https_proxy -e HTTPS_PROXY -e no_proxy -e NO_PROXY curlimages/curl "$@" + docker run --rm -e http_proxy -e https_proxy -e HTTPS_PROXY -e no_proxy -e NO_PROXY curlimages/curl:7.77.0 "$@" } if [[ $WAL2JSON_VERSION == "latest" ]]; then From 93078f052d2d31128a5ab463c19c182ea1cedada Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 4 Aug 2021 22:44:35 +0300 Subject: [PATCH 371/417] upgrade: docker-compose version to 1.29.2 (#1050) --- .github/workflows/test.yml | 2 +- CHANGELOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 2a33ea1815..282537ade9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -9,7 +9,7 @@ on: - "release/**" pull_request: env: - DOCKER_COMPOSE_VERSION: 1.24.1 + DOCKER_COMPOSE_VERSION: 1.29.2 defaults: run: shell: bash diff --git a/CHANGELOG.md b/CHANGELOG.md index bd03a88950..31d829c830 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. - fix: Fix `curl` image to version 7.77.0 +- upgrade: docker-compose version to 1.29.2 ## 21.7.0 @@ -24,4 +25,3 @@ ## 21.6.0 - feat: Add healthchecks for redis, memcached and postgres (#975) - From 76c33bbbe7f37a3ae37519f00f906cd8d49c8b06 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 5 Aug 2021 13:06:24 +0300 Subject: [PATCH 372/417] upgrade: use --ansi never for docker-compose (#1051) Starting from `docker-compose` v1.28.0, the `--no-ansi` option is deprecated and a new, `--ansi never` option is introduced instead. This PR makes the deprecation warnings around this go away but bumps the minimum docker-compose version required to `1.28.0` as the older versions don't support the new option. --- README.md | 2 +- install/_lib.sh | 2 +- install/check-minimum-requirements.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3c2c02b956..6c4874d19e 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docke ## Requirements * Docker 19.03.6+ -* Compose 1.24.1+ +* Compose 1.28.0+ * 4 CPU Cores * 8 GB RAM * 20 GB Free Disk Space diff --git a/install/_lib.sh b/install/_lib.sh index 0b5417f456..7b89461fa2 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -25,7 +25,7 @@ else _endgroup="" fi -dc="docker-compose --no-ansi" +dc="docker-compose --ansi never" dcr="$dc run --rm" # A couple of the config files are referenced from other subscripts, so they diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh index 4527a222ef..b3804df787 100644 --- a/install/check-minimum-requirements.sh +++ b/install/check-minimum-requirements.sh @@ -1,7 +1,7 @@ echo "${_group}Checking minimum requirements ..." MIN_DOCKER_VERSION='19.03.6' -MIN_COMPOSE_VERSION='1.24.1' +MIN_COMPOSE_VERSION='1.28.0' MIN_RAM_HARD=3800 # MB MIN_RAM_SOFT=7800 # MB MIN_CPU_HARD=2 From 5d695614842d7e1282b53a8abebc1b8ad92de9d4 Mon Sep 17 00:00:00 2001 From: josh Date: Thu, 5 Aug 2021 10:07:45 +0000 Subject: [PATCH 373/417] fix(tests): Django 2.0 and 2.1 compatible csrf token extraction (#1052) --- _integration-test/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/_integration-test/run.sh b/_integration-test/run.sh index f25a302639..dc13de40d8 100755 --- a/_integration-test/run.sh +++ b/_integration-test/run.sh @@ -57,7 +57,7 @@ login () { exit -1 fi - CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' + CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "['\"]" ' /csrfmiddlewaretoken/ { print $4 "=" $6; exit; From d11fb14ceecb92078cf59449a93e1fb02e8b101f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Florian=20Fu=C3=9F?= <5619511+pharindoko@users.noreply.github.com> Date: Thu, 5 Aug 2021 12:13:05 +0200 Subject: [PATCH 374/417] fix(snuba-api): wait for clickhouse to be healthy (#1053) To avoid a race condition in install.sh process at step https://github.com/getsentry/onpremise/blob/7e7401a668987c44863c403548ba3dbf91c701da/install/bootstrap-snuba.sh#L4 Closes #1033 --- docker-compose.yml | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 86f3ed44be..78c7547e13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -54,9 +54,12 @@ x-sentry-defaults: &sentry_defaults x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: - - redis - - clickhouse - - kafka + clickhouse: + condition: service_healthy + kafka: + condition: service_healthy + redis: + condition: service_healthy image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker @@ -180,6 +183,11 @@ services: # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 + healthcheck: + test: ["CMD-SHELL", "wget -nv -t1 --spider 'http://localhost:9000/' || exit 1"] + interval: 3s + timeout: 600s + retries: 200 geoipupdate: image: "maxmindinc/geoipupdate:v4.7.1" # Override the entrypoint in order to avoid using envvars for config. From 9aa34dd6120fb51d005fa42cac1c1fff1f044eb4 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 16:20:01 +0300 Subject: [PATCH 375/417] feat: Leverage health checks for depends_on (#1057) Expands the work on #1053 to all applicable services. --- CHANGELOG.md | 1 + docker-compose.yml | 78 +++++++++++++++++++++++++++++++++------------- 2 files changed, 57 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d829c830..edf49c7de4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. - fix: Fix `curl` image to version 7.77.0 - upgrade: docker-compose version to 1.29.2 +- feat: Leverage health checks for depends_on ## 21.7.0 diff --git a/docker-compose.yml b/docker-compose.yml index 78c7547e13..63b76619e6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,10 @@ version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped +x-depends_on-healthy: &depends_on-healthy + condition: service_healthy +x-depends_on-default: &depends_on-default + condition: service_started x-healthcheck-defaults: &healthcheck_defaults # Avoid setting the interval too small, as docker uses much more CPU than one would expect. # Related issues: @@ -15,20 +19,34 @@ x-sentry-defaults: &sentry_defaults <<: *restart_policy image: "$SENTRY_IMAGE" depends_on: - - redis - - postgres - - memcached - - smtp - - snuba-api - - snuba-consumer - - snuba-outcomes-consumer - - snuba-sessions-consumer - - snuba-transactions-consumer - - snuba-subscription-consumer-events - - snuba-subscription-consumer-transactions - - snuba-replacer - - symbolicator - - kafka + redis: + <<: *depends_on-healthy + kafka: + <<: *depends_on-healthy + postgres: + <<: *depends_on-healthy + memcached: + <<: *depends_on-default + smtp: + <<: *depends_on-default + snuba-api: + <<: *depends_on-default + snuba-consumer: + <<: *depends_on-default + snuba-outcomes-consumer: + <<: *depends_on-default + snuba-sessions-consumer: + <<: *depends_on-default + snuba-transactions-consumer: + <<: *depends_on-default + snuba-subscription-consumer-events: + <<: *depends_on-default + snuba-subscription-consumer-transactions: + <<: *depends_on-default + snuba-replacer: + <<: *depends_on-default + symbolicator: + <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: @@ -55,11 +73,11 @@ x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: - condition: service_healthy + <<: *depends_on-healthy kafka: - condition: service_healthy + <<: *depends_on-healthy redis: - condition: service_healthy + <<: *depends_on-healthy image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker @@ -143,7 +161,8 @@ services: kafka: <<: *restart_policy depends_on: - - zookeeper + zookeeper: + <<: *depends_on-healthy image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" @@ -184,7 +203,11 @@ services: # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 healthcheck: - test: ["CMD-SHELL", "wget -nv -t1 --spider 'http://localhost:9000/' || exit 1"] + test: + [ + "CMD-SHELL", + "wget -nv -t1 --spider 'http://localhost:9000/' || exit 1", + ] interval: 3s timeout: 600s retries: 200 @@ -264,6 +287,14 @@ services: - "sentry-symbolicator:/data" web: <<: *sentry_defaults + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/bin/bash" + - '-c' + # Courtesy of https://unix.stackexchange.com/a/234089/108960 + - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' cron: <<: *sentry_defaults command: run cron @@ -318,9 +349,12 @@ services: source: ./geoip target: /geoip depends_on: - - kafka - - redis - - web + kafka: + <<: *depends_on-healthy + redis: + <<: *depends_on-healthy + web: + <<: *depends_on-healthy volumes: sentry-data: external: true From 9327228a04e5a11fea6d641bcedcaf1165540a72 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Sun, 15 Aug 2021 18:03:59 +0000 Subject: [PATCH 376/417] release: 21.8.0 --- .env | 8 ++++---- CHANGELOG.md | 2 +- LICENSE | 2 +- README.md | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index d49dae9c4f..f6fd53349c 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.8.0 +SNUBA_IMAGE=getsentry/snuba:21.8.0 +RELAY_IMAGE=getsentry/relay:21.8.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest diff --git a/CHANGELOG.md b/CHANGELOG.md index edf49c7de4..f388b8732e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 21.8.0 - feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. - fix: Fix `curl` image to version 7.77.0 diff --git a/LICENSE b/LICENSE index 5bc19b3201..a55050d90c 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-07-15 +Change Date: 2024-08-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 6c4874d19e..5e2c66f250 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.8.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 42aa60e42fa30d3cdfa59898fc203588f22a111e Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Sun, 15 Aug 2021 18:15:39 +0000 Subject: [PATCH 377/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index f6fd53349c..d49dae9c4f 100644 --- a/.env +++ b/.env @@ -3,8 +3,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 -SENTRY_IMAGE=getsentry/sentry:21.8.0 -SNUBA_IMAGE=getsentry/snuba:21.8.0 -RELAY_IMAGE=getsentry/relay:21.8.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest diff --git a/README.md b/README.md index 5e2c66f250..6c4874d19e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.8.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 47c4a1df299fffb8340160d9633c05a6c8f75adc Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Tue, 17 Aug 2021 08:31:09 -0400 Subject: [PATCH 378/417] meta(gha): Deploy action issue-routing-helper.yml (#1064) --- .github/workflows/issue-routing-helper.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/issue-routing-helper.yml b/.github/workflows/issue-routing-helper.yml index 22e381c32e..a36179554e 100644 --- a/.github/workflows/issue-routing-helper.yml +++ b/.github/workflows/issue-routing-helper.yml @@ -10,6 +10,8 @@ jobs: route: runs-on: ubuntu-latest if: >- + github.event.issue.state == 'open' + && startsWith(github.event.label.name, 'Team: ') && !contains(github.event.issue.labels.*.name, 'Status: Backlog') From 672b1f7f4d228d212ea0ad06dbc649b2e0dc2250 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 17 Aug 2021 17:40:35 +0300 Subject: [PATCH 379/417] feat: Have an empty certificates folder ready (#1065) Addresses this: https://forum.sentry.io/t/configure-gitlab-self-signed-ssl-certificate/14766/3?u=byk --- .github/workflows/test.yml | 3 --- .gitignore | 3 --- certificates/.gitignore | 3 +++ 3 files changed, 3 insertions(+), 6 deletions(-) create mode 100644 certificates/.gitignore diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 282537ade9..039aa0e82e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -42,9 +42,6 @@ jobs: - name: Integration Test run: | echo "Testing initial install" - # Create ./certificates here because install.sh will create it with root:root - # and then run.sh (-> setup.sh) won't be able to write to it. - mkdir certificates ./install.sh ./_integration-test/run.sh echo "Testing in-place upgrade" diff --git a/.gitignore b/.gitignore index c8967cdfd6..4a86daa0ee 100644 --- a/.gitignore +++ b/.gitignore @@ -88,9 +88,6 @@ geoip/.geoipupdate.lock # wal2json download postgres/wal2json -# custom certificate authorities -certificates - # integration testing _integration-test/custom-ca-roots/nginx/* sentry/test-custom-ca-roots.py diff --git a/certificates/.gitignore b/certificates/.gitignore new file mode 100644 index 0000000000..30d0607b1f --- /dev/null +++ b/certificates/.gitignore @@ -0,0 +1,3 @@ +# Add all custom CAs in this folder +* +!.gitignore From 497759c1e83280f43cc481735f37eabd37873a13 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 18 Aug 2021 18:35:21 +0300 Subject: [PATCH 380/417] ci: Test with the required minimum docker-compose (#1066) Refs #1061, refs #1062, refs #1063. --- .github/workflows/test.yml | 17 ++++++++--------- install/_min-requirements.sh | 7 +++++++ install/check-minimum-requirements.sh | 7 +------ 3 files changed, 16 insertions(+), 15 deletions(-) create mode 100644 install/_min-requirements.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 039aa0e82e..1c5a938588 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,8 +8,6 @@ on: - "master" - "release/**" pull_request: -env: - DOCKER_COMPOSE_VERSION: 1.29.2 defaults: run: shell: bash @@ -29,15 +27,16 @@ jobs: runs-on: ubuntu-20.04 name: "integration test" steps: - - name: Pin docker-compose - run: | - sudo rm /usr/local/bin/docker-compose - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose - chmod +x docker-compose - sudo mv docker-compose /usr/local/bin - - name: Checkout uses: actions/checkout@v2 + + - name: Pin docker-compose + run: | + COMPOSE_PATH=/usr/local/bin/docker-compose + source ./install/_min-requirements.sh + sudo rm $COMPOSE_PATH + sudo curl -L https://github.com/docker/compose/releases/download/${MIN_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o $COMPOSE_PATH + sudo chmod +x $COMPOSE_PATH - name: Integration Test run: | diff --git a/install/_min-requirements.sh b/install/_min-requirements.sh new file mode 100644 index 0000000000..fb1b8a1a00 --- /dev/null +++ b/install/_min-requirements.sh @@ -0,0 +1,7 @@ +# Don't forget to update the README and othes docs when you change these! +MIN_DOCKER_VERSION='19.03.6' +MIN_COMPOSE_VERSION='1.28.0' +MIN_RAM_HARD=3800 # MB +MIN_RAM_SOFT=7800 # MB +MIN_CPU_HARD=2 +MIN_CPU_SOFT=4 diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh index b3804df787..9f3b43b355 100644 --- a/install/check-minimum-requirements.sh +++ b/install/check-minimum-requirements.sh @@ -1,11 +1,6 @@ echo "${_group}Checking minimum requirements ..." -MIN_DOCKER_VERSION='19.03.6' -MIN_COMPOSE_VERSION='1.28.0' -MIN_RAM_HARD=3800 # MB -MIN_RAM_SOFT=7800 # MB -MIN_CPU_HARD=2 -MIN_CPU_SOFT=4 +source "$(dirname $0)/_min-requirements.sh" DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') From 7b96408734c932fd347656979c40c47c30056fee Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Wed, 18 Aug 2021 23:43:39 +0300 Subject: [PATCH 381/417] fix(requirements): Make compose version check bw-compatible (#1068) Refs #1062. --- install/check-minimum-requirements.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh index 9f3b43b355..149b7d91da 100644 --- a/install/check-minimum-requirements.sh +++ b/install/check-minimum-requirements.sh @@ -3,7 +3,8 @@ echo "${_group}Checking minimum requirements ..." source "$(dirname $0)/_min-requirements.sh" DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') -COMPOSE_VERSION=$($dc --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') +# Do NOT use $dc instead of `docker-compose` below as older versions don't support certain options and fail +COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); From 59c0df393f8b9158486baf6210305503b2792fcd Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 19 Aug 2021 12:21:18 +0300 Subject: [PATCH 382/417] fix(clickhouse): Use correct HTTP port for healthcheck (#1069) Should fix #1058 --- CHANGELOG.md | 5 +++++ docker-compose.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f388b8732e..2329abfa44 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # Changelog +## Unreleased + +- fix(clickhouse): Use correct HTTP port for healthcheck (#1069) + Fixes the regular `Unexpected packet` errors in Clickhouse + ## 21.8.0 - feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. diff --git a/docker-compose.yml b/docker-compose.yml index 63b76619e6..6b1a4b9972 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,7 +206,7 @@ services: test: [ "CMD-SHELL", - "wget -nv -t1 --spider 'http://localhost:9000/' || exit 1", + "wget -nv -t1 --spider 'http://localhost:8123/' || exit 1", ] interval: 3s timeout: 600s From 4b5ab5fb2119864e4882e4c4e81ac8982254f05c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 19 Aug 2021 16:01:16 +0300 Subject: [PATCH 383/417] docs(changelog): Add missing entries for docker-compose changes (#1071) Refs #1066 and #1068 --- CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2329abfa44..a9d4dd5bb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,9 @@ ## Unreleased +- fix(requirements): Make compose version check bw-compatible (#1068) +- ci: Test with the required minimum docker-compose (#1066) + Run tests using docker-compose `1.28.0` instead of latest - fix(clickhouse): Use correct HTTP port for healthcheck (#1069) Fixes the regular `Unexpected packet` errors in Clickhouse From 610f7136bc76b2dcaf1fbe2b83819eb3809e8c09 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 19 Aug 2021 19:12:09 +0300 Subject: [PATCH 384/417] fix(healthcheck): Increase retries to 5 (#1072) Fixes #1070. --- CHANGELOG.md | 1 + docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a9d4dd5bb7..4798f50f0a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## Unreleased +- fix(healthcheck): Increase retries to 5 (#1072) - fix(requirements): Make compose version check bw-compatible (#1068) - ci: Test with the required minimum docker-compose (#1066) Run tests using docker-compose `1.28.0` instead of latest diff --git a/docker-compose.yml b/docker-compose.yml index 6b1a4b9972..008bb14a02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ x-healthcheck-defaults: &healthcheck_defaults # https://github.com/getsentry/onpremise/issues/1000 interval: 30s timeout: 5s - retries: 3 + retries: 5 start_period: 10s x-sentry-defaults: &sentry_defaults <<: *restart_policy From 674a600770acad37ab8560e323e38ead49b0cf97 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Tue, 24 Aug 2021 16:26:52 +0300 Subject: [PATCH 385/417] fix(compose): Drop version from compose file (#1074) We've switched to using the [compose spec](https://github.com/compose-spec/compose-spec/blob/master/spec.md) with the recent upgrades and health-check related upgrades anyway so drop the incorrect and confusing compose file version. --- docker-compose.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 008bb14a02..c5d9ec77d2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped x-depends_on-healthy: &depends_on-healthy From b32de84d31dd1cc0b9c9534fae2e95ad86a0cbee Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Thu, 26 Aug 2021 17:40:22 +0300 Subject: [PATCH 386/417] feat(smtp): Add hostname to SMTP (#1076) Fixes #1045. Co-authored-by: William Desportes Co-authored-by: Chad Whitacre --- .env | 4 +++- _integration-test/run.sh | 2 +- docker-compose.yml | 2 ++ sentry/config.example.yml | 6 ++++++ sentry/sentry.conf.example.py | 7 +++++++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/.env b/.env index d49dae9c4f..a7830cfc4c 100644 --- a/.env +++ b/.env @@ -3,8 +3,10 @@ SENTRY_EVENT_RETENTION_DAYS=90 # You can either use a port number or an IP:PORT combo for SENTRY_BIND # See https://docs.docker.com/compose/compose-file/#ports for more SENTRY_BIND=9000 +# Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails! +# SENTRY_MAIL_HOST=example.com SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly -WAL2JSON_VERSION=latest +WAL2JSON_VERSION=latest \ No newline at end of file diff --git a/_integration-test/run.sh b/_integration-test/run.sh index dc13de40d8..6b66052bf5 100755 --- a/_integration-test/run.sh +++ b/_integration-test/run.sh @@ -82,7 +82,7 @@ echo "${_endgroup}" echo "${_group}Running moar tests !!!" # Set up initial/required settings (InstallWizard request) -sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null +sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" && $4 ~ "^http" { print $4; exit; }') # We ignore the protocol and the host as we already know those diff --git a/docker-compose.yml b/docker-compose.yml index c5d9ec77d2..6314f8803d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -63,6 +63,7 @@ x-sentry-defaults: &sentry_defaults # Leaving the value empty to just pass whatever is set # on the host system (or in the .env file) SENTRY_EVENT_RETENTION_DAYS: + SENTRY_MAIL_HOST: volumes: - "sentry-data:/data" - "./sentry:/etc/sentry" @@ -92,6 +93,7 @@ services: smtp: <<: *restart_policy image: tianon/exim4 + hostname: ${SENTRY_MAIL_HOST:-} volumes: - "sentry-smtp:/var/spool/exim4" - "sentry-smtp-log:/var/log/exim4" diff --git a/sentry/config.example.yml b/sentry/config.example.yml index 951916c5e4..22a236ab1f 100644 --- a/sentry/config.example.yml +++ b/sentry/config.example.yml @@ -14,6 +14,12 @@ mail.host: 'smtp' # mail.password: '' # mail.use-tls: false # mail.use-ssl: false + +# NOTE: The following 2 configs (mail.from and mail.list-namespace) are set +# through SENTRY_MAIL_HOST in sentry.conf.py so remove those first if +# you want your values in this file to be effective! + + # The email address to send on behalf of # mail.from: 'root@localhost' diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 2d89b5677f..355ae63363 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -228,6 +228,13 @@ def get_internal_network(): # End of SSL/TLS settings +######## +# Mail # +######## + +SENTRY_OPTIONS["mail.list-namespace"] = env('SENTRY_MAIL_HOST', 'localhost') +SENTRY_OPTIONS["mail.from"] = f"sentry@{SENTRY_OPTIONS['mail.list-namespace']}" + ############ # Features # ############ From d571da2c162244e99ede06b9bf1bda835283ca56 Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Wed, 15 Sep 2021 18:04:26 +0000 Subject: [PATCH 387/417] release: 21.9.0 --- .env | 8 ++++---- CHANGELOG.md | 2 +- LICENSE | 2 +- README.md | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.env b/.env index a7830cfc4c..aad71292e9 100644 --- a/.env +++ b/.env @@ -5,8 +5,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 SENTRY_BIND=9000 # Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails! # SENTRY_MAIL_HOST=example.com -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.9.0 +SNUBA_IMAGE=getsentry/snuba:21.9.0 +RELAY_IMAGE=getsentry/relay:21.9.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 4798f50f0a..57f37add7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # Changelog -## Unreleased +## 21.9.0 - fix(healthcheck): Increase retries to 5 (#1072) - fix(requirements): Make compose version check bw-compatible (#1068) diff --git a/LICENSE b/LICENSE index a55050d90c..cb64c4a508 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-08-15 +Change Date: 2024-09-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 6c4874d19e..1cc6b64343 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.9.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 58874cf9dd68aed0c6f5228e657411c362af02de Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Wed, 15 Sep 2021 21:51:40 +0000 Subject: [PATCH 388/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index aad71292e9..a7830cfc4c 100644 --- a/.env +++ b/.env @@ -5,8 +5,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 SENTRY_BIND=9000 # Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails! # SENTRY_MAIL_HOST=example.com -SENTRY_IMAGE=getsentry/sentry:21.9.0 -SNUBA_IMAGE=getsentry/snuba:21.9.0 -RELAY_IMAGE=getsentry/relay:21.9.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest \ No newline at end of file diff --git a/README.md b/README.md index 1cc6b64343..6c4874d19e 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.9.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From f2e2dc2bb3c0c2505d0a6044989bdd29c7905fef Mon Sep 17 00:00:00 2001 From: jnm Date: Tue, 28 Sep 2021 07:43:36 -0400 Subject: [PATCH 389/417] docs: simplify Linux `sudo` instructions in README (#1096) Describe an easier way to pass the `SENTRY_IMAGE` environment variable to `install.sh` when using `sudo` that doesn't require modifying the `sudo` configuration --- README.md | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 6c4874d19e..5c93853f16 100644 --- a/README.md +++ b/README.md @@ -38,23 +38,10 @@ Note that this may not work for all commit SHAs as this repository evolves with ### Using Linux -If you are using Linux and you need to use `sudo` when running `./install.sh`, modifying the version of Sentry is slightly different. First, run the following: +If you are using Linux and you need to use `sudo` when running `./install.sh`, make sure to place the environment variable *after* `sudo`: ```shell -sudo visudo -``` - -Then add the following line: - -```shell -Defaults env_keep += "SENTRY_IMAGE" -``` - -Save the file then in your terminal run the following - -```shell -export SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 -sudo ./install.sh +sudo SENTRY_IMAGE=us.gcr.io/sentryio/sentry:83b1380 ./install.sh ``` Where you replace `83b1380` with the sha you want to use. From 78a5c3cb876a19a878a79bc9657d5b594084b75c Mon Sep 17 00:00:00 2001 From: EricsonMacedo Date: Fri, 15 Oct 2021 15:21:19 -0300 Subject: [PATCH 390/417] feat: Support docker compose CLI (#1116) Check if docker compose v2, CLI, is available and get semantic version from it, or fallback to get semantic version out of docker-compose v1 when checking minimum requirements during install.sh script Fixes #962 --- .github/workflows/test.yml | 25 ++++++++++++++++++------- _integration-test/run.sh | 4 ++-- docker-compose.yml | 2 +- install/_lib.sh | 3 ++- install/check-minimum-requirements.sh | 9 +++++++-- install/wrap-up.sh | 2 +- 6 files changed, 31 insertions(+), 14 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 1c5a938588..dda158491d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -26,17 +26,28 @@ jobs: integration-test: runs-on: ubuntu-20.04 name: "integration test" + strategy: + max-parallel: 1 + fail-fast: false + matrix: + include: + - compose_version: '1.28.0' + compose_path: '/usr/local/bin' + - compose_version: 'v2.0.1' + compose_path: '/usr/local/lib/docker/cli-plugins' steps: - name: Checkout uses: actions/checkout@v2 - - - name: Pin docker-compose + + - name: Get Compose run: | - COMPOSE_PATH=/usr/local/bin/docker-compose - source ./install/_min-requirements.sh - sudo rm $COMPOSE_PATH - sudo curl -L https://github.com/docker/compose/releases/download/${MIN_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` -o $COMPOSE_PATH - sudo chmod +x $COMPOSE_PATH + # Always remove `docker compose` support as that's the newer version + # and comes installed by default nowadays. + sudo rm -f "/usr/local/lib/docker/cli-plugins/docker-compose" + sudo rm -f "${{ matrix.compose_path }}/docker-compose" + sudo mkdir -p "${{ matrix.compose_path }}" + sudo curl -L https://github.com/docker/compose/releases/download/${{ matrix.compose_version }}/docker-compose-`uname -s`-`uname -m` -o "${{ matrix.compose_path }}/docker-compose" + sudo chmod +x "${{ matrix.compose_path }}/docker-compose" - name: Integration Test run: | diff --git a/_integration-test/run.sh b/_integration-test/run.sh index 6b66052bf5..ddb9973403 100755 --- a/_integration-test/run.sh +++ b/_integration-test/run.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -ex source "$(dirname $0)/../install/_lib.sh" @@ -118,7 +118,7 @@ done echo "${_endgroup}" echo "${_group}Ensure cleanup crons are working ..." -$dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+" +$dc ps | grep -q -E "\-cleanup\s+running\s+|\-cleanup_.+\s+Up\s+" echo "${_endgroup}" echo "${_group}Test custom CAs work ..." diff --git a/docker-compose.yml b/docker-compose.yml index 6314f8803d..29342e4abc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -93,7 +93,7 @@ services: smtp: <<: *restart_policy image: tianon/exim4 - hostname: ${SENTRY_MAIL_HOST:-} + hostname: ${SENTRY_MAIL_HOST:-''} volumes: - "sentry-smtp:/var/spool/exim4" - "sentry-smtp-log:/var/log/exim4" diff --git a/install/_lib.sh b/install/_lib.sh index 7b89461fa2..70f73c6e07 100644 --- a/install/_lib.sh +++ b/install/_lib.sh @@ -25,7 +25,8 @@ else _endgroup="" fi -dc="docker-compose --ansi never" +dc_base="$(docker compose version >/dev/null && echo 'docker compose' || echo 'docker-compose')" +dc="$dc_base --ansi never" dcr="$dc run --rm" # A couple of the config files are referenced from other subscripts, so they diff --git a/install/check-minimum-requirements.sh b/install/check-minimum-requirements.sh index 149b7d91da..fc564a2c52 100644 --- a/install/check-minimum-requirements.sh +++ b/install/check-minimum-requirements.sh @@ -3,8 +3,13 @@ echo "${_group}Checking minimum requirements ..." source "$(dirname $0)/_min-requirements.sh" DOCKER_VERSION=$(docker version --format '{{.Server.Version}}') -# Do NOT use $dc instead of `docker-compose` below as older versions don't support certain options and fail -COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') +# Get semantic version of Docker Compose v2 +if docker compose version >/dev/null; then + COMPOSE_VERSION=$(docker compose version --short | sed 's/v\{0,1\}\(.\{1,\}\)/\1/') +else + # Do NOT use $dc instead of `docker-compose` below as older versions don't support certain options and fail + COMPOSE_VERSION=$(docker-compose --version | sed 's/docker-compose version \(.\{1,\}\),.*/\1/') +fi RAM_AVAILABLE_IN_DOCKER=$(docker run --rm busybox free -m 2>/dev/null | awk '/Mem/ {print $2}'); CPU_AVAILABLE_IN_DOCKER=$(docker run --rm busybox nproc --all); diff --git a/install/wrap-up.sh b/install/wrap-up.sh index bbe08fac17..125e7e0133 100644 --- a/install/wrap-up.sh +++ b/install/wrap-up.sh @@ -18,7 +18,7 @@ else echo "" echo "You're all done! Run the following command to get Sentry running:" echo "" - echo " docker-compose up -d" + echo " $dc_base up -d" echo "" echo "-----------------------------------------------------------------" echo "" From 6f0b91f2855c8f975a2098236f110fd15d245e46 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 15 Oct 2021 22:41:26 +0300 Subject: [PATCH 391/417] ci(test): Relax cleanup test regex a bit (#1119) --- .github/workflows/test.yml | 4 ++-- _integration-test/run.sh | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dda158491d..0d3db0e286 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -63,5 +63,5 @@ jobs: - name: Inspect failure if: failure() run: | - docker-compose ps - docker-compose logs + docker compose ps + docker compose logs diff --git a/_integration-test/run.sh b/_integration-test/run.sh index ddb9973403..67ec747cd3 100755 --- a/_integration-test/run.sh +++ b/_integration-test/run.sh @@ -118,7 +118,7 @@ done echo "${_endgroup}" echo "${_group}Ensure cleanup crons are working ..." -$dc ps | grep -q -E "\-cleanup\s+running\s+|\-cleanup_.+\s+Up\s+" +$dc ps | grep -q -E -e '\-cleanup\s+running\s+' -e '\-cleanup[_-].+\s+Up\s+' echo "${_endgroup}" echo "${_group}Test custom CAs work ..." From a28c5580fda3247adfee4733a0e6551057110beb Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Fri, 15 Oct 2021 19:45:01 +0000 Subject: [PATCH 392/417] release: 21.10.0 --- .env | 8 ++++---- CHANGELOG.md | 13 +++++++++++++ LICENSE | 2 +- README.md | 2 +- 4 files changed, 19 insertions(+), 6 deletions(-) diff --git a/.env b/.env index a7830cfc4c..9748df43fb 100644 --- a/.env +++ b/.env @@ -5,8 +5,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 SENTRY_BIND=9000 # Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails! # SENTRY_MAIL_HOST=example.com -SENTRY_IMAGE=getsentry/sentry:nightly -SNUBA_IMAGE=getsentry/snuba:nightly -RELAY_IMAGE=getsentry/relay:nightly -SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +SENTRY_IMAGE=getsentry/sentry:21.10.0 +SNUBA_IMAGE=getsentry/snuba:21.10.0 +RELAY_IMAGE=getsentry/relay:21.10.0 +SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 WAL2JSON_VERSION=latest \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 57f37add7e..c6defdd4ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 21.10.0 + +### Support for Docker Compose v2 (ongoing) + +You asked for it and you did it! Sentry self-hosted now can work with Docker Compose v2 thanks to our community's contributions. + +PRs: #1116 + +### Various fixes & improvements + +- docs: simplify Linux `sudo` instructions in README (#1096) +- build: Set master version to nightly (58874cf9) + ## 21.9.0 - fix(healthcheck): Increase retries to 5 (#1072) diff --git a/LICENSE b/LICENSE index cb64c4a508..4e414943d8 100644 --- a/LICENSE +++ b/LICENSE @@ -16,7 +16,7 @@ Additional Use Grant: You may make use of the Licensed Work, provided that you d error-reporting or application monitoring features of the Licensed Work. -Change Date: 2024-09-15 +Change Date: 2024-10-15 Change License: Apache License, Version 2.0 diff --git a/README.md b/README.md index 5c93853f16..b16ee9d594 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry nightly +# Self-Hosted Sentry 21.10.0 Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From d3e77857058d3906b8d08a333d2b5c1e55492bca Mon Sep 17 00:00:00 2001 From: getsentry-bot Date: Fri, 15 Oct 2021 20:10:04 +0000 Subject: [PATCH 393/417] build: Set master version to nightly --- .env | 8 ++++---- README.md | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.env b/.env index 9748df43fb..a7830cfc4c 100644 --- a/.env +++ b/.env @@ -5,8 +5,8 @@ SENTRY_EVENT_RETENTION_DAYS=90 SENTRY_BIND=9000 # Set SENTRY_MAIL_HOST to a valid FQDN (host/domain name) to be able to send emails! # SENTRY_MAIL_HOST=example.com -SENTRY_IMAGE=getsentry/sentry:21.10.0 -SNUBA_IMAGE=getsentry/snuba:21.10.0 -RELAY_IMAGE=getsentry/relay:21.10.0 -SYMBOLICATOR_IMAGE=getsentry/symbolicator:0.3.4 +SENTRY_IMAGE=getsentry/sentry:nightly +SNUBA_IMAGE=getsentry/snuba:nightly +RELAY_IMAGE=getsentry/relay:nightly +SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly WAL2JSON_VERSION=latest \ No newline at end of file diff --git a/README.md b/README.md index b16ee9d594..5c93853f16 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# Self-Hosted Sentry 21.10.0 +# Self-Hosted Sentry nightly Official bootstrap for running your own [Sentry](https://sentry.io/) with [Docker](https://www.docker.com/). From 920be6771d2ea0af88995613f8dabbb297899650 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 15 Oct 2021 23:14:06 +0300 Subject: [PATCH 394/417] build: Omit nightly bump commit from changelog (#1120) --- scripts/post-release.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/post-release.sh b/scripts/post-release.sh index a05afe6230..d0a8f7b210 100755 --- a/scripts/post-release.sh +++ b/scripts/post-release.sh @@ -7,4 +7,4 @@ cd $SCRIPT_DIR/.. # Bring master back to nightlies after merge from release branch git checkout master && git pull SYMBOLICATOR_VERSION=nightly ./scripts/bump-version.sh '' 'nightly' -git diff --quiet || git commit -anm 'build: Set master version to nightly' && git pull --rebase && git push +git diff --quiet || git commit -anm $'build: Set master version to nightly\n\n#skip-changelog' && git pull --rebase && git push From f9728d193898c5d0d39840c4c677c6f85e96163d Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Mon, 11 Mar 2019 19:05:56 +0200 Subject: [PATCH 395/417] Adding .idea to .gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 4a86daa0ee..da5404445a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +# IDEA +.idea/ + # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] From 368da48702e50123f17ad342f5424685ed90fc36 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Mon, 11 Mar 2019 19:06:24 +0200 Subject: [PATCH 396/417] Adding an ability to use S3 as a backend in sentry --- Dockerfile.hypertrack | 7 + Makefile.hypertrack | 21 +++ sentry.conf.hypertrack.py | 339 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 367 insertions(+) create mode 100644 Dockerfile.hypertrack create mode 100644 Makefile.hypertrack create mode 100644 sentry.conf.hypertrack.py diff --git a/Dockerfile.hypertrack b/Dockerfile.hypertrack new file mode 100644 index 0000000000..c15b8ca498 --- /dev/null +++ b/Dockerfile.hypertrack @@ -0,0 +1,7 @@ +FROM sentry:9.0-onbuild + +LABEL io.sentry.base_version="9.0-onbuild" \ + vendor="Hypertrack, Inc" \ + maintainer="andrey@hypertrack.io" + +COPY sentry.conf.hypertrack.py /etc/sentry/sentry.conf.py diff --git a/Makefile.hypertrack b/Makefile.hypertrack new file mode 100644 index 0000000000..3602a9fc02 --- /dev/null +++ b/Makefile.hypertrack @@ -0,0 +1,21 @@ +REPOSITORY?=sentry-onpremise +TAG?=latest + +OK_COLOR=\033[32;01m +NO_COLOR=\033[0m + +build: + @echo "$(OK_COLOR)==>$(NO_COLOR) Building $(REPOSITORY):$(TAG)" + @docker build --file Dockerfile.hypertrack --rm -t $(REPOSITORY):$(TAG) . + +$(REPOSITORY)_$(TAG).tar: build + @echo "$(OK_COLOR)==>$(NO_COLOR) Saving $(REPOSITORY):$(TAG) > $@" + @docker save $(REPOSITORY):$(TAG) > $@ + +push: build + @echo "$(OK_COLOR)==>$(NO_COLOR) Pushing $(REPOSITORY):$(TAG)" + @docker push $(REPOSITORY):$(TAG) + +all: build push + +.PHONY: all build push diff --git a/sentry.conf.hypertrack.py b/sentry.conf.hypertrack.py new file mode 100644 index 0000000000..00fdc3a39c --- /dev/null +++ b/sentry.conf.hypertrack.py @@ -0,0 +1,339 @@ +# This file is just Python, with a touch of Django which means +# you can inherit and tweak settings to your hearts content. + +# For Docker, the following environment variables are supported: +# SENTRY_POSTGRES_HOST +# SENTRY_POSTGRES_PORT +# SENTRY_DB_NAME +# SENTRY_DB_USER +# SENTRY_DB_PASSWORD +# SENTRY_RABBITMQ_HOST +# SENTRY_RABBITMQ_USERNAME +# SENTRY_RABBITMQ_PASSWORD +# SENTRY_RABBITMQ_VHOST +# SENTRY_REDIS_HOST +# SENTRY_REDIS_PASSWORD +# SENTRY_REDIS_PORT +# SENTRY_REDIS_DB +# SENTRY_MEMCACHED_HOST +# SENTRY_MEMCACHED_PORT +# SENTRY_FILESTORE_DIR +# SENTRY_FILESTORE_BACKEND +# SENTRY_FILESTORE_S3_BUCKET +# SENTRY_FILESTORE_S3_ACCESS_KEY +# SENTRY_FILESTORE_S3_SECRET_KEY +# SENTRY_SERVER_EMAIL +# SENTRY_EMAIL_HOST +# SENTRY_EMAIL_PORT +# SENTRY_EMAIL_USER +# SENTRY_EMAIL_PASSWORD +# SENTRY_EMAIL_USE_TLS +# SENTRY_ENABLE_EMAIL_REPLIES +# SENTRY_SMTP_HOSTNAME +# SENTRY_MAILGUN_API_KEY +# SENTRY_SINGLE_ORGANIZATION +# SENTRY_SECRET_KEY +# SLACK_CLIENT_ID +# SLACK_CLIENT_SECRET +# SLACK_VERIFICATION_TOKEN +# GITHUB_APP_ID +# GITHUB_API_SECRET +# BITBUCKET_CONSUMER_KEY +# BITBUCKET_CONSUMER_SECRET +from sentry.conf.server import * # NOQA + +import os +import os.path + +CONF_ROOT = os.path.dirname(__file__) + +postgres = env('SENTRY_POSTGRES_HOST') or (env('POSTGRES_PORT_5432_TCP_ADDR') and 'postgres') +if postgres: + DATABASES = { + 'default': { + 'ENGINE': 'sentry.db.postgres', + 'NAME': ( + env('SENTRY_DB_NAME') + or env('POSTGRES_ENV_POSTGRES_USER') + or 'postgres' + ), + 'USER': ( + env('SENTRY_DB_USER') + or env('POSTGRES_ENV_POSTGRES_USER') + or 'postgres' + ), + 'PASSWORD': ( + env('SENTRY_DB_PASSWORD') + or env('POSTGRES_ENV_POSTGRES_PASSWORD') + or '' + ), + 'HOST': postgres, + 'PORT': ( + env('SENTRY_POSTGRES_PORT') + or '' + ), + 'OPTIONS': { + 'autocommit': True, + }, + }, + } + +# You should not change this setting after your database has been created +# unless you have altered all schemas first +SENTRY_USE_BIG_INTS = True + +# If you're expecting any kind of real traffic on Sentry, we highly recommend +# configuring the CACHES and Redis settings + +########### +# General # +########### + +# Instruct Sentry that this install intends to be run by a single organization +# and thus various UI optimizations should be enabled. +SENTRY_SINGLE_ORGANIZATION = env('SENTRY_SINGLE_ORGANIZATION', True) + +######### +# Redis # +######### + +# Generic Redis configuration used as defaults for various things including: +# Buffers, Quotas, TSDB + +redis = env('SENTRY_REDIS_HOST') or (env('REDIS_PORT_6379_TCP_ADDR') and 'redis') +if not redis: + raise Exception('Error: REDIS_PORT_6379_TCP_ADDR (or SENTRY_REDIS_HOST) is undefined, did you forget to `--link` a redis container?') + +redis_password = env('SENTRY_REDIS_PASSWORD') or '' +redis_port = env('SENTRY_REDIS_PORT') or '6379' +redis_db = env('SENTRY_REDIS_DB') or '0' + +SENTRY_OPTIONS.update({ + 'redis.clusters': { + 'default': { + 'hosts': { + 0: { + 'host': redis, + 'password': redis_password, + 'port': redis_port, + 'db': redis_db, + }, + }, + }, + }, +}) + +######### +# Cache # +######### + +# Sentry currently utilizes two separate mechanisms. While CACHES is not a +# requirement, it will optimize several high throughput patterns. + +memcached = env('SENTRY_MEMCACHED_HOST') or (env('MEMCACHED_PORT_11211_TCP_ADDR') and 'memcached') +if memcached: + memcached_port = ( + env('SENTRY_MEMCACHED_PORT') + or '11211' + ) + CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', + 'LOCATION': [memcached + ':' + memcached_port], + 'TIMEOUT': 3600, + } + } + +# A primary cache is required for things such as processing events +SENTRY_CACHE = 'sentry.cache.redis.RedisCache' + +######### +# Queue # +######### + +# See https://docs.getsentry.com/on-premise/server/queue/ for more +# information on configuring your queue broker and workers. Sentry relies +# on a Python framework called Celery to manage queues. + +rabbitmq = env('SENTRY_RABBITMQ_HOST') or (env('RABBITMQ_PORT_5672_TCP_ADDR') and 'rabbitmq') + +if rabbitmq: + BROKER_URL = ( + 'amqp://' + ( + env('SENTRY_RABBITMQ_USERNAME') + or env('RABBITMQ_ENV_RABBITMQ_DEFAULT_USER') + or 'guest' + ) + ':' + ( + env('SENTRY_RABBITMQ_PASSWORD') + or env('RABBITMQ_ENV_RABBITMQ_DEFAULT_PASS') + or 'guest' + ) + '@' + rabbitmq + '/' + ( + env('SENTRY_RABBITMQ_VHOST') + or env('RABBITMQ_ENV_RABBITMQ_DEFAULT_VHOST') + or '/' + ) + ) +else: + BROKER_URL = 'redis://:' + redis_password + '@' + redis + ':' + redis_port + '/' + redis_db + + +############### +# Rate Limits # +############### + +# Rate limits apply to notification handlers and are enforced per-project +# automatically. + +SENTRY_RATELIMITER = 'sentry.ratelimits.redis.RedisRateLimiter' + +################## +# Update Buffers # +################## + +# Buffers (combined with queueing) act as an intermediate layer between the +# database and the storage API. They will greatly improve efficiency on large +# numbers of the same events being sent to the API in a short amount of time. +# (read: if you send any kind of real data to Sentry, you should enable buffers) + +SENTRY_BUFFER = 'sentry.buffer.redis.RedisBuffer' + +########## +# Quotas # +########## + +# Quotas allow you to rate limit individual projects or the Sentry install as +# a whole. + +SENTRY_QUOTAS = 'sentry.quotas.redis.RedisQuota' + +######## +# TSDB # +######## + +# The TSDB is used for building charts as well as making things like per-rate +# alerts possible. + +SENTRY_TSDB = 'sentry.tsdb.redis.RedisTSDB' + +########### +# Digests # +########### + +# The digest backend powers notification summaries. + +SENTRY_DIGESTS = 'sentry.digests.backends.redis.RedisBackend' + +################ +# File storage # +################ + +# Uploaded media uses these `filestore` settings. The available +# backends are either `filesystem` or `s3`. +if env('SENTRY_FILESTORE_BACKEND', 'filesystem') == 's3': + SENTRY_OPTIONS['filestore.backend'] = 's3' + if env('SENTRY_FILESTORE_S3_ACCESS_KEY') and env('SENTRY_FILESTORE_S3_SECRET_KEY'): + # We rely on Access and Secret keys as the parameters + SENTRY_OPTIONS['filestore.options'] = { + 'access_key': env('SENTRY_FILESTORE_S3_ACCESS_KEY'), + 'secret_key': env('SENTRY_FILESTORE_S3_SECRET_KEY'), + 'bucket_name': env('SENTRY_FILESTORE_S3_BUCKET'), + } + else: + # If this is running on an ec2 instance, allow boto to connect using an IAM role + # instead of explicitly provided an access key and secret + # http://boto3.readthedocs.io/en/latest/guide/configuration.html#iam-role + SENTRY_OPTIONS['filestore.options'] = { + 'bucket_name': env('SENTRY_FILESTORE_S3_BUCKET'), + } +else: + SENTRY_OPTIONS['filestore.backend'] = 'filesystem' + SENTRY_OPTIONS['filestore.options'] = { + 'location': env('SENTRY_FILESTORE_DIR'), + } + + +############## +# Web Server # +############## + +# If you're using a reverse SSL proxy, you should enable the X-Forwarded-Proto +# header and set `SENTRY_USE_SSL=1` + +if env('SENTRY_USE_SSL', False): + SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https') + SESSION_COOKIE_SECURE = True + CSRF_COOKIE_SECURE = True + SOCIAL_AUTH_REDIRECT_IS_HTTPS = True + +SENTRY_WEB_HOST = '0.0.0.0' +SENTRY_WEB_PORT = 9000 +SENTRY_WEB_OPTIONS = { + # 'workers': 3, # the number of web workers +} + +############### +# Mail Server # +############### + + +email = env('SENTRY_EMAIL_HOST') or (env('SMTP_PORT_25_TCP_ADDR') and 'smtp') +if email: + SENTRY_OPTIONS['mail.backend'] = 'smtp' + SENTRY_OPTIONS['mail.host'] = email + SENTRY_OPTIONS['mail.password'] = env('SENTRY_EMAIL_PASSWORD') or '' + SENTRY_OPTIONS['mail.username'] = env('SENTRY_EMAIL_USER') or '' + SENTRY_OPTIONS['mail.port'] = int(env('SENTRY_EMAIL_PORT') or 25) + SENTRY_OPTIONS['mail.use-tls'] = env('SENTRY_EMAIL_USE_TLS', False) +else: + SENTRY_OPTIONS['mail.backend'] = 'dummy' + +# The email address to send on behalf of +SENTRY_OPTIONS['mail.from'] = env('SENTRY_SERVER_EMAIL') or 'root@localhost' + +# If you're using mailgun for inbound mail, set your API key and configure a +# route to forward to /api/hooks/mailgun/inbound/ +SENTRY_OPTIONS['mail.mailgun-api-key'] = env('SENTRY_MAILGUN_API_KEY') or '' + +# If you specify a MAILGUN_API_KEY, you definitely want EMAIL_REPLIES +if SENTRY_OPTIONS['mail.mailgun-api-key']: + SENTRY_OPTIONS['mail.enable-replies'] = True +else: + SENTRY_OPTIONS['mail.enable-replies'] = env('SENTRY_ENABLE_EMAIL_REPLIES', False) + +if SENTRY_OPTIONS['mail.enable-replies']: + SENTRY_OPTIONS['mail.reply-hostname'] = env('SENTRY_SMTP_HOSTNAME') or '' + +##################### +# SLACK INTEGRATION # +##################### +slack = env('SLACK_CLIENT_ID') and env('SLACK_CLIENT_SECRET') +if slack: + SENTRY_OPTIONS['slack.client-id'] = env('SLACK_CLIENT_ID') + SENTRY_OPTIONS['slack.client-secret'] = env('SLACK_CLIENT_SECRET') + SENTRY_OPTIONS['slack.verification-token'] = env('SLACK_VERIFICATION_TOKEN') or '' + +# If this value ever becomes compromised, it's important to regenerate your +# SENTRY_SECRET_KEY. Changing this value will result in all current sessions +# being invalidated. +secret_key = env('SENTRY_SECRET_KEY') +if not secret_key: + raise Exception('Error: SENTRY_SECRET_KEY is undefined, run `generate-secret-key` and set to -e SENTRY_SECRET_KEY') + +if 'SENTRY_RUNNING_UWSGI' not in os.environ and len(secret_key) < 32: + print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + print('!! CAUTION !!') + print('!! Your SENTRY_SECRET_KEY is potentially insecure. !!') + print('!! We recommend at least 32 characters long. !!') + print('!! Regenerate with `generate-secret-key`. !!') + print('!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!') + +SENTRY_OPTIONS['system.secret-key'] = secret_key + +if 'GITHUB_APP_ID' in os.environ: + GITHUB_EXTENDED_PERMISSIONS = ['repo'] + GITHUB_APP_ID = env('GITHUB_APP_ID') + GITHUB_API_SECRET = env('GITHUB_API_SECRET') + +if 'BITBUCKET_CONSUMER_KEY' in os.environ: + BITBUCKET_CONSUMER_KEY = env('BITBUCKET_CONSUMER_KEY') + BITBUCKET_CONSUMER_SECRET = env('BITBUCKET_CONSUMER_SECRET') From b834c79239aeed3079ab79a6758a3e6fa6ab477b Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 03:52:40 +0200 Subject: [PATCH 397/417] Adding the spec file for CodeBuild --- buildspec.hypertrack.yml | 60 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 buildspec.hypertrack.yml diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml new file mode 100644 index 0000000000..1b79f654ee --- /dev/null +++ b/buildspec.hypertrack.yml @@ -0,0 +1,60 @@ +version: 0.2 + +phases: + install: + run-as: root + commands: + - echo "Running install step on `date`" + - echo "Installing WGET" + - apt-get update -qq + - apt-get install -y -qq wget + - echo "Installing JQ" + - wget -nv https://github.com/stedolan/jq/releases/download/jq-1.6/jq-linux64 -O /usr/local/bin/jq + - chmod +x /usr/local/bin/jq + + pre_build: + commands: + - pip install awscli --upgrade --user + - echo `aws --version` + - echo "Logging in to Amazon ECR..." + - $(aws ecr get-login --region $HT_REGION --no-include-email) + - IMAGE_TAG=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) + - echo "Entered the pre_build phase..." + + build: + commands: + - echo "Build started on `date`" + - echo "Building the Docker image..." + - docker build --file Dockerfile.hypertrack --rm -t $HT_REPOSITORY_URI:latest . + - docker tag $REPOSITORY_URI:latest $HT_REPOSITORY_URI:$IMAGE_TAG + + post_build: + commands: + - echo "Build completed on `date`" + - echo "Pushing the Docker images..." + - docker push $HT_REPOSITORY_URI:latest + - docker push $HT_REPOSITORY_URI:$IMAGE_TAG + - echo "Writing image definitions file..." + - printf '[{"name":"web","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > imagedefinitions.json + - echo "Upgrading db-migrate task definitions" + - RUNTASK=$(aws ecs run-task --launch-type FARGATE --cluster $HT_SENTRY_ECS_CLUSTER --task-definition $HT_SENTRY_DBMIGRATE_TASK --network-configuration "awsvpcConfiguration={subnets=[$HT_RUN_TASK_SUBNETS],securityGroups=[$HT_RUN_TASK_SGS],assignPublicIp=\"DISABLED\"}") + - TASK_ID=$( echo "$RUNTASK" | jq -r '.tasks[0].taskArn' ) + - echo "Started new Task - $TASK_ID" + - echo "Waiting for dbmigrate task to be running..." + - aws ecs wait tasks-running --tasks $TASK_ID --cluster $HT_SENTRY_ECS_CLUSTER + - echo "Dbmigrate task is running" + - echo "Waiting for task to stop..." + - aws ecs wait tasks-stopped --tasks $TASK_ID --cluster $HT_SENTRY_ECS_CLUSTER + - | + if [[ "x$HT_CREATE_SUPERUSER" == "xyes" ]]; then + echo "Creating Superuser" + RUNTASK=$(aws ecs run-task --launch-type FARGATE --cluster $HT_SENTRY_ECS_CLUSTER --task-definition $HT_SENTRY_CREATEUSER_TASK --network-configuration "awsvpcConfiguration={subnets=[$HT_RUN_TASK_SUBNETS],securityGroups=[$HT_RUN_TASK_SGS],assignPublicIp=\"DISABLED\"}") + TASK_ID=$( echo "$RUNTASK" | jq -r '.tasks[0].taskArn' ) + aws ecs wait tasks-running --tasks $TASK_ID --cluster $HT_SENTRY_ECS_CLUSTER && aws ecs wait tasks-stopped --tasks $TASK_ID --cluster $HT_SENTRY_ECS_CLUSTER + else + echo "We dont create a superuser right now" + fi + - echo "Post Build step is done." + +artifacts: + files: imagedefinitions.json From 017750b6c372788b17a96e8eb5927c4804a5521b Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 03:59:20 +0200 Subject: [PATCH 398/417] Using pip3 instead of pip --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 1b79f654ee..9a71b3b8d0 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -14,7 +14,7 @@ phases: pre_build: commands: - - pip install awscli --upgrade --user + - pip3 install awscli --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." - $(aws ecr get-login --region $HT_REGION --no-include-email) From 09e458e37b9e6a688df417837fe31d86878ec408 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 04:04:06 +0200 Subject: [PATCH 399/417] Fixing typo in the variable name --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 9a71b3b8d0..ddca70e0d6 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -26,7 +26,7 @@ phases: - echo "Build started on `date`" - echo "Building the Docker image..." - docker build --file Dockerfile.hypertrack --rm -t $HT_REPOSITORY_URI:latest . - - docker tag $REPOSITORY_URI:latest $HT_REPOSITORY_URI:$IMAGE_TAG + - docker tag $HT_REPOSITORY_URI:latest $HT_REPOSITORY_URI:$IMAGE_TAG post_build: commands: From 586b2d5cf0a1d7bec9ad5ae1683c2723b446da90 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 14:46:26 +0200 Subject: [PATCH 400/417] Fixing typo in the condition --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index ddca70e0d6..a7e10592e7 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -46,7 +46,7 @@ phases: - echo "Waiting for task to stop..." - aws ecs wait tasks-stopped --tasks $TASK_ID --cluster $HT_SENTRY_ECS_CLUSTER - | - if [[ "x$HT_CREATE_SUPERUSER" == "xyes" ]]; then + if [ "x$HT_CREATE_SUPERUSER" == "xyes" ]; then echo "Creating Superuser" RUNTASK=$(aws ecs run-task --launch-type FARGATE --cluster $HT_SENTRY_ECS_CLUSTER --task-definition $HT_SENTRY_CREATEUSER_TASK --network-configuration "awsvpcConfiguration={subnets=[$HT_RUN_TASK_SUBNETS],securityGroups=[$HT_RUN_TASK_SGS],assignPublicIp=\"DISABLED\"}") TASK_ID=$( echo "$RUNTASK" | jq -r '.tasks[0].taskArn' ) From 9acb6e84c32ea5a9fa59793bb1bf8c77256356d9 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 15:01:42 +0200 Subject: [PATCH 401/417] Fixing conditional expression --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index a7e10592e7..304b124294 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -46,7 +46,7 @@ phases: - echo "Waiting for task to stop..." - aws ecs wait tasks-stopped --tasks $TASK_ID --cluster $HT_SENTRY_ECS_CLUSTER - | - if [ "x$HT_CREATE_SUPERUSER" == "xyes" ]; then + if expr "x$HT_CREATE_SUPERUSER" : "xyes" > /dev/null; then echo "Creating Superuser" RUNTASK=$(aws ecs run-task --launch-type FARGATE --cluster $HT_SENTRY_ECS_CLUSTER --task-definition $HT_SENTRY_CREATEUSER_TASK --network-configuration "awsvpcConfiguration={subnets=[$HT_RUN_TASK_SUBNETS],securityGroups=[$HT_RUN_TASK_SGS],assignPublicIp=\"DISABLED\"}") TASK_ID=$( echo "$RUNTASK" | jq -r '.tasks[0].taskArn' ) From ccab9ea5083a87f2ee664a5905bc4b604af6961e Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 15:17:07 +0200 Subject: [PATCH 402/417] setting up different imagedefinition.json files per sentry container --- buildspec.hypertrack.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 304b124294..777132758d 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -34,8 +34,12 @@ phases: - echo "Pushing the Docker images..." - docker push $HT_REPOSITORY_URI:latest - docker push $HT_REPOSITORY_URI:$IMAGE_TAG - - echo "Writing image definitions file..." - - printf '[{"name":"web","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > imagedefinitions.json + - echo "Writing image definitions files..." + - printf '[{"name":"$HT_CONTAINER_CREATEUSER","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > createuser_imagedefinitions.json + - printf '[{"name":"$HT_CONTAINER_CRON","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > cron_imagedefinitions.json + - printf '[{"name":"$HT_CONTAINER_DBMIGRATE","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > dbmigrate_imagedefinitions.json + - printf '[{"name":"$HT_CONTAINER_WEB","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > web_imagedefinitions.json + - printf '[{"name":"$HT_CONTAINER_WORKER","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > worker_imagedefinitions.json - echo "Upgrading db-migrate task definitions" - RUNTASK=$(aws ecs run-task --launch-type FARGATE --cluster $HT_SENTRY_ECS_CLUSTER --task-definition $HT_SENTRY_DBMIGRATE_TASK --network-configuration "awsvpcConfiguration={subnets=[$HT_RUN_TASK_SUBNETS],securityGroups=[$HT_RUN_TASK_SGS],assignPublicIp=\"DISABLED\"}") - TASK_ID=$( echo "$RUNTASK" | jq -r '.tasks[0].taskArn' ) From f5c94714d4b0b2c9023fca645f7aade2b04b449f Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 15:28:51 +0200 Subject: [PATCH 403/417] Setting up which files to grab as artifacts --- buildspec.hypertrack.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 777132758d..3f6d3c0b5c 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -61,4 +61,5 @@ phases: - echo "Post Build step is done." artifacts: - files: imagedefinitions.json + files: + - '*_imagedefinitions.json' From 9eaaeb21d2789d2d0a516dd86b9a88d3b5865f1a Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Wed, 13 Mar 2019 15:36:03 +0200 Subject: [PATCH 404/417] Fixing the typo with quotes when generating *_imagedefinitions.json artifacts --- buildspec.hypertrack.yml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 3f6d3c0b5c..a878fef575 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -35,11 +35,13 @@ phases: - docker push $HT_REPOSITORY_URI:latest - docker push $HT_REPOSITORY_URI:$IMAGE_TAG - echo "Writing image definitions files..." - - printf '[{"name":"$HT_CONTAINER_CREATEUSER","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > createuser_imagedefinitions.json - - printf '[{"name":"$HT_CONTAINER_CRON","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > cron_imagedefinitions.json - - printf '[{"name":"$HT_CONTAINER_DBMIGRATE","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > dbmigrate_imagedefinitions.json - - printf '[{"name":"$HT_CONTAINER_WEB","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > web_imagedefinitions.json - - printf '[{"name":"$HT_CONTAINER_WORKER","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > worker_imagedefinitions.json + - printf '[{"name":"'$HT_CONTAINER_CREATEUSER'","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > createuser_imagedefinitions.json + - printf '[{"name":"'$HT_CONTAINER_CRON'","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > cron_imagedefinitions.json + - printf '[{"name":"'$HT_CONTAINER_DBMIGRATE'","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > dbmigrate_imagedefinitions.json + - printf '[{"name":"'$HT_CONTAINER_WEB'","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > web_imagedefinitions.json + - printf '[{"name":"'$HT_CONTAINER_WORKER'","imageUri":"%s"}]' $HT_REPOSITORY_URI:$IMAGE_TAG > worker_imagedefinitions.json + - echo "Dumping *_imagedefinitions.json" + - cat *_imagedefinitions.json - echo "Upgrading db-migrate task definitions" - RUNTASK=$(aws ecs run-task --launch-type FARGATE --cluster $HT_SENTRY_ECS_CLUSTER --task-definition $HT_SENTRY_DBMIGRATE_TASK --network-configuration "awsvpcConfiguration={subnets=[$HT_RUN_TASK_SUBNETS],securityGroups=[$HT_RUN_TASK_SGS],assignPublicIp=\"DISABLED\"}") - TASK_ID=$( echo "$RUNTASK" | jq -r '.tasks[0].taskArn' ) From 9cefe4b25d07963550e5d6c141ec2bbeb1b39023 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Mon, 18 Mar 2019 23:29:33 +0200 Subject: [PATCH 405/417] Adding SSO plugins for Gmail and Github --- Dockerfile.hypertrack | 9 +++++++++ requirements-hypertrack.txt | 3 +++ sentry.conf.hypertrack.py | 14 ++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 requirements-hypertrack.txt diff --git a/Dockerfile.hypertrack b/Dockerfile.hypertrack index c15b8ca498..3410e96253 100644 --- a/Dockerfile.hypertrack +++ b/Dockerfile.hypertrack @@ -4,4 +4,13 @@ LABEL io.sentry.base_version="9.0-onbuild" \ vendor="Hypertrack, Inc" \ maintainer="andrey@hypertrack.io" +# Sane defaults for pip +ENV PIP_NO_CACHE_DIR off +ENV PIP_DISABLE_PIP_VERSION_CHECK on + COPY sentry.conf.hypertrack.py /etc/sentry/sentry.conf.py +COPY requirements-hypertrack.txt /tmp/requirements-hypertrack.txt + +RUN set -x \ + && pip install -r /tmp/requirements-hypertrack.txt \ + && rm -f /tmp/requirements-hypertrack.txt diff --git a/requirements-hypertrack.txt b/requirements-hypertrack.txt new file mode 100644 index 0000000000..4efa7f4182 --- /dev/null +++ b/requirements-hypertrack.txt @@ -0,0 +1,3 @@ +# Add plugins here +sentry-auth-google +https://github.com/getsentry/sentry-auth-github/archive/master.zip diff --git a/sentry.conf.hypertrack.py b/sentry.conf.hypertrack.py index 00fdc3a39c..90c1404a05 100644 --- a/sentry.conf.hypertrack.py +++ b/sentry.conf.hypertrack.py @@ -38,6 +38,8 @@ # SLACK_VERIFICATION_TOKEN # GITHUB_APP_ID # GITHUB_API_SECRET +# GOOGLE_CLIENT_ID +# GOOGLE_CLIENT_SECRET # BITBUCKET_CONSUMER_KEY # BITBUCKET_CONSUMER_SECRET from sentry.conf.server import * # NOQA @@ -312,6 +314,14 @@ SENTRY_OPTIONS['slack.client-secret'] = env('SLACK_CLIENT_SECRET') SENTRY_OPTIONS['slack.verification-token'] = env('SLACK_VERIFICATION_TOKEN') or '' +##################### +# GMAIL SSO # +##################### +gmail_sso = env('GOOGLE_CLIENT_ID') and env('GOOGLE_CLIENT_SECRET') +if gmail_sso: + GOOGLE_CLIENT_ID = env('GOOGLE_CLIENT_ID') + GOOGLE_CLIENT_SECRET = env('GOOGLE_CLIENT_SECRET') + # If this value ever becomes compromised, it's important to regenerate your # SENTRY_SECRET_KEY. Changing this value will result in all current sessions # being invalidated. @@ -329,10 +339,14 @@ SENTRY_OPTIONS['system.secret-key'] = secret_key +##################### +# GITHUB SSO # +##################### if 'GITHUB_APP_ID' in os.environ: GITHUB_EXTENDED_PERMISSIONS = ['repo'] GITHUB_APP_ID = env('GITHUB_APP_ID') GITHUB_API_SECRET = env('GITHUB_API_SECRET') + GITHUB_REQUIRE_VERIFIED_EMAIL = env('GITHUB_REQUIRE_VERIFIED_EMAIL', True) if 'BITBUCKET_CONSUMER_KEY' in os.environ: BITBUCKET_CONSUMER_KEY = env('BITBUCKET_CONSUMER_KEY') From 79016140153e035f5a0e830c87d90edcf244da62 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Thu, 14 Mar 2019 21:23:26 +0200 Subject: [PATCH 406/417] Update buildspec.hypertrack.yml --- buildspec.hypertrack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index a878fef575..288bbc5f04 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -14,6 +14,7 @@ phases: pre_build: commands: + - echo "Starting pre-build phase..." - pip3 install awscli --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." From 0b7a7ff65b84de40ce3053740b775efb5f76fcaa Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Thu, 14 Mar 2019 22:13:39 +0200 Subject: [PATCH 407/417] Update buildspec.hypertrack.yml --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 288bbc5f04..6e08c1b3eb 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -14,7 +14,7 @@ phases: pre_build: commands: - - echo "Starting pre-build phase..." + - echo "Starting pre-build phase on `date` ..." - pip3 install awscli --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." From a709ccdb6a26ad392f84892938d11f36a62bb489 Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Thu, 14 Mar 2019 22:27:34 +0200 Subject: [PATCH 408/417] Update buildspec.hypertrack.yml --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 6e08c1b3eb..6ef5e9b165 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -14,7 +14,7 @@ phases: pre_build: commands: - - echo "Starting pre-build phase on `date` ..." + - echo "Starting pre-build phase on `date`" - pip3 install awscli --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." From c6e41ba67c5c72cd01c4e1242b800a676f774bec Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Thu, 14 Mar 2019 22:47:21 +0200 Subject: [PATCH 409/417] Update buildspec.hypertrack.yml --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 6ef5e9b165..d0f9e27190 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -20,7 +20,7 @@ phases: - echo "Logging in to Amazon ECR..." - $(aws ecr get-login --region $HT_REGION --no-include-email) - IMAGE_TAG=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) - - echo "Entered the pre_build phase..." + - echo " pre_build phase is done..." build: commands: From 0d85f47507f02727a5925f88185c613efd02552b Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Thu, 14 Mar 2019 23:30:39 +0200 Subject: [PATCH 410/417] Update 2 --- buildspec.hypertrack.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index d0f9e27190..875d37dbce 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -28,10 +28,11 @@ phases: - echo "Building the Docker image..." - docker build --file Dockerfile.hypertrack --rm -t $HT_REPOSITORY_URI:latest . - docker tag $HT_REPOSITORY_URI:latest $HT_REPOSITORY_URI:$IMAGE_TAG + - echo "Build completed on `date`" post_build: commands: - - echo "Build completed on `date`" + - echo "Starting Post Build phase" - echo "Pushing the Docker images..." - docker push $HT_REPOSITORY_URI:latest - docker push $HT_REPOSITORY_URI:$IMAGE_TAG From 0fa728d79bc84ac33b81d6cb683de77ec19890de Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Thu, 14 Mar 2019 23:44:26 +0200 Subject: [PATCH 411/417] Update 4 --- buildspec.hypertrack.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 875d37dbce..89cff867dd 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -27,6 +27,7 @@ phases: - echo "Build started on `date`" - echo "Building the Docker image..." - docker build --file Dockerfile.hypertrack --rm -t $HT_REPOSITORY_URI:latest . + - echo "Tagging the image" - docker tag $HT_REPOSITORY_URI:latest $HT_REPOSITORY_URI:$IMAGE_TAG - echo "Build completed on `date`" From a1c035527e7c9285e25e8390481dbe50bd51b81d Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Tue, 25 Jun 2019 15:09:03 +0300 Subject: [PATCH 412/417] Upgrading Sentry --- Dockerfile.hypertrack | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.hypertrack b/Dockerfile.hypertrack index 3410e96253..da9453a0ec 100644 --- a/Dockerfile.hypertrack +++ b/Dockerfile.hypertrack @@ -1,6 +1,6 @@ -FROM sentry:9.0-onbuild +FROM sentry:9.1.1-onbuild -LABEL io.sentry.base_version="9.0-onbuild" \ +LABEL io.sentry.base_version="9.1.1-onbuild" \ vendor="Hypertrack, Inc" \ maintainer="andrey@hypertrack.io" From 6137fe313a3cfb7f3b2fe0a35f05986fb37e149f Mon Sep 17 00:00:00 2001 From: Andrey Miroshnichenko Date: Mon, 5 Aug 2019 18:27:54 +0300 Subject: [PATCH 413/417] Upgrading Sentry to sentry:9.1.2-onbuild --- Dockerfile.hypertrack | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile.hypertrack b/Dockerfile.hypertrack index da9453a0ec..abce6763a8 100644 --- a/Dockerfile.hypertrack +++ b/Dockerfile.hypertrack @@ -1,6 +1,6 @@ -FROM sentry:9.1.1-onbuild +FROM sentry:9.1.2-onbuild -LABEL io.sentry.base_version="9.1.1-onbuild" \ +LABEL io.sentry.base_version="9.1.2-onbuild" \ vendor="Hypertrack, Inc" \ maintainer="andrey@hypertrack.io" From fe878c2acbdb16be1eb723f61e350d69bb857ec8 Mon Sep 17 00:00:00 2001 From: thomashypertrack <46773379+thomashypertrack@users.noreply.github.com> Date: Thu, 10 Jun 2021 11:37:54 -0700 Subject: [PATCH 414/417] Update config to enable custom inbound filters --- sentry.conf.hypertrack.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/sentry.conf.hypertrack.py b/sentry.conf.hypertrack.py index 90c1404a05..ea5a6eb5cb 100644 --- a/sentry.conf.hypertrack.py +++ b/sentry.conf.hypertrack.py @@ -339,6 +339,9 @@ SENTRY_OPTIONS['system.secret-key'] = secret_key +# Enables the Custom Inbound Filters feature +SENTRY_FEATURES['projects:custom-inbound-filters'] = True + ##################### # GITHUB SSO # ##################### From 906008e2065f43d9bf648531f7d88f0cfe8a6139 Mon Sep 17 00:00:00 2001 From: thomashypertrack <46773379+thomashypertrack@users.noreply.github.com> Date: Thu, 10 Jun 2021 12:33:10 -0700 Subject: [PATCH 415/417] Use latest 1.x aws cli --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 89cff867dd..14613ec107 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -15,7 +15,7 @@ phases: pre_build: commands: - echo "Starting pre-build phase on `date`" - - pip3 install awscli --upgrade --user + - pip3 install awscli<1.19.91 --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." - $(aws ecr get-login --region $HT_REGION --no-include-email) From ae8954848e55c6bebaef268613cddb2afbd70b32 Mon Sep 17 00:00:00 2001 From: thomashypertrack <46773379+thomashypertrack@users.noreply.github.com> Date: Thu, 10 Jun 2021 12:38:06 -0700 Subject: [PATCH 416/417] Another attempt to make ECR login work --- buildspec.hypertrack.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index 14613ec107..e33867d4fb 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -15,10 +15,10 @@ phases: pre_build: commands: - echo "Starting pre-build phase on `date`" - - pip3 install awscli<1.19.91 --upgrade --user + - pip3 install awscli --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." - - $(aws ecr get-login --region $HT_REGION --no-include-email) + -aws ecr get-login-password | docker login --username AWS --password-stdin $HT_REPOSITORY_URI - IMAGE_TAG=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) - echo " pre_build phase is done..." From 77568f719c85b95041d02d6e1180a26ccccd3620 Mon Sep 17 00:00:00 2001 From: thomashypertrack <46773379+thomashypertrack@users.noreply.github.com> Date: Thu, 10 Jun 2021 12:38:28 -0700 Subject: [PATCH 417/417] Update buildspec.hypertrack.yml --- buildspec.hypertrack.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildspec.hypertrack.yml b/buildspec.hypertrack.yml index e33867d4fb..15cf520f91 100644 --- a/buildspec.hypertrack.yml +++ b/buildspec.hypertrack.yml @@ -18,7 +18,7 @@ phases: - pip3 install awscli --upgrade --user - echo `aws --version` - echo "Logging in to Amazon ECR..." - -aws ecr get-login-password | docker login --username AWS --password-stdin $HT_REPOSITORY_URI + - aws ecr get-login-password | docker login --username AWS --password-stdin $HT_REPOSITORY_URI - IMAGE_TAG=$(echo $CODEBUILD_RESOLVED_SOURCE_VERSION | cut -c 1-7) - echo " pre_build phase is done..."