diff --git a/.circleci/config.yml b/.circleci/config.yml index 915d1bd9ab..e8c970c754 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -5,24 +5,18 @@ version: 2.1 ######################################################################################################################## executors: - # Our brand new builder - clojure-and-node: - working_directory: /home/circleci/metabase/metabase/ - docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 - - # CircleCI base (Lein 2.9.5) + Node + Headless browsers + Clojure CLI - big one + # CircleCI base Node + Headless browsers + Clojure CLI - big one # Maildev runs by default with all Cypress tests clojure-and-node-and-browsers: working_directory: /home/circleci/metabase/metabase/ docker: - - image: circleci/clojure:lein-2.9.5-node-browsers + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: maildev/maildev java-8: working_directory: /home/circleci/metabase/metabase/ docker: - - image: circleci/clojure:openjdk-8-lein-2.9.5-buster + - image: metabase/ci:circleci-java-8-clj-1.10.3.929-07-27-2021-node-browsers # Java 11 tests also test Metabase with the at-rest encryption enabled. See # https://metabase.com/docs/latest/operations-guide/encrypting-database-details-at-rest.html for an explanation of @@ -30,19 +24,19 @@ executors: java-11: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_ENCRYPTION_SECRET_KEY: Orw0AAyzkO/kPTLJRxiyKoBHXa/d6ZcO+p+gpZO/wSQ= java-16: working_directory: /home/circleci/metabase/metabase/ docker: - - image: circleci/clojure:openjdk-16-lein-2.9.5-buster + - image: metabase/ci:circleci-java-16-clj-1.10.3.929-07-27-2021-node-browsers postgres-9-6: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_DB_TYPE: postgres MB_DB_PORT: 5432 @@ -58,7 +52,7 @@ executors: postgres-latest: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_DB_TYPE: postgres MB_DB_PORT: 5432 @@ -75,7 +69,7 @@ executors: mysql-5-7: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_DB_TYPE: mysql MB_DB_HOST: localhost @@ -88,7 +82,7 @@ executors: mysql-latest: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_DB_TYPE: mysql MB_DB_HOST: localhost @@ -101,7 +95,7 @@ executors: mariadb-10-2: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_DB_TYPE: mysql MB_DB_HOST: localhost @@ -114,7 +108,7 @@ executors: mariadb-latest: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_DB_TYPE: mysql MB_DB_HOST: localhost @@ -131,13 +125,13 @@ executors: mongo: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: circleci/mongo:4.0 presto-186: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/presto-mb-ci:0.186 environment: JAVA_TOOL_OPTIONS: "-Xmx2g" @@ -148,7 +142,7 @@ executors: presto-jdbc-env: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/presto-mb-ci:latest # version 0.254 environment: JAVA_TOOL_OPTIONS: "-Xmx2g" @@ -167,19 +161,19 @@ executors: sparksql: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/spark:2.1.1 vertica: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: sumitchawla/vertica sqlserver: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers environment: MB_SQLSERVER_TEST_HOST: localhost MB_SQLSERVER_TEST_PASSWORD: 'P@ssw0rd' @@ -193,7 +187,7 @@ executors: druid: working_directory: /home/circleci/metabase/metabase/ docker: - - image: metabase/ci:lein-2.9.5-clojure-1.10.3.814 + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/druid:0.20.2 environment: CLUSTER_SIZE: nano-quickstart @@ -205,19 +199,19 @@ executors: fe-mongo-4: working_directory: /home/circleci/metabase/metabase/ docker: - - image: circleci/clojure:lein-2.9.5-node-browsers + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/qa-databases:mongo-sample-4.0 fe-postgres-12: working_directory: /home/circleci/metabase/metabase/ docker: - - image: circleci/clojure:lein-2.9.5-node-browsers + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/qa-databases:postgres-sample-12 fe-mysql-8: working_directory: /home/circleci/metabase/metabase/ docker: - - image: circleci/clojure:lein-2.9.5-node-browsers + - image: metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers - image: metabase/qa-databases:mysql-sample-8 @@ -228,7 +222,8 @@ executors: # `default_parameters` isn't a key that CircleCI uses, but this form lets us reuse parameter definitions default_parameters: &Params edition: - type: string + type: enum + enum: ["oss", "ee"] default: "oss" # .BACKEND-CHECKSUMS, .FRONTEND-CHECKSUMS, and .MODULE-CHECKSUMS are created during the checkout step; see that step @@ -245,23 +240,25 @@ default_parameters: &Params # uncontrollably since old deps would continue to accumulate. Restoring big caches is really slow in Circle. It's # actually faster to recreate the deps cache from scratch whenever we need to which keeps the size down. cache-key-backend-deps: &CacheKeyBackendDeps - key: v1-{{ checksum ".CACHE-PREFIX" }}-be-deps-{{ checksum "project.clj" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }} + # TODO -- this should actually include the Java source files and the Spark SQL AOT source files as well since we now + # compile those as part of this step. FIXME + key: v5-{{ checksum ".CACHE-PREFIX" }}-be-deps-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }} cache-key-frontend-deps: &CacheKeyFrontendDeps - key: v1-{{ checksum ".CACHE-PREFIX" }}-fe-deps-{{ checksum "yarn.lock" }} + key: v5-{{ checksum ".CACHE-PREFIX" }}-fe-deps-{{ checksum "yarn.lock" }} # Key used for implementation of run-on-change -- this is the cache key that contains the .SUCCESS dummy file # By default the key ALWAYS includes the name of the test job itself ($CIRCLE_STAGE) so you don't need to add that yourself. cache-key-run-on-change: &CacheKeyRunOnChange - key: v1-{{ checksum ".CACHE-PREFIX" }}-run-on-change-{{ .Environment.CIRCLE_STAGE }}-<< parameters.checksum >> + key: v5-{{ checksum ".CACHE-PREFIX" }}-run-on-change-{{ .Environment.CIRCLE_STAGE }}-<< parameters.checksum >> # Key for the local maven installation of metabase-core (used by build-uberjar-drivers) cache-key-metabase-core: &CacheKeyMetabaseCore - key: v1-{{ checksum ".CACHE-PREFIX" }}-metabase-core-{{ checksum ".BACKEND-CHECKSUMS" }} + key: v5-{{ checksum ".CACHE-PREFIX" }}-metabase-core-{{ checksum ".BACKEND-CHECKSUMS" }} # Key for the drivers built by build-uberjar-drivers cache-key-drivers: &CacheKeyDrivers - key: v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}-<< parameters.edition >> + key: v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }}-<< parameters.edition >> # This is also used by the uberjar-build-drivers step; this is a unique situation because the build-drivers script has # logic to determine whether to rebuild drivers or not that is quite a bit more sophisticated that the run-on-change @@ -269,17 +266,17 @@ cache-key-drivers: &CacheKeyDrivers # redshift driver. cache-keys-drivers-with-fallback-keys: &CacheKeyDrivers_WithFallbackKeys keys: - - v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }} - - v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }} - - v1-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>- + - v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }}-{{ checksum ".BACKEND-CHECKSUMS" }} + - v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>-{{ checksum ".MODULES-CHECKSUMS" }} + - v5-{{ checksum ".CACHE-PREFIX" }}-drivers-<< parameters.edition >>- # Key for frontend client built by uberjar-build-frontend step cache-key-frontend: &CacheKeyFrontend - key: v1-{{ checksum ".CACHE-PREFIX" }}-frontend-<< parameters.edition >>-{{ checksum ".FRONTEND-CHECKSUMS" }} + key: v5-{{ checksum ".CACHE-PREFIX" }}-frontend-<< parameters.edition >>-{{ checksum ".FRONTEND-CHECKSUMS" }} # Key for uberjar built by build-uberjar cache-key-uberjar: &CacheKeyUberjar - key: v1-{{ checksum ".CACHE-PREFIX" }}-uberjar-<< parameters.edition >>-{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }} + key: v5-{{ checksum ".CACHE-PREFIX" }}-uberjar-<< parameters.edition >>-{{ checksum ".BACKEND-CHECKSUMS" }}-{{ checksum ".FRONTEND-CHECKSUMS" }} ######################################################################################################################## @@ -415,12 +412,12 @@ commands: fi echo "Created checksums for $(cat << parameters.filename >> | wc -l) files" - run-lein-command: + run-clojure-command: parameters: before-steps: type: steps default: [] - lein-command: + clojure-args: type: string after-steps: type: steps @@ -430,9 +427,9 @@ commands: - restore-be-deps-cache - steps: << parameters.before-steps >> - run: - name: lein << parameters.lein-command >> + name: clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev command: | - lein with-profile +ci,+<< parameters.edition >> << parameters.lein-command >> + clojure << parameters.clojure-args >>:<< parameters.edition >>:<< parameters.edition >>-dev no_output_timeout: 15m - steps: << parameters.after-steps >> - store_test_results: @@ -522,7 +519,7 @@ jobs: ######################################################################################################################## checkout: - executor: clojure-and-node + executor: clojure-and-node-and-browsers steps: - checkout # .BACKEND-CHECKSUMS is every Clojure source file as well as dependency files like deps.edn and plugin manifests @@ -571,7 +568,7 @@ jobs: check-migrations: executor: - clojure-and-node + clojure-and-node-and-browsers steps: - attach-workspace - create-checksum-file: @@ -594,35 +591,51 @@ jobs: ######################################################################################################################## be-deps: - executor: clojure-and-node + executor: clojure-and-node-and-browsers parameters: <<: *Params steps: - attach-workspace # This step is pretty slow, even with the cache, so only run it if project.clj has changed - # TODO -- we should cache the build script deps as well, and driver deps? - run-on-change: - checksum: '{{ checksum "project.clj" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}' + checksum: 'v5-{{ checksum "deps.edn" }}-{{ checksum ".SCRIPTS-DEPS-CHECKSUMS" }}' steps: - restore-be-deps-cache - - run: lein with-profile +include-all-drivers,+cloverage,+junit,+dev,+<< parameters.edition >> deps - - run: | - cd /home/circleci/metabase/metabase/bin/build-mb && clojure -P + - run: + name: Compile Java source file(s) + command: clojure -X:deps prep + - run: + name: Compile driver AOT namespaces + command: cd modules/drivers && clojure -X:deps prep + - run: + name: Fetch dependencies + command: clojure -P -X:dev:ci:ee:ee-dev:drivers:drivers-dev + - run: + name: Fetch dependencies (./bin/build/build-mb) + command: cd /home/circleci/metabase/metabase/bin/build-mb && clojure -P -M:test + # Not sure why this is needed since you would think build-mb would fetch this stuff as well. It doesn't + # seem to fetch everything tho. :shrug: + - run: + name: Fetch dependencies (./bin/build/build-drivers) + command: cd /home/circleci/metabase/metabase/bin/build-drivers && clojure -P -M:test - save_cache: name: Cache backend dependencies <<: *CacheKeyBackendDeps paths: - /home/circleci/.m2 + - /home/circleci/.gitlibs + - /home/circleci/metabase/metabase/java/target/classes + - /home/circleci/metabase/metabase/modules/drivers/sparksql/target/classes - lein: + clojure: parameters: e: type: executor - default: clojure-and-node + default: clojure-and-node-and-browsers before-steps: type: steps default: [] - lein-command: + clojure-args: type: string after-steps: type: steps @@ -640,22 +653,22 @@ jobs: - run-on-change: checksum: '{{ checksum ".BACKEND-CHECKSUMS" }}' steps: - - run-lein-command: + - run-clojure-command: before-steps: << parameters.before-steps >> - lein-command: << parameters.lein-command >> + clojure-args: << parameters.clojure-args >> after-steps: << parameters.after-steps >> edition: << parameters.edition >> - unless: condition: << parameters.skip-when-no-change >> steps: - - run-lein-command: + - run-clojure-command: before-steps: << parameters.before-steps >> - lein-command: << parameters.lein-command >> + clojure-args: << parameters.clojure-args >> after-steps: << parameters.after-steps >> edition: << parameters.edition >> be-linter-reflection-warnings: - executor: clojure-and-node + executor: clojure-and-node-and-browsers steps: - attach-workspace - run-on-change: @@ -671,7 +684,7 @@ jobs: parameters: e: type: executor - default: clojure-and-node + default: clojure-and-node-and-browsers driver: type: string timeout: @@ -702,14 +715,14 @@ jobs: name: Test << parameters.driver >> driver << parameters.description >> environment: DRIVERS: << parameters.driver >> - command: << parameters.extra-env >> lein with-profile +ci,+junit,+ee test + command: << parameters.extra-env >> clojure -X:dev:ci:ee:ee-dev:drivers:drivers-dev:test no_output_timeout: << parameters.timeout >> - store_test_results: path: /home/circleci/metabase/metabase/target/junit - steps: << parameters.after-steps >> test-build-scripts: - executor: clojure-and-node + executor: clojure-and-node-and-browsers steps: - attach-workspace - run-on-change: @@ -753,7 +766,7 @@ jobs: ######################################################################################################################## fe-deps: - executor: clojure-and-node + executor: clojure-and-node-and-browsers steps: - attach-workspace # This step is *really* slow, so we can skip it if yarn.lock hasn't changed since last time we ran it @@ -775,7 +788,7 @@ jobs: - /home/circleci/.cache/Cypress shared-tests-cljs: - executor: clojure-and-node + executor: clojure-and-node-and-browsers steps: - run-yarn-command: command-name: Run Cljs tests for shared/ code @@ -785,7 +798,7 @@ jobs: # Unlike the other build-uberjar steps, this step should be run once overall and the results can be shared between # OSS and EE uberjars. build-uberjar-drivers: - executor: clojure-and-node + executor: clojure-and-node-and-browsers parameters: <<: *Params steps: @@ -819,27 +832,13 @@ jobs: name: Cache the built drivers <<: *CacheKeyDrivers paths: - - /home/circleci/metabase/metabase/modules/drivers/bigquery/target - - /home/circleci/metabase/metabase/modules/drivers/druid/target - - /home/circleci/metabase/metabase/modules/drivers/google/target - - /home/circleci/metabase/metabase/modules/drivers/googleanalytics/target - - /home/circleci/metabase/metabase/modules/drivers/mongo/target - - /home/circleci/metabase/metabase/modules/drivers/oracle/target - - /home/circleci/metabase/metabase/modules/drivers/presto-common/target - - /home/circleci/metabase/metabase/modules/drivers/presto/target - - /home/circleci/metabase/metabase/modules/drivers/presto-jdbc/target - - /home/circleci/metabase/metabase/modules/drivers/redshift/target - - /home/circleci/metabase/metabase/modules/drivers/snowflake/target - - /home/circleci/metabase/metabase/modules/drivers/sparksql/target - - /home/circleci/metabase/metabase/modules/drivers/sqlite/target - - /home/circleci/metabase/metabase/modules/drivers/sqlserver/target - - /home/circleci/metabase/metabase/modules/drivers/vertica/target + - /home/circleci/metabase/metabase/resources/modules # Build the frontend client. parameters.edition determines whether we build the OSS or EE version. build-uberjar-frontend: parameters: <<: *Params - executor: clojure-and-node + executor: clojure-and-node-and-browsers resource_class: large steps: - attach-workspace @@ -864,7 +863,7 @@ jobs: build-uberjar: parameters: <<: *Params - executor: clojure-and-node + executor: clojure-and-node-and-browsers steps: - attach-workspace - run-on-change: @@ -892,7 +891,7 @@ jobs: # INTERACTIVE=false will tell the clojure build scripts not to do interactive retries etc. INTERACTIVE: "false" MB_EDITION: << parameters.edition >> - command: ./bin/build version drivers uberjar + command: ./bin/build version uberjar no_output_timeout: 15m - store_artifacts: path: /home/circleci/metabase/metabase/target/uberjar/metabase.jar @@ -979,38 +978,39 @@ workflows: requires: - checkout - - lein: + - clojure: name: be-tests-<< matrix.edition >> requires: - be-deps e: java-8 - lein-command: with-profile +junit test + clojure-args: -X:dev:ci:test skip-when-no-change: true <<: *Matrix - - lein: + - clojure: name: be-tests-java-11-<< matrix.edition >> requires: - be-deps e: java-11 - lein-command: with-profile +junit test + clojure-args: -X:dev:ci:test skip-when-no-change: true <<: *Matrix - - lein: + - clojure: name: be-tests-java-16-<< matrix.edition >> requires: - be-deps e: java-16 - lein-command: with-profile +junit test + clojure-args: -X:dev:ci:test skip-when-no-change: true <<: *Matrix - - lein: + - clojure: name: be-linter-cloverage requires: - be-deps - lein-command: cloverage --codecov + # TODO FIXME + clojure-args: -X:dev:ee:ee-dev:test:cloverage after-steps: - run: name: Upload code coverage to codecov.io @@ -1144,9 +1144,6 @@ workflows: - run: name: Create temp cacerts file based on bundled JDK one command: cp $JAVA_HOME/lib/security/cacerts /tmp/cacerts-with-presto-ssl.jks - - run: - name: Install openssl - command: apk add openssl - run: name: Capture Presto server self signed CA command: | @@ -1160,7 +1157,7 @@ workflows: - run: name: Import Presto CA into temp cacerts file command: | - keytool -noprompt -import -alias presto -keystore /tmp/cacerts-with-presto-ssl.jks \ + sudo keytool -noprompt -import -alias presto -keystore /tmp/cacerts-with-presto-ssl.jks \ -storepass changeit -file /tmp/presto-ssl-ca.der -trustcacerts after-steps: - run: diff --git a/.dir-locals.el b/.dir-locals.el index 72cadc8d07..d0919155e3 100644 --- a/.dir-locals.el +++ b/.dir-locals.el @@ -21,6 +21,7 @@ (let-404) (macros/case 0) (match 1) + (c/step 1) (mbql.match/match 1) (mt/test-drivers 1) (mt/query 1) @@ -39,6 +40,7 @@ (qp.streaming/streaming-response 1) (prop/for-all 1) (tools.macro/macrolet '(1 (:defn)))))) + (cider-clojure-cli-aliases . "dev:drivers:drivers-dev:ee:ee-dev:build:user") (clojure-indent-style . always-align) ;; if you're using clj-refactor (highly recommended!) (cljr-favor-prefix-notation . nil) @@ -46,4 +48,4 @@ ;; it's nicer to look at code in GH when you don't have to scroll back and forth (fill-column . 118) (clojure-docstring-fill-column . 118) - (cider-preferred-build-tool . lein)))) + (cider-preferred-build-tool . clojure-cli)))) diff --git a/.dockerignore b/.dockerignore index 6891f6c8b2..66307ca562 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,12 +1,27 @@ -.babel_cache/* +.babel_cache docs/* OSX/* target/* +.circleci +.cpcache +.devcontainer +.github +.husky +.lsp +.shadow-cljs +.github +.git +.vscode +hooks/* +test/* +test_config/* +test_modules/* +test_resources/* +node_modules -**node_modules **metabase.jar *.db -.dockerignore Dockerfile +.dockerignore \ No newline at end of file diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1d5a32fa87..ab16313bc1 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -5,6 +5,6 @@ ### Tests - [ ] Run the frontend and Cypress end-to-end tests with `yarn lint && yarn test`) -- [ ] If there are changes to the backend codebase, run the backend tests with `lein test && lein lint` +- [ ] If there are changes to the backend codebase, run the backend tests with `clojure -X:dev:test` - [ ] Sign the [Contributor License Agreement](https://docs.google.com/a/metabase.com/forms/d/1oV38o7b9ONFSwuzwmERRMi9SYrhYeOrkbmNaq9pOJ_E/viewform) (unless it's a tiny documentation change). diff --git a/.github/workflows/backend.yml b/.github/workflows/backend.yml index 5a81b184fa..0b050a73ea 100644 --- a/.github/workflows/backend.yml +++ b/.github/workflows/backend.yml @@ -4,13 +4,9 @@ on: pull_request: push: branches: - - master - - 'release**' - - 'feature**' - tags: - '**' paths: - - '**.clj' + - '**.clj*' - '**.edn' - '**.java' - '**/metabase-plugin.yaml' @@ -35,12 +31,22 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install Clojure CLI + run: | + curl -O https://download.clojure.org/install/linux-install-1.10.3.933.sh && + sudo bash ./linux-install-1.10.3.933.sh - name: Get M2 cache uses: actions/cache@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-eastwood-${{ hashFiles('**/project.clj') }} - - run: lein with-profile +ci eastwood + path: | + ~/.m2 + ~/.gitlibs + key: ${{ runner.os }}-eastwood-${{ hashFiles('**/deps.edn') }} + - name: Compile Java & AOT Sources + run: | + source ./bin/prep.sh && prep_deps + - run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:eastwood + name: Run Eastwood linter be-linter-namespace-decls: runs-on: ubuntu-20.04 @@ -51,9 +57,19 @@ jobs: uses: actions/setup-java@v1 with: java-version: 11 + - name: Install Clojure CLI + run: | + curl -O https://download.clojure.org/install/linux-install-1.10.3.933.sh && + sudo bash ./linux-install-1.10.3.933.sh - name: Get M2 cache uses: actions/cache@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-namespace-decls-${{ hashFiles('**/project.clj') }} - - run: lein with-profile +ci check-namespace-decls + path: | + ~/.m2 + ~/.gitlibs + key: ${{ runner.os }}-namespace-decls-${{ hashFiles('**/deps.edn') }} + - name: Compile Java & AOT Sources + run: | + source ./bin/prep.sh && prep_deps + - run: clojure -X:dev:ee:ee-dev:drivers:drivers-dev:namespace-checker + name: Check ns forms diff --git a/.github/workflows/frontend.yml b/.github/workflows/frontend.yml index 8f499829c3..55aed287f1 100644 --- a/.github/workflows/frontend.yml +++ b/.github/workflows/frontend.yml @@ -4,12 +4,6 @@ on: pull_request: push: branches: - - master - - 'release**' - - 'feature**' - - 'fix**' - - 'ci**' - tags: - '**' paths: - 'frontend/**' diff --git a/.github/workflows/i18n.yml b/.github/workflows/i18n.yml index 04606c99a5..2f4ee2e837 100644 --- a/.github/workflows/i18n.yml +++ b/.github/workflows/i18n.yml @@ -4,13 +4,9 @@ on: pull_request: push: branches: - - master - - 'release**' - - 'feature**' - tags: - '**' paths: - - '**.clj' + - '**.clj*' - '**.js' - '**.jsx' - '.github/workflows/**' @@ -38,8 +34,8 @@ jobs: - name: Install Clojure CLI run: | - curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh && - sudo bash ./linux-install-1.10.1.708.sh + curl -O https://download.clojure.org/install/linux-install-1.10.3.933.sh && + sudo bash ./linux-install-1.10.3.933.sh - run: ./bin/i18n/update-translation-template name: Check i18n tags/make sure template can be built diff --git a/.github/workflows/percy-issue-comment.yml b/.github/workflows/percy-issue-comment.yml index 499c62fbf5..3625581727 100644 --- a/.github/workflows/percy-issue-comment.yml +++ b/.github/workflows/percy-issue-comment.yml @@ -62,7 +62,6 @@ jobs: echo "yarn `yarn --version`" java -version echo "Clojure `clojure -e "(println (clojure-version))"`" - lein --version - name: Get yarn cache uses: actions/cache@v2 @@ -76,7 +75,6 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/project.clj') }}-${{ hashFiles('**/deps.edn') }} - run: yarn install --frozen-lockfile --prefer-offline - - run: lein with-profile +include-all-drivers,+cloverage,+junit,+${{ matrix.edition }} deps - run: ./bin/build - name: Mark with the commit hash diff --git a/.github/workflows/percy.yml b/.github/workflows/percy.yml index 1658e10563..c876436172 100644 --- a/.github/workflows/percy.yml +++ b/.github/workflows/percy.yml @@ -35,15 +35,14 @@ jobs: java-version: 8 - name: Install Clojure CLI run: | - curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh && - sudo bash ./linux-install-1.10.1.708.sh + curl -O https://download.clojure.org/install/linux-install-1.10.3.933.sh && + sudo bash ./linux-install-1.10.3.933.sh - name: Check versions run: | echo "Node.js `node --version`" echo "yarn `yarn --version`" java -version echo "Clojure `clojure -e "(println (clojure-version))"`" - lein --version - name: Get yarn cache uses: actions/cache@v2 @@ -57,7 +56,6 @@ jobs: key: ${{ runner.os }}-m2-${{ hashFiles('**/project.clj') }}-${{ hashFiles('**/deps.edn') }} - run: yarn install --frozen-lockfile --prefer-offline - - run: lein with-profile +include-all-drivers,+cloverage,+junit,+${{ matrix.edition }} deps - run: ./bin/build - name: Mark with the commit hash @@ -89,8 +87,8 @@ jobs: java-version: 8 - name: Install Clojure CLI run: | - curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh && - sudo bash ./linux-install-1.10.1.708.sh + curl -O https://download.clojure.org/install/linux-install-1.10.3.933.sh && + sudo bash ./linux-install-1.10.3.933.sh - name: Check versions run: | echo "Node.js `node --version`" diff --git a/.github/workflows/uberjar.yml b/.github/workflows/uberjar.yml index a4183f2dea..6a38a48f0b 100644 --- a/.github/workflows/uberjar.yml +++ b/.github/workflows/uberjar.yml @@ -3,9 +3,6 @@ name: Uberjar on: push: branches: - - master - - 'release-**' - tags: - '**' paths-ignore: - 'docs/**' @@ -35,15 +32,15 @@ jobs: java-version: 8 - name: Install Clojure CLI run: | - curl -O https://download.clojure.org/install/linux-install-1.10.1.708.sh && - sudo bash ./linux-install-1.10.1.708.sh + curl -O https://download.clojure.org/install/linux-install-1.10.3.933.sh && + sudo bash ./linux-install-1.10.3.933.sh - name: Check versions run: | echo "Node.js `node --version`" echo "yarn `yarn --version`" java -version echo "Clojure `clojure -e "(println (clojure-version))"`" - lein --version + clojure --help | grep Version - name: Get yarn cache uses: actions/cache@v2 @@ -53,12 +50,13 @@ jobs: - name: Get M2 cache uses: actions/cache@v2 with: - path: ~/.m2 - key: ${{ runner.os }}-m2-${{ hashFiles('**/project.clj') }}-${{ hashFiles('**/deps.edn') }} + path: | + ~/.m2 + ~/.gitlibs + key: ${{ runner.os }}-m2-${{ hashFiles('**/deps.edn') }} - run: yarn install --frozen-lockfile --prefer-offline - - run: lein with-profile +include-all-drivers,+cloverage,+junit,+${{ matrix.edition }} deps - - run: ./bin/build + - run: MB_EDITION=${{ matrix.edition }} ./bin/build - name: Mark with the commit hash run: git rev-parse --short HEAD > COMMIT-ID diff --git a/.github/workflows/yaml.yml b/.github/workflows/yaml.yml index 630d5c7ab2..3f5f4a3b15 100644 --- a/.github/workflows/yaml.yml +++ b/.github/workflows/yaml.yml @@ -4,10 +4,6 @@ on: pull_request: push: branches: - - master - - 'release**' - - 'feature**' - tags: - '**' paths: - '**.yml' diff --git a/.gitignore b/.gitignore index 1676473ceb..4c8a9b32a4 100644 --- a/.gitignore +++ b/.gitignore @@ -50,6 +50,8 @@ /resources/frontend_client/index.html /resources/frontend_client/public.html /resources/i18n/*.edn +/resources/license-backend-third-party.txt +/resources/license-frontend-third-party.txt /resources/namespaces.edn /resources/sample-dataset.db.trace.db /resources/version.properties diff --git a/Dockerfile b/Dockerfile index 4c1983d9d3..9793b01718 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,65 +2,31 @@ # STAGE 1.1: builder frontend ################### -FROM metabase/ci:java-11-lein-2.9.6-clj-1.10.3.822-04-22-2021 as frontend +FROM metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers as frontend ARG MB_EDITION=oss -WORKDIR /app/source +WORKDIR /home/circleci -COPY . . +COPY --chown=circleci . . RUN NODE_ENV=production MB_EDITION=$MB_EDITION yarn --frozen-lockfile && yarn build && bin/i18n/build-translation-resources -################### -# STAGE 1.2: backend deps -################### - -FROM metabase/ci:java-11-lein-2.9.6-clj-1.10.3.822-04-22-2021 as backend - -WORKDIR /app/source - -# backend dependencies -COPY project.clj . -RUN lein deps :tree - -################### -# STAGE 1.3: drivers -################### - -FROM metabase/ci:java-11-lein-2.9.6-clj-1.10.3.822-04-22-2021 as drivers - -ARG MB_EDITION=oss - -WORKDIR /app/source - -COPY --from=backend /root/.m2/repository/. /root/.m2/repository/. - -# add the rest of the source -COPY . . - -# build the app -RUN INTERACTIVE=false MB_EDITION=$MB_EDITION sh bin/build-drivers.sh - ################### # STAGE 1.4: main builder ################### -FROM metabase/ci:java-11-lein-2.9.6-clj-1.10.3.822-04-22-2021 as builder +FROM metabase/ci:circleci-java-11-clj-1.10.3.929-07-27-2021-node-browsers as builder ARG MB_EDITION=oss -WORKDIR /app/source +WORKDIR /home/circleci # try to reuse caching as much as possible -COPY --from=frontend /root/.m2/repository/. /root/.m2/repository/. -COPY --from=frontend /app/source/. . -COPY --from=backend /root/.m2/repository/. /root/.m2/repository/. -COPY --from=backend /app/source/. . -COPY --from=drivers /root/.m2/repository/. /root/.m2/repository/. -COPY --from=drivers /app/source/. . +COPY --from=frontend /home/circleci/.m2/repository/. /home/circleci/.m2/repository/. +COPY --from=frontend /home/circleci/. . # build the app -RUN INTERACTIVE=false MB_EDITION=$MB_EDITION bin/build version uberjar +RUN INTERACTIVE=false MB_EDITION=$MB_EDITION bin/build version drivers uberjar # ################### # # STAGE 2: runner @@ -83,7 +49,7 @@ RUN apk upgrade && apk add --update-cache --no-cache bash ttf-dejavu fontconfig mkdir -p /plugins && chmod a+rwx /plugins # add Metabase script and uberjar -COPY --from=builder /app/source/target/uberjar/metabase.jar /app/ +COPY --from=builder /home/circleci/target/uberjar/metabase.jar /app/ COPY bin/docker/run_metabase.sh /app/ # expose our default runtime port diff --git a/OSX/Metabase/Backend/ResetPasswordTask.m b/OSX/Metabase/Backend/ResetPasswordTask.m index 1260ff6313..b675fd1e06 100644 --- a/OSX/Metabase/Backend/ResetPasswordTask.m +++ b/OSX/Metabase/Backend/ResetPasswordTask.m @@ -23,58 +23,47 @@ - (void)resetPasswordForEmailAddress:(NSString *)emailAddress success:(void (^)( // first, we need to stop the main Metabase task so we can access the DB NSLog(@"Stopping Metabase task in order to reset password..."); [[AppDelegate instance] stopMetabaseTask]; - + self.task = [[NSTask alloc] init]; - - // time travelers from the future: this is hardcoded since I'm the only one who works on this. I give you permission to fix it - Cam - #define DEBUG_RUN_LEIN_TASK 0 - - #if DEBUG_RUN_LEIN_TASK - self.task.environment = @{@"MB_DB_FILE": DBPath()}; - self.task.currentDirectoryPath = @"/Users/cam/metabase"; - self.task.launchPath = @"/usr/local/bin/lein"; - self.task.arguments = @[@"run", @"reset-password", emailAddress]; - NSLog(@"Launching ResetPasswordTask\nMB_DB_FILE='%@' lein run reset-password %@", DBPath(), emailAddress); - #else - self.task.environment = @{@"MB_DB_FILE": DBPath()}; - self.task.launchPath = JREPath(); - self.task.arguments = @[@"-Djava.awt.headless=true", // this prevents the extra java icon from popping up in the dock when running - @"-Xverify:none", // disable bytecode verification for faster launch speed, not really needed here since JAR is packaged as part of signed .app - @"-jar", UberjarPath(), - @"reset-password", emailAddress]; - NSLog(@"Launching ResetPasswordTask\nMB_DB_FILE='%@' %@ -jar %@ reset-password %@", DBPath(), JREPath(), UberjarPath(), emailAddress); - #endif - + + self.task.environment = @{@"MB_DB_FILE": DBPath()}; + self.task.launchPath = JREPath(); + self.task.arguments = @[@"-Djava.awt.headless=true", // this prevents the extra java icon from popping up in the dock when running + @"-Xverify:none", // disable bytecode verification for faster launch speed, not really needed here since JAR is packaged as part of signed .app + @"-jar", UberjarPath(), + @"reset-password", emailAddress]; + NSLog(@"Launching ResetPasswordTask\nMB_DB_FILE='%@' %@ -jar %@ reset-password %@", DBPath(), JREPath(), UberjarPath(), emailAddress); + __weak ResetPasswordTask *weakSelf = self; self.task.terminationHandler = ^(NSTask *task) { NSLog(@"ResetPasswordTask terminated with status: %d", task.terminationStatus); [weakSelf terminate]; - + dispatch_async(dispatch_get_main_queue(), ^{ if (!task.terminationStatus && weakSelf.output.length >= 38) { // should be of format _<36-char-uuid>, e.g. "1_b20466b9-1f5b-488d-8ab6-5039107482f8" successBlock(weakSelf.output); } else { errorBlock(weakSelf.output.length ? weakSelf.output : @"An unknown error has occured."); } - + // now restart the main Metabase task NSLog(@"Reset password complete, restarting Metabase task..."); [[AppDelegate instance] startMetabaseTask]; }); }; - + [self.task launch]; }); } - (void)readHandleDidRead:(NSString *)message { NSLog(@"[PasswordResetTask] %@", message); - + /// output comes back like "STATUS [[[message]]]" NSRegularExpression *regex = [NSRegularExpression regularExpressionWithPattern:@"^(?:(?:OK)||(?:FAIL))\\s+\\[\\[\\[(.+)\\]\\]\\]\\s*$" options:NSRegularExpressionAnchorsMatchLines|NSRegularExpressionAllowCommentsAndWhitespace error:NULL]; if (![regex numberOfMatchesInString:message options:0 range:NSMakeRange(0, message.length)]) return; - + NSString *result = [regex stringByReplacingMatchesInString:message options:0 range:NSMakeRange(0, message.length) withTemplate:@"$1"]; if (result) { self.output = result; diff --git a/backend/junit/test/metabase/junit.clj b/backend/junit/test/metabase/junit.clj deleted file mode 100644 index 49bb3aec0f..0000000000 --- a/backend/junit/test/metabase/junit.clj +++ /dev/null @@ -1,73 +0,0 @@ -(ns metabase.junit - "Formatter for JUnit test output for CI." - (:require [clojure.pprint :as pp] - [clojure.string :as str] - [medley.core :as m] - [metabase.util :as u] - [pjstadig.print :as p] - [test-report-junit-xml.core :as junit-xml] - [test_report_junit_xml.shaded.clojure.data.xml :as xml])) - -(defn- escape-unprintable-characters - [s] - (str/join (for [c s] - (if (and (Character/isISOControl c) - (not (Character/isWhitespace c))) - (format "&#%d;" (int c)) - c)))) - -(defn- decolorize-and-escape - "Remove ANSI color escape sequences, then encode things as character entities as needed" - [s] - (-> s u/decolorize escape-unprintable-characters)) - -(defn- event-description [{:keys [file line context message]}] - (str - (format "%s:%d" file line) - (when (seq context) - (str "\n" (str/join " " (reverse context)))) - (when message - (str "\n" message)))) - -(defn- print-expected [expected actual] - (p/rprint "expected: ") - (pp/pprint expected) - (p/rprint " actual: ") - (pp/pprint actual) - (p/clear)) - -(defn- result-output [{:keys [expected actual diffs message], :as event}] - (let [s (with-out-str - (println (event-description event)) - ;; this code is adapted from `pjstadig.util` - (p/with-pretty-writer - (fn [] - (if (seq diffs) - (doseq [[actual [a b]] diffs] - (print-expected expected actual) - (p/rprint " diff:") - (if a - (do (p/rprint " - ") - (pp/pprint a) - (p/rprint " + ")) - (p/rprint " + ")) - (when b - (pp/pprint b)) - (p/clear)) - (print-expected expected actual)))))] - (decolorize-and-escape s))) - -(defmulti format-result - {:arglists '([event])} - :type) - -(defmethod format-result :default - [event] - (-> (#'junit-xml/format-result event) - (m/update-existing-in [:attrs :message] decolorize-and-escape) - (m/update-existing :content (comp xml/cdata decolorize-and-escape)))) - -(defmethod format-result :fail - [event] - {:tag :failure - :content (xml/cdata (result-output event))}) diff --git a/bin/.dir-locals.el b/bin/.dir-locals.el new file mode 100644 index 0000000000..5106b70d73 --- /dev/null +++ b/bin/.dir-locals.el @@ -0,0 +1,2 @@ +((nil . ((ftf-project-finders . (ftf-get-top-git-dir)))) ; tell find-things-fast not assume this folder is the project root. + (clojure-mode . (cider-clojure-cli-aliases . "dev"))) diff --git a/bin/build b/bin/build index 7833eef848..d360de0214 100755 --- a/bin/build +++ b/bin/build @@ -2,8 +2,18 @@ set -euo pipefail +# switch to project root directory if we're not already there +script_directory=`dirname "${BASH_SOURCE[0]}"` +cd "$script_directory/.." + source "./bin/check-clojure-cli.sh" check_clojure_cli +source "./bin/clear-outdated-cpcaches.sh" +clear_outdated_cpcaches + +source "./bin/prep.sh" +prep_deps + cd bin/build-mb clojure -M -m build $@ diff --git a/bin/build-driver.sh b/bin/build-driver.sh index ed438085bb..8380131c24 100755 --- a/bin/build-driver.sh +++ b/bin/build-driver.sh @@ -5,12 +5,22 @@ set -eo pipefail driver="$1" if [ ! "$driver" ]; then - echo "Usage: ./bin/build-driver.sh [driver]" + echo "Usage: ./bin/build-driver.sh [edition]" exit -1 fi +# switch to project root directory if we're not already there +script_directory=`dirname "${BASH_SOURCE[0]}"` +cd "$script_directory/.." + source "./bin/check-clojure-cli.sh" check_clojure_cli +source "./bin/clear-outdated-cpcaches.sh" +clear_outdated_cpcaches + +source "./bin/prep.sh" +prep_deps + cd bin/build-drivers -clojure -M -m build-driver "$driver" +clojure -M -m build-driver $@ diff --git a/bin/build-drivers.sh b/bin/build-drivers.sh index f50aac9d22..bea160d25b 100755 --- a/bin/build-drivers.sh +++ b/bin/build-drivers.sh @@ -2,8 +2,18 @@ set -euo pipefail +# switch to project root directory if we're not already there +script_directory=`dirname "${BASH_SOURCE[0]}"` +cd "$script_directory/.." + source "./bin/check-clojure-cli.sh" check_clojure_cli +source "./bin/clear-outdated-cpcaches.sh" +clear_outdated_cpcaches + +source "./bin/prep.sh" +prep_deps + cd bin/build-drivers clojure -M -m build-drivers $@ diff --git a/bin/build-drivers/README.md b/bin/build-drivers/README.md index 0033455aba..8862c9502d 100644 --- a/bin/build-drivers/README.md +++ b/bin/build-drivers/README.md @@ -7,7 +7,7 @@ There are three main entrypoints. Shell script wrappers are provided for conveni ### `build-drivers` -Builds *all* drivers as needed. If drivers were recently built and no relevant source code changed, skips rebuild. +Builds *all* drivers as needed. ``` cd bin/build-drivers diff --git a/bin/build-drivers/deps.edn b/bin/build-drivers/deps.edn index 0231cb4d62..d54f3d75fe 100644 --- a/bin/build-drivers/deps.edn +++ b/bin/build-drivers/deps.edn @@ -1,17 +1,29 @@ {:paths ["src"] :deps - {common/common {:local/root "../common"} - cheshire/cheshire {:mvn/version "5.8.1"} - commons-codec/commons-codec {:mvn/version "1.14"} - hiccup/hiccup {:mvn/version "1.0.5"} - io.forward/yaml {:mvn/version "1.0.9"} ; don't upgrade to 1.0.10 -- doesn't work on Java 8 (!) - leiningen/leiningen {:mvn/version "2.9.5"} ; for parsing Leiningen projects - org.flatland/ordered {:mvn/version "1.5.9"} ; used by io.forward/yaml -- need the newer version - stencil/stencil {:mvn/version "0.5.0"}} + {common/common {:local/root "../common"} + com.github.seancorfield/depstar {:git/tag "v2.1.267", :git/sha "1a45f79"} + cheshire/cheshire {:mvn/version "5.8.1"} + commons-codec/commons-codec {:mvn/version "1.14"} + hiccup/hiccup {:mvn/version "1.0.5"} + io.forward/yaml {:mvn/version "1.0.9"} ; don't upgrade to 1.0.10 -- doesn't work on Java 8 (!) + io.github.clojure/tools.build {:git/tag "v0.1.6", :git/sha "5636e61"} + org.clojure/tools.deps.alpha {:mvn/version "0.12.985"} + org.flatland/ordered {:mvn/version "1.5.9"} ; used by io.forward/yaml -- need the newer version + stencil/stencil {:mvn/version "0.5.0"} + ;; local source + metabase/metabase-core {:local/root "../.."} + metabase/driver-modules {:local/root "../../modules/drivers"}} + + :jvm-opts + ["-XX:-OmitStackTraceInFastThrow"] :aliases - {:test {:extra-paths ["test"] - :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" - :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}} - :main-opts ["-m" "cognitect.test-runner"]}}} + {:dev + {:extra-paths ["test"]} + + :test + {:extra-paths ["test"] + :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" + :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"}} + :main-opts ["-m" "cognitect.test-runner"]}}} diff --git a/bin/build-drivers/src/build_drivers.clj b/bin/build-drivers/src/build_drivers.clj index 2dd1ba2d3b..c494ce944d 100644 --- a/bin/build-drivers/src/build_drivers.clj +++ b/bin/build-drivers/src/build_drivers.clj @@ -7,12 +7,15 @@ (defn- all-drivers [] (->> (.listFiles (io/file (u/filename u/project-root-directory "modules" "drivers"))) - (filter (fn [^File d] - (and (.isDirectory d) ;; watch for errant DS_Store files on os_x - ;; only consider a directory to be a driver if it contains a lein or deps build file - (some true? (map (fn [f] - (.exists (io/file d f))) ["project.clj" "deps.edn"]))))) - (map (comp keyword #(.getName %))))) + (filter (fn [^File d] ; + (and + ;; watch for errant DS_Store files on os_x + (.isDirectory d) + ;; ignore stuff like .cpcache + (not (.isHidden d)) + ;; only consider a directory to be a driver if it contains a lein or deps build file + (.exists (io/file d "deps.edn"))))) + (map (comp keyword #(.getName ^File %))))) (defn build-drivers! [edition] (let [edition (or edition :oss)] diff --git a/bin/build-drivers/src/build_drivers/build_driver.clj b/bin/build-drivers/src/build_drivers/build_driver.clj index 498e51f175..5bbb617668 100644 --- a/bin/build-drivers/src/build_drivers/build_driver.clj +++ b/bin/build-drivers/src/build_drivers/build_driver.clj @@ -1,168 +1,24 @@ (ns build-drivers.build-driver - "Logic for building a single driver." - (:require [build-drivers.checksum :as checksum] - [build-drivers.common :as c] - [build-drivers.install-driver-locally :as install-locally] - [build-drivers.metabase :as metabase] - [build-drivers.plugin-manifest :as manifest] - [build-drivers.strip-and-compress :as strip-and-compress] + (:require [build-drivers.common :as c] + [build-drivers.compile-source-files :as compile-source-files] + [build-drivers.copy-source-files :as copy-source-files] + [build-drivers.create-uberjar :as create-uberjar] [build-drivers.verify :as verify] - [clojure.string :as str] - [colorize.core :as colorize] - [environ.core :as env] [metabuild-common.core :as u])) -(defn- copy-driver! - "Copy the driver JAR from its `target/` directory to `resources/modules`/." - [driver] - (u/step (format "Copy %s driver uberjar from %s -> %s" - driver - (u/assert-file-exists (c/driver-jar-build-path driver)) - (c/driver-jar-destination-path driver)) - (u/delete-file-if-exists! (c/driver-jar-destination-path driver)) - (u/create-directory-unless-exists! c/driver-jar-destination-directory) - (u/copy-file! (c/driver-jar-build-path driver) - (c/driver-jar-destination-path driver)))) - -(defn- clean-driver-artifacts! - "Delete built JARs of `driver`." - [driver] - (u/step (format "Delete %s driver artifacts" driver) - (u/delete-file-if-exists! (c/driver-target-directory driver)) +(defn clean! [driver] + (u/step "Clean" + (u/delete-file-if-exists! (c/compiled-source-target-dir driver)) (u/delete-file-if-exists! (c/driver-jar-destination-path driver)))) -(defn- clean-parents! - "Delete built JARs and local Maven installations of the parent drivers of `driver`." - [driver] - (u/step (format "Clean %s parent driver artifacts" driver) - (doseq [parent (manifest/parent-drivers driver)] - (clean-driver-artifacts! parent) - (install-locally/clean! parent) - (clean-parents! parent)))) - -(defn- clean-all! - "Delete all artifacts relating to building `driver`, including the driver JAR itself and installed - `metabase-core`/Metabase uberjar and any parent driver artifacts." - [driver] - (u/step "Clean all" - (clean-driver-artifacts! driver) - (clean-parents! driver) - (metabase/clean-metabase!))) - -(declare build-driver!) - -(defn- build-parents! - "Build and install to the local Maven repo any parent drivers of `driver` (e.g. `:google` is a parent of `:bigquery`). - The driver must be built as an uberjar so we can remove duplicate classes during the `strip-and-compress` stage; it - must be installed as a library so we can use it as a `:provided` dependency when building the child driver." - [driver edition] - (u/step (format "Build %s parent drivers" driver) - (when-let [parents (not-empty (manifest/parent-drivers driver))] - (doseq [parent parents] - (build-parents! parent edition) - (install-locally/install-locally! parent edition) - (build-driver! parent edition)) - (u/announce "%s parents built successfully." driver)))) - -(defn- build-uberjar! [driver edition] - (u/step (format "Build %s uberjar (%s edition)" driver edition) - (u/delete-file-if-exists! (c/driver-target-directory driver)) - (u/sh {:dir (c/driver-project-dir driver)} "lein" "clean") - (u/sh {:dir (c/driver-project-dir driver) - :env {"LEIN_SNAPSHOTS_IN_RELEASE" "true" - "HOME" (env/env :user-home) - "PATH" (env/env :path) - "JAVA_HOME" (env/env :java-home)}} - "lein" "with-profile" (format "+%s" (name edition)) "uberjar") - (strip-and-compress/strip-and-compress-uberjar! driver) - (u/announce "%s uberjar (%s edition) built successfully." driver edition))) - -(defn- build-and-verify! - "Build `driver` and verify the built JAR. This function ignores any existing artifacts and will always rebuild." - [driver edition] - {:pre [(#{:oss :ee} edition)]} - (u/step (str/join " " [(colorize/green "Build") - (colorize/yellow driver) - (colorize/green "driver") - (colorize/yellow (format "(%s edition)" edition))]) - (clean-driver-artifacts! driver) - (u/step (format "Build %s driver (%s edition) prerequisites if needed" driver edition) - (metabase/build-metabase!) - (build-parents! driver edition)) - (build-uberjar! driver edition) - (copy-driver! driver) - (verify/verify-driver driver) - (u/step (format "Save checksum for %s driver (%s edition) to %s" - driver edition (c/driver-checksum-filename driver)) - (let [filename (c/driver-checksum-filename driver) - checksum (checksum/driver-checksum driver edition)] - (spit filename checksum) - (u/announce "Wrote checksum %s to file %s" (pr-str checksum) filename))))) - -(defn- driver-checksum-matches? - "Check whether the saved checksum for the driver from the last build is the same as the current one. If so, we don't - need to build again. This checksum is based on driver sources as well as the checksums for Metabase sources and - parent drivers." - [driver edition] - (u/step (format "Determine whether %s driver (%s edition) source files have changed since last build" driver edition) - (let [existing-checksum (checksum/existing-driver-checksum driver)] - (cond - (not existing-checksum) - (do - (u/announce "No previous checksum. Need to rebuild driver") - false) - - (= existing-checksum (checksum/driver-checksum driver edition)) - (do - (u/announce "Checksum is the same. Do not need to rebuild driver.") - true) - - :else - (do - (u/announce "Checksum is different. Need to rebuild driver.") - false))))) - (defn build-driver! - "Build `driver`, if needed." [driver edition] - {:pre [(#{:oss :ee nil} edition)]} - (let [edition (or edition :oss)] - (u/step (str/join " " [(colorize/green "Build") - (colorize/yellow driver) - (colorize/green "driver") - (colorize/yellow (format "(%s edition)" edition)) - (colorize/green "if needed")]) - ;; When we build a driver, we save a checksum of driver source code + metabase source code + parent drivers - ;; alongside the built driver JAR. The next time this script is called, we recalculate that checksum -- if the - ;; current checksum matches the saved one associated with the built driver JAR, we do not need to rebuild the - ;; driver. If anything relevant has changed, we have to rebuild the driver. - (if (driver-checksum-matches? driver edition) - ;; even if we're not rebuilding the driver, copy the artifact from `modules/drivers//target/uberjar/` - ;; to `resources/modules` so we can be sure we have the most up-to-date version there. - (try - (copy-driver! driver) - (verify/verify-driver driver) - ;; if verification fails, delete all the existing artifacts and just rebuild the driver from scratch. - (catch Throwable e - (u/error "Error verifying existing driver:\n%s" (pr-str e)) - (u/announce "Deleting existing driver artifacts and rebuilding.") - (clean-driver-artifacts! driver) - (build-driver! driver edition))) - ;; if checksum does not match, build and verify the driver - (try - (build-and-verify! driver edition) - ;; if building fails, clean everything, including metabase-core, the metabase uberjar, and parent - ;; dependencies, *then* retry. - (catch Throwable e - (u/announce "Cleaning ALL and retrying...") - (clean-all! driver) - (try - (build-and-verify! driver edition) - ;; if building the driver failed again, even after cleaning, delete anything that was built and then - ;; give up. - (catch Throwable e - (u/safe-println (colorize/red (format "Failed to build %s driver." driver))) - (clean-driver-artifacts! driver) - (throw e)))))) - ;; if we make it this far, we've built the driver successfully. - (u/announce "Success.")))) + (let [edition (or edition :oss) + start-time-ms (System/currentTimeMillis)] + (u/step (format "Build driver %s (edition = %s)" driver edition) + (clean! driver) + (copy-source-files/copy-source-files! driver edition) + (compile-source-files/compile-clojure-source-files! driver edition) + (create-uberjar/create-uberjar! driver edition) + (u/announce "Built %s driver in %d ms." driver (- (System/currentTimeMillis) start-time-ms)) + (verify/verify-driver driver)))) diff --git a/bin/build-drivers/src/build_drivers/checksum.clj b/bin/build-drivers/src/build_drivers/checksum.clj deleted file mode 100644 index a46e91e415..0000000000 --- a/bin/build-drivers/src/build_drivers/checksum.clj +++ /dev/null @@ -1,82 +0,0 @@ -(ns build-drivers.checksum - "Shared code for calculating and reading hex-encoded MD5 checksums for relevant files." - (:require [build-drivers.common :as c] - [build-drivers.plugin-manifest :as manifest] - [clojure.java.io :as io] - [clojure.string :as str] - [colorize.core :as colorize] - [metabuild-common.core :as u]) - (:import org.apache.commons.codec.digest.DigestUtils)) - -(defn checksum-from-file - "Read a saved MD5 hash checksum from a file." - [filename] - (u/step (format "Read saved checksum from %s" filename) - (let [file (io/file filename)] - (if-not (.exists file) - (u/announce "%s does not exist" filename) - (or (when-let [[checksum-line] (not-empty (str/split-lines (slurp file)))] - (when-let [[_ checksum-hex] (re-matches #"(^(?:\w+-)?[0-9a-f]{32}).*$" checksum-line)] - (u/safe-println (format "Saved checksum is %s" (colorize/cyan checksum-hex))) - checksum-hex)) - (u/error (format "Checksum file %s exists, but does not contain a valid checksum" filename))))))) - -;;; -------------------------------------------- Metabase source checksum -------------------------------------------- - -(defn- metabase-source-paths [] - (sort - (cons - (u/filename u/project-root-directory "project.clj") - (mapcat (fn [dir] - (try - (u/find-files dir (fn [s] - (or (str/ends-with? s ".clj") - (str/ends-with? s ".cljc")))) - (catch Throwable _ - []))) - [(u/filename u/project-root-directory "src") - (u/filename u/project-root-directory "enterprise" "backend" "src") - (u/filename u/project-root-directory "shared" "src")])))) - -(defn metabase-source-checksum - "Checksum of Metabase backend source files and `project.clj`." - ^String [] - (let [paths (metabase-source-paths)] - (u/step (format "Calculate checksum for %d Metabase source files" (count paths)) - (let [checksum (DigestUtils/md5Hex (str/join (map slurp paths)))] - (u/safe-println (format "Current checksum of Metabase files is %s" (colorize/cyan checksum))) - checksum)))) - - -;;; ---------------------------------------------- Driver source files ----------------------------------------------- - -(defn existing-driver-checksum - "Checksum from the relevant sources from last time we built `driver`." - [driver] - (checksum-from-file (c/driver-checksum-filename driver))) - -(defn- driver-source-paths - "Returns sequence of the source filenames for `driver`." - [driver] - (u/find-files (c/driver-project-dir driver) - (fn [path] - (or (and (str/ends-with? path ".clj") - (not (str/starts-with? path (u/filename (c/driver-project-dir driver) "test")))) - (str/ends-with? path ".yaml"))))) - -(defn driver-checksum - "The driver checksum is based on a checksum of all the driver source files (`.clj` files and the plugin manifest YAML - file) combined with the checksums for `metabase-core` *and* the parent drivers. After building a driver, we save - this checksum. Next time the script is ran, we recalculate the checksum to determine whether anything relevant has - changed -- if it has, and the current checksum doesn't match the saved one, we need to rebuild the driver." - ^String [driver edition] - (let [source-paths (driver-source-paths driver)] - (u/step (format "Calculate checksum for %d files: %s ..." (count source-paths) (first source-paths)) - (let [checksum (str - (c/edition-checksum-prefix driver edition) - (DigestUtils/md5Hex (str/join (concat [(metabase-source-checksum)] - (map #(driver-checksum % edition) - (manifest/parent-drivers driver)) - (map slurp (driver-source-paths driver))))))] - (u/safe-println (format "Current checksum of %s driver (%s edition) is %s" driver edition (colorize/cyan checksum))) - checksum)))) diff --git a/bin/build-drivers/src/build_drivers/common.clj b/bin/build-drivers/src/build_drivers/common.clj index 36de81a789..86f099024b 100644 --- a/bin/build-drivers/src/build_drivers/common.clj +++ b/bin/build-drivers/src/build_drivers/common.clj @@ -1,33 +1,17 @@ (ns build-drivers.common - "Shared constants and functions related to source and artifact paths used throughout this code." - (:require [environ.core :as env] - [leiningen.core.project :as lein.project] + (:require [clojure.tools.deps.alpha :as deps] [metabuild-common.core :as u])) -(def ^String maven-repository-path - (u/filename (env/env :user-home) ".m2" "repository")) - -;;; -------------------------------------------------- Driver Paths -------------------------------------------------- - (defn driver-project-dir "e.g. \"/home/cam/metabase/modules/drivers/redshift\"" - [driver] + ^String [driver] (u/filename u/project-root-directory "modules" "drivers" (name driver))) (defn driver-jar-name "e.g. \"redshift.metabase-driver.jar\"" - [driver] + ^String [driver] (format "%s.metabase-driver.jar" (name driver))) -(defn driver-target-directory - [driver] - (u/filename (driver-project-dir driver) "target")) - -(defn driver-jar-build-path - "e.g. \"/home/cam/metabase/modules/drivers/redshift/target/uberjar/redshift.metabase-driver.jar\"" - [driver] - (u/filename (driver-target-directory driver) "uberjar" (driver-jar-name driver))) - (def ^String driver-jar-destination-directory (u/filename u/project-root-directory "resources" "modules")) @@ -36,42 +20,13 @@ ^String [driver] (u/filename driver-jar-destination-directory (driver-jar-name driver))) -(defn- lein-project-map - "Read the `project.clj` file for `driver` and return it as a map." - [driver & profiles] - (let [project-filename (u/assert-file-exists (u/filename (driver-project-dir driver) "project.clj"))] - (lein.project/read project-filename profiles))) - -(defn has-edition-profile? - "Whether `driver` has a separate profile for `edition`, e.g. `:ee`. This means this version of the driver is different - from other versions of the driver (e.g. :ee Oracle ships with the non-free Oracle JDBC driver, :oss does not)." - [driver edition] - (let [has-profile? (boolean - (contains? (:profiles (lein-project-map driver)) edition))] - (u/safe-println (format "%s %s have a separate %s profile" driver (if has-profile? "DOES" "DOES NOT") edition)) - has-profile?)) - -(defn edition-checksum-prefix - "Prefix to add to checksums of driver for `edition` -- normally this is `nil`, but if the driver has a specific - profile for `edition` (e.g. Oracle has a different profile for `:ee` builds) this is a prefix to make the checksum - different from the normal one." - [driver edition] - (when (has-edition-profile? driver edition) - (format "%s-" (name edition)))) - -(defn driver-checksum-filename - "e.g. \"/home/cam/metabase/modules/drivers/redshift/target/checksum.md5\"" - [driver] - (u/filename (driver-project-dir driver) "target" "checksum.md5")) - -(defn driver-plugin-manifest-filename - "e.g. \"/home/cam/metabase/modules/drivers/bigquery/resources/plugin-manifest.yaml\"" - [driver] - (u/filename (driver-project-dir driver) "resources" "metabase-plugin.yaml")) - +(defn compiled-source-target-dir [driver] + (u/filename (driver-project-dir driver) "target" "jar")) -;;; ------------------------------------------ Metabase Local Install Paths ------------------------------------------ +(defn driver-edn-filename [driver] + (u/filename (driver-project-dir driver) "deps.edn")) -(def ^String metabase-uberjar-path - "e.g. \"home/cam/metabase/target/uberjar/metabase.jar\"" - (u/filename u/project-root-directory "target" "uberjar" "metabase.jar")) +(defn driver-edn [driver edition] + (let [edn (deps/merge-edns ((juxt :root-edn :project-edn) (deps/find-edn-maps (driver-edn-filename driver)))) + combined (deps/combine-aliases edn #{edition})] + (deps/tool edn combined))) diff --git a/bin/build-drivers/src/build_drivers/compile_source_files.clj b/bin/build-drivers/src/build_drivers/compile_source_files.clj new file mode 100644 index 0000000000..ac33a6982a --- /dev/null +++ b/bin/build-drivers/src/build_drivers/compile_source_files.clj @@ -0,0 +1,52 @@ +(ns build-drivers.compile-source-files + (:require [build-drivers.common :as c] + [clojure.java.io :as io] + [clojure.tools.namespace.dependency :as ns.deps] + [clojure.tools.namespace.find :as ns.find] + [clojure.tools.namespace.parse :as ns.parse] + [metabuild-common.core :as u])) + +(defn driver-source-paths [driver edition] + (for [path (:paths (c/driver-edn driver edition))] + (u/filename (c/driver-project-dir driver) path))) + +(defn- dependencies-graph + "Return a `clojure.tools.namespace` dependency graph of namespaces named by `ns-symbol`." + [ns-decls] + (reduce + (fn [graph ns-decl] + (let [ns-symbol (ns.parse/name-from-ns-decl ns-decl)] + (reduce + (fn [graph dep] + (ns.deps/depend graph ns-symbol dep)) + graph + (ns.parse/deps-from-ns-decl ns-decl)))) + (ns.deps/graph) + ns-decls)) + +;; topologically sort the namespaces so we don't end up with weird compilation issues. +(defn source-path-namespaces [source-paths] + (let [ns-decls (mapcat + (comp ns.find/find-ns-decls-in-dir io/file) + source-paths) + ns-symbols (set (map ns.parse/name-from-ns-decl ns-decls))] + (->> (dependencies-graph ns-decls) + ns.deps/topo-sort + (filter ns-symbols)))) + +(defn compile-clojure-source-files! [driver edition] + (u/step "Compile clojure source files" + (let [start-time-ms (System/currentTimeMillis) + source-paths (driver-source-paths driver edition) + target-dir (c/compiled-source-target-dir driver) + namespaces (source-path-namespaces source-paths)] + (u/announce "Compiling Clojure source files in %s to %s" (pr-str source-paths) target-dir) + (u/create-directory-unless-exists! target-dir) + (u/announce "Compiling namespaces %s" (pr-str namespaces)) + (binding [*compile-path* target-dir] + (doseq [a-namespace namespaces] + (#'clojure.core/serialized-require a-namespace) + (compile a-namespace))) + (u/announce "Compiled %d namespace(s) in %d ms." + (count namespaces) + (- (System/currentTimeMillis) start-time-ms))))) diff --git a/bin/build-drivers/src/build_drivers/copy_source_files.clj b/bin/build-drivers/src/build_drivers/copy_source_files.clj new file mode 100644 index 0000000000..2be62241a9 --- /dev/null +++ b/bin/build-drivers/src/build_drivers/copy_source_files.clj @@ -0,0 +1,17 @@ +(ns build-drivers.copy-source-files + (:require [build-drivers.common :as c] + [clojure.tools.build.api :as build] + [metabuild-common.core :as u])) + +(defn copy-source-files! [driver edition] + (u/step (format "Copy %s source files" driver) + (let [start-time-ms (System/currentTimeMillis) + dirs (for [path (:paths (c/driver-edn driver edition))] + (u/filename (c/driver-project-dir driver) path))] + (u/announce "Copying files in %s" (pr-str dirs)) + (build/copy-dir + {:src-dirs dirs + :target-dir (c/compiled-source-target-dir driver)}) + (u/announce "Copied files in %d directories in %d ms." + (count dirs) + (- (System/currentTimeMillis) start-time-ms))))) diff --git a/bin/build-drivers/src/build_drivers/create_uberjar.clj b/bin/build-drivers/src/build_drivers/create_uberjar.clj new file mode 100644 index 0000000000..c4f08ca203 --- /dev/null +++ b/bin/build-drivers/src/build_drivers/create_uberjar.clj @@ -0,0 +1,62 @@ +(ns build-drivers.create-uberjar + (:require [build-drivers.common :as c] + [clojure.java.io :as io] + [clojure.tools.build.api :as build] + [clojure.tools.deps.alpha :as deps] + [clojure.tools.deps.alpha.util.dir :as deps.dir] + [colorize.core :as colorize] + [hf.depstar.api :as depstar] + [metabuild-common.core :as u])) + +(defn driver-basis [driver edition] + (let [edn (c/driver-edn driver edition)] + (binding [deps.dir/*the-dir* (io/file (c/driver-project-dir driver))] + (deps/calc-basis edn)))) + +(defonce metabase-core-edn + (deps/merge-edns + ((juxt :root-edn :project-edn) + (deps/find-edn-maps (u/filename u/project-root-directory "deps.edn"))))) + +(defonce metabase-core-basis + (binding [deps.dir/*the-dir* (io/file u/project-root-directory)] + (deps/calc-basis metabase-core-edn))) + +(defonce metabase-core-provided-libs + (set (keys (:libs metabase-core-basis)))) + +(defn driver-parents [driver edition] + (when-let [parents (not-empty (:metabase.build-driver/parents (c/driver-edn driver edition)))] + (u/announce "Driver has parent drivers %s" (pr-str parents)) + parents)) + +(defn parent-provided-libs [driver edition] + (into {} (for [parent (driver-parents driver edition) + lib (keys (:libs (driver-basis parent edition)))] + [lib parent]))) + +(defn remove-provided-libs [libs driver edition] + (let [parent-provided (parent-provided-libs driver edition)] + (into {} (for [[lib info] (sort-by first (seq libs)) + :let [provider (if (contains? metabase-core-provided-libs lib) + "metabase-core" + (get parent-provided lib)) + _ (u/announce (if provider + (format "SKIP %%s (provided by %s)" provider) + "INCLUDE %s") + (colorize/yellow lib))] + :when (not provider)] + [lib info])))) + +(defn uberjar-basis [driver edition] + (u/step "Determine which dependencies to include" + (update (driver-basis driver edition) :libs remove-provided-libs driver edition))) + +(defn create-uberjar! [driver edition] + (u/step (format "Write %s %s uberjar -> %s" driver edition (c/driver-jar-destination-path driver)) + (let [start-time-ms (System/currentTimeMillis)] + (depstar/uber + {:class-dir (c/compiled-source-target-dir driver) + :uber-file (c/driver-jar-destination-path driver) + :basis (uberjar-basis driver edition)}) + (u/announce "Created uberjar in %d ms." (- (System/currentTimeMillis) start-time-ms))))) diff --git a/bin/build-drivers/src/build_drivers/install_driver_locally.clj b/bin/build-drivers/src/build_drivers/install_driver_locally.clj deleted file mode 100644 index 7189c74bd9..0000000000 --- a/bin/build-drivers/src/build_drivers/install_driver_locally.clj +++ /dev/null @@ -1,49 +0,0 @@ -(ns build-drivers.install-driver-locally - "Logic related to installing a driver as a library in the local Maven repository so it can be used as a dependency - when building descandant drivers. Right now this is only used for `:google`, which is used by `:bigquery` and - `:googleanalytics`." - (:require [build-drivers.checksum :as checksum] - [build-drivers.common :as c] - [colorize.core :as colorize] - [metabuild-common.core :as u])) - -(defn- local-install-path [driver] - (u/filename c/maven-repository-path "metabase" (format "%s-driver" (name driver)))) - -(defn- local-install-checksum-filename [driver edition] - (u/filename (local-install-path driver) (str (c/edition-checksum-prefix driver edition) "checksum.md5"))) - -(defn clean! - "Delete local Maven installation of the library version of `driver`." - [driver] - (u/step (format "Deleting existing Maven installation of %s driver" driver) - (u/delete-file-if-exists! (local-install-path driver)))) - -(defn- local-install-checksum-matches? - "After installing the library version of `driver`, we save a checksum based on its sources; next time we call - `install-locally!`, we can recalculate the checksum; if the saved one matches the current one, we do not need to - reinstall." - [driver edition] - (u/step "Determine whether %s driver source files have changed since last local install" - (let [existing-checksum (checksum/checksum-from-file (local-install-checksum-filename driver edition)) - current-checksum (checksum/driver-checksum driver edition) - same? (= existing-checksum current-checksum)] - (u/announce (if same? - "Checksum is the same. Do not need to rebuild driver." - "Checksum is different. Need to rebuild driver.")) - same?))) - -(defn install-locally! - "Install `driver` as a library in the local Maven repository IF NEEDED so descendant drivers can use it as a - `:provided` dependency when building. E.g. before building `:bigquery` we need to install `:google` as a library - locally." - [driver edition] - {:pre [(keyword? driver)]} - (u/step (str (colorize/green "Install ") (colorize/yellow driver) (colorize/green " driver to local Maven repo if needed")) - (if (local-install-checksum-matches? driver edition) - (u/announce "Already installed locally.") - (u/step (str (colorize/green "Install ") (colorize/yellow driver) (colorize/green " driver to local Maven repo")) - (u/sh {:dir (c/driver-project-dir driver)} "lein" "clean") - (u/sh {:dir (c/driver-project-dir driver)} "lein" "install-for-building-drivers") - (u/step (format "Save checksum to %s" driver (local-install-checksum-filename driver edition)) - (spit (local-install-checksum-filename driver edition) (checksum/driver-checksum driver edition))))))) diff --git a/bin/build-drivers/src/build_drivers/metabase.clj b/bin/build-drivers/src/build_drivers/metabase.clj deleted file mode 100644 index e94b29798e..0000000000 --- a/bin/build-drivers/src/build_drivers/metabase.clj +++ /dev/null @@ -1,88 +0,0 @@ -(ns build-drivers.metabase - "Code for installing the main Metabase project as a library (`metabase-core`) in the local Maven repository, and for - building a Metabase uberjar. Both are needed when building drivers." - (:require [build-drivers - [checksum :as checksum] - [common :as c]] - [metabuild-common.core :as u])) - -(def ^String ^:private uberjar-checksum-path - (str c/metabase-uberjar-path ".md5")) - -(def ^String ^:private metabase-core-install-path - (u/filename c/maven-repository-path "metabase-core")) - -(def ^String ^:private metabase-core-checksum-path - (u/filename metabase-core-install-path "checksum.md5")) - -(defn metabase-core-checksum-matches? [] - (u/step "Determine whether Metabase source files checksum has changed since last install of metabase-core" - (let [existing-checksum (checksum/checksum-from-file metabase-core-checksum-path) - current-checksum (checksum/metabase-source-checksum) - same? (= existing-checksum current-checksum)] - (u/announce (if same? - "Checksum is the same. Do not need to reinstall metabase-core locally." - "Checksum is different. Need to reinstall metabase-core locally.")) - same?))) - -(defn- delete-metabase-core-install! [] - (u/step "Delete local installation of metabase-core" - (u/delete-file-if-exists! metabase-core-install-path))) - -(defn- install-metabase-core! [] - (u/step "Install metabase-core locally if needed" - (if (metabase-core-checksum-matches?) - (u/announce "Up-to-date metabase-core already installed to local Maven repo") - (do - (delete-metabase-core-install!) - (u/sh {:dir u/project-root-directory} "lein" "clean") - (u/sh {:dir u/project-root-directory} "lein" "install-for-building-drivers") - (u/step "Save checksum for local installation of metabase-core" - (spit metabase-core-checksum-path (checksum/metabase-source-checksum))) - (u/announce "metabase-core dep installed to local Maven repo successfully."))))) - -(defn uberjar-checksum-matches? - "After installing/building Metabase we save a MD5 hex checksum of Metabase backend source files (including - `project.clj`). The next time we run `build-metabase!`, if the checksums have changed we know we need to - rebuild/reinstall." - [] - (u/step "Determine whether Metabase source files checksum has changed since last build of uberjar" - (let [existing-checksum (checksum/checksum-from-file uberjar-checksum-path) - current-checksum (checksum/metabase-source-checksum) - same? (= existing-checksum current-checksum)] - (u/announce (if same? - "Checksum is the same. Do not need to rebuild Metabase uberjar." - "Checksum is different. Need to rebuild Metabase uberjar.")) - same?))) - -(defn- delete-metabase-uberjar! [] - (u/step "Delete exist metabase uberjar" - (u/delete-file-if-exists! (u/filename u/project-root-directory "target")))) - -(defn- build-metabase-uberjar! [] - (u/step "Build Metabase uberjar if needed" - (if (uberjar-checksum-matches?) - (u/announce "Update-to-date Metabase uberjar already built") - (do - (delete-metabase-uberjar!) - (u/sh {:dir u/project-root-directory} "lein" "clean") - (u/sh {:dir u/project-root-directory} "lein" "uberjar") - (u/step "Save checksum for Metabase uberar" - (spit uberjar-checksum-path (checksum/metabase-source-checksum))) - (u/announce "Metabase uberjar built successfully"))))) - -(defn clean-metabase! - "Delete local Maven repository installation of the `metabase-core` library and delete the built Metabase uberjar." - [] - (u/step "Clean local Metabase deps" - (delete-metabase-core-install!) - (delete-metabase-uberjar!))) - -(defn build-metabase! - "Install `metabase-core` as a library in the local Maven repo, and build the Metabase uberjar IF NEEDED. We need to do - both because `metabase-core` is used as a dependency for drivers, and the Metabase uberjar is checked to make sure - we don't ship duplicate classes in the driver JAR (as part of the `strip-and-compress` stage.)" - [] - (u/step "Build metabase-core and install locally" - (install-metabase-core!) - (build-metabase-uberjar!))) diff --git a/bin/build-drivers/src/build_drivers/plugin_manifest.clj b/bin/build-drivers/src/build_drivers/plugin_manifest.clj deleted file mode 100644 index 163c609647..0000000000 --- a/bin/build-drivers/src/build_drivers/plugin_manifest.clj +++ /dev/null @@ -1,55 +0,0 @@ -(ns build-drivers.plugin-manifest - "Code for reading the YAML plugin manifest for a driver. " - (:require [build-drivers.common :as c] - [metabuild-common.core :as u] - [yaml.core :as yaml])) - -(defn- plugin-manifest - "Read `driver` plugin manifest and return a map." - [driver] - {:post [(map? %)]} - (yaml/from-file (u/assert-file-exists (c/driver-plugin-manifest-filename driver)))) - -(defn- driver-declarations [manifest] - ;; driver plugin manifest can have a single `:driver`, or multiple drivers, e.g. Spark SQL which also has the - ;; `:hive-like` abstract driver - (let [{driver-declaration :driver} manifest] - (if (map? driver-declaration) - [driver-declaration] - driver-declaration))) - -(defn- declared-drivers - "Sequence of all drivers declared in a plugin `manifest`. Usually only one driver, except for Spark SQL which declares - both `:hive-like` and `:sparksql`." - [manifest] - (map (comp keyword :name) (driver-declarations manifest))) - -(def ^:private metabase-core-drivers - "Drivers that ship as part of the core Metabase project (as opposed to a plugin) and thus do not need to be built." - #{:sql - :sql-jdbc - :mysql - :h2 - :postgres}) - -(defn parent-drivers - "Get the parent drivers of a driver for purposes of building a driver. Excludes drivers that ship as part of - `metabase-core`, since we don't need to worry about building those. - - e.g. - - (parent-drivers :googleanalytics) ;-> (:google)" - [driver] - (let [manifest (plugin-manifest driver) - declared (declared-drivers manifest)] - (or (not-empty - (for [{parent-declaration :parent} (driver-declarations manifest) - :let [parents (if (string? parent-declaration) - [parent-declaration] - parent-declaration)] - parent parents - :let [parent (keyword parent)] - :when (and (not (contains? (set declared) parent)) - (not (contains? metabase-core-drivers parent)))] - parent)) - (u/announce "%s does not have any parents" driver)))) diff --git a/bin/build-drivers/src/build_drivers/strip_and_compress.clj b/bin/build-drivers/src/build_drivers/strip_and_compress.clj deleted file mode 100644 index b2d594ea8d..0000000000 --- a/bin/build-drivers/src/build_drivers/strip_and_compress.clj +++ /dev/null @@ -1,65 +0,0 @@ -(ns build-drivers.strip-and-compress - (:require [build-drivers.common :as c] - [build-drivers.plugin-manifest :as manifest] - [metabuild-common.core :as u]) - (:import java.io.FileOutputStream - [java.util.zip ZipEntry ZipFile ZipOutputStream] - org.apache.commons.io.IOUtils)) - -(def ^:private files-to-always-include - "Files to always include regardless of whether they are present in blacklist JAR." - #{"metabase-plugin.yaml"}) - -(defn- jar-contents - "Get a set of all files in a JAR that we should strip out from the driver JAR -- either the Metabase uberjar itself or - a parent driver JAR." - [^String jar-path] - (with-open [zip-file (ZipFile. jar-path)] - (set - (for [^ZipEntry zip-entry (enumeration-seq (.entries zip-file)) - :let [filename (str zip-entry)] - :when (not (files-to-always-include filename))] - filename)))) - -(defn- strip-classes! [^String driver-jar-path ^String blacklist-jar-path] - (u/step (format "Remove classes from %s that are present in %s and recompress" driver-jar-path blacklist-jar-path) - (let [jar-contents (jar-contents blacklist-jar-path) - temp-driver-jar-path "/tmp/driver.jar" - wrote (atom 0) - skipped (atom 0)] - (u/delete-file-if-exists! temp-driver-jar-path) - (with-open [source-zip (ZipFile. (u/assert-file-exists driver-jar-path)) - os (doto (ZipOutputStream. (FileOutputStream. temp-driver-jar-path)) - (.setMethod ZipOutputStream/DEFLATED) - (.setLevel 9))] - (doseq [^ZipEntry entry (enumeration-seq (.entries source-zip))] - (if (jar-contents (str entry)) - (swap! skipped inc) - (with-open [is (.getInputStream source-zip entry)] - (.putNextEntry os (ZipEntry. (.getName entry))) - (IOUtils/copy is os) - (.closeEntry os) - (swap! wrote inc))))) - (u/announce (format "Done. wrote: %d skipped: %d" @wrote @skipped)) - (u/safe-println (format "Original size: %s" (u/format-bytes (u/file-size driver-jar-path)))) - (u/safe-println (format "Stripped/extra-compressed size: %s" (u/format-bytes (u/file-size temp-driver-jar-path)))) - (u/step "replace the original source JAR with the stripped one" - (u/delete-file-if-exists! driver-jar-path) - (u/copy-file! temp-driver-jar-path driver-jar-path))))) - -(defn strip-and-compress-uberjar! - "Remove any classes in compiled `driver` that are also present in the Metabase uberjar or parent drivers. The classes - will be available at runtime, and we don't want to make things unpredictable by including them more than once in - different drivers. - - This is only needed because `lein uberjar` does not seem to reliably exclude classes from `:provided` Clojure - dependencies like `metabase-core` and the parent drivers." - [driver] - (u/step (str (format "Strip out any classes in %s driver JAR found in core Metabase uberjar or parent JARs" driver) - " and recompress with higher compression ratio") - (let [driver-jar-path (u/assert-file-exists (c/driver-jar-build-path driver))] - (u/step "strip out any classes also found in the core Metabase uberjar" - (strip-classes! driver-jar-path (u/assert-file-exists c/metabase-uberjar-path))) - (u/step "remove any classes also found in any of the parent JARs" - (doseq [parent (manifest/parent-drivers driver)] - (strip-classes! driver-jar-path (u/assert-file-exists (c/driver-jar-build-path parent)))))))) diff --git a/bin/build-drivers/test/build_drivers/build_driver_test.clj b/bin/build-drivers/test/build_drivers/build_driver_test.clj index 8eab16884f..20afc365fb 100644 --- a/bin/build-drivers/test/build_drivers/build_driver_test.clj +++ b/bin/build-drivers/test/build_drivers/build_driver_test.clj @@ -15,19 +15,11 @@ (build-driver/build-driver! :oracle :oss) (is (.exists (java.io.File. (jar-path)))) (testing "JAR should not contain the JDBC driver classes" - (is (not (jar-contains-jdbc-classes?)))) - (testing "Wouldn't need to rebuild :oss version of the driver" - (is (#'build-driver/driver-checksum-matches? :oracle :oss))) - (testing "WOULD need to build :ee version of the driver" - (is (not (#'build-driver/driver-checksum-matches? :oracle :ee)))))) + (is (not (jar-contains-jdbc-classes?)))))) (deftest build-ee-driver-test (testing "We should be able to build an EE driver" (build-driver/build-driver! :oracle :ee) (is (.exists (java.io.File. (jar-path)))) (testing "JAR *should* contain the JDBC driver classes" - (is (jar-contains-jdbc-classes?))) - (testing "Wouldn't need to rebuild :ee version of the driver" - (is (#'build-driver/driver-checksum-matches? :oracle :ee))) - (testing "WOULD need to build :oss version of the driver" - (is (not (#'build-driver/driver-checksum-matches? :oracle :oss)))))) + (is (jar-contains-jdbc-classes?))))) diff --git a/bin/build-drivers/test/build_drivers/checksum_test.clj b/bin/build-drivers/test/build_drivers/checksum_test.clj deleted file mode 100644 index 5c1bf49649..0000000000 --- a/bin/build-drivers/test/build_drivers/checksum_test.clj +++ /dev/null @@ -1,11 +0,0 @@ -(ns build-drivers.checksum-test - (:require [build-drivers.checksum :as checksum] - [clojure.test :refer :all])) - -(deftest driver-checksum-test - (testing "OSS/EE checksums should be the same for drivers that don't have different oss/ee profiles" - (is (= (checksum/driver-checksum :sqlite :oss) - (checksum/driver-checksum :sqlite :ee)))) - (testing "OSS/EE checksums should be different for drivers that have different oss/ee profiles" - (is (not= (checksum/driver-checksum :oracle :oss) - (checksum/driver-checksum :oracle :ee))))) diff --git a/bin/build-drivers/test/build_drivers/common_test.clj b/bin/build-drivers/test/build_drivers/common_test.clj deleted file mode 100644 index 8aa517fc3b..0000000000 --- a/bin/build-drivers/test/build_drivers/common_test.clj +++ /dev/null @@ -1,15 +0,0 @@ -(ns build-drivers.common-test - (:require [build-drivers.common :as c] - [clojure.test :refer :all])) - -(deftest has-edition-profile?-test - (testing :ee - (is (= true - (c/has-edition-profile? :oracle :ee))) - (is (= false - (c/has-edition-profile? :sqlite :ee)))) - (testing :oss - (is (= false - (c/has-edition-profile? :oracle :oss))) - (is (= false - (c/has-edition-profile? :sqlite :oss))))) diff --git a/bin/build-drivers/test/build_drivers/install_driver_locally_test.clj b/bin/build-drivers/test/build_drivers/install_driver_locally_test.clj deleted file mode 100644 index 3a3dd0e58e..0000000000 --- a/bin/build-drivers/test/build_drivers/install_driver_locally_test.clj +++ /dev/null @@ -1,16 +0,0 @@ -(ns build-drivers.install-driver-locally-test - (:require [build-drivers.install-driver-locally :as install-driver-locally] - [clojure.string :as str] - [clojure.test :refer :all])) - -(deftest local-install-checksum-filename-test - (is (str/ends-with? - (#'install-driver-locally/local-install-checksum-filename :oracle :ee) - ".m2/repository/metabase/oracle-driver/ee-checksum.md5")) - (is (str/ends-with? - (#'install-driver-locally/local-install-checksum-filename :oracle :oss) - ".m2/repository/metabase/oracle-driver/checksum.md5")) - (doseq [edition [:oss :ee]] - (is (str/ends-with? - (#'install-driver-locally/local-install-checksum-filename :sqlite edition) - ".m2/repository/metabase/sqlite-driver/checksum.md5")))) diff --git a/bin/build-mb/src/build.clj b/bin/build-mb/src/build.clj index 34744a9a15..6ab8a3b714 100644 --- a/bin/build-mb/src/build.clj +++ b/bin/build-mb/src/build.clj @@ -54,48 +54,47 @@ "./node_modules/.bin/webpack" "--bail" "--config" "webpack.static-viz.config.js")) (u/announce "Frontend built successfully.")))) +(defn- build-licenses! + [edition] + {:pre [(#{:oss :ee} edition)]} + (u/step "Generate backend license information from jar files" + (let [[classpath] (u/sh {:dir u/project-root-directory + :quiet? true} + "clojure" (str "-A" edition) "-Spath") + output-filename (u/filename u/project-root-directory + "resources" + "license-backend-third-party.txt") + {:keys [without-license]} (license/generate {:classpath classpath + :backfill (edn/read-string + (slurp (io/resource "overrides.edn"))) + :output-filename output-filename + :report? false})] + (when (seq without-license) + (run! (comp (partial u/error "Missing License: %s") first) + without-license)) + (u/announce "License information generated at %s" output-filename))) + + (u/step "Run `yarn licenses generate-disclaimer`" + (let [license-text (str/join \newline + (u/sh {:dir u/project-root-directory + :quiet? true} + "yarn" "licenses" "generate-disclaimer"))] + (spit (u/filename u/project-root-directory + "resources" + "license-frontend-third-party.txt") license-text)))) + (def uberjar-filename (u/filename u/project-root-directory "target" "uberjar" "metabase.jar")) (defn- build-uberjar! [edition] {:pre [(#{:oss :ee} edition)]} (u/delete-file-if-exists! uberjar-filename) (u/step (format "Build uberjar with profile %s" edition) - (u/sh {:dir u/project-root-directory} "lein" "clean") - (u/sh {:dir u/project-root-directory} "lein" "with-profile" (str \+ (name edition)) "uberjar") + ;; TODO -- we (probably) don't need to shell out in order to do this anymore, we should be able to do all this + ;; stuff directly in Clojure land by including this other `build` namespace directly (once we dedupe the names) + (u/sh {:dir u/project-root-directory} "clojure" "-T:build" "uberjar" :edition edition) (u/assert-file-exists uberjar-filename) (u/announce "Uberjar built successfully."))) -(defn- build-backend-licenses-file! [edition] - {:pre [(#{:oss :ee} edition)]} - (let [classpath-and-logs (u/sh {:dir u/project-root-directory - :quiet? true} - "lein" - "with-profile" (str \- "dev" - (str \, \+ (name edition)) - \,"+include-all-drivers") - "classpath") - classpath (last - classpath-and-logs) - output-filename (u/filename u/project-root-directory "license-backend-third-party") - {:keys [with-license - without-license]} (license/generate {:classpath classpath - :backfill (edn/read-string - (slurp (io/resource "overrides.edn"))) - :output-filename output-filename - :report? false})] - (when (seq without-license) - (run! (comp (partial u/error "Missing License: %s") first) - without-license)) - (u/announce "License information generated at %s" output-filename))) - -(defn- build-frontend-licenses-file! - [] - (let [license-text (str/join \newline - (u/sh {:dir u/project-root-directory - :quiet? true} - "yarn" "licenses" "generate-disclaimer"))] - (spit (u/filename u/project-root-directory "license-frontend-third-party") license-text))) - (def all-steps (ordered-map/ordered-map :version (fn [{:keys [edition version]}] @@ -104,12 +103,10 @@ (i18n/create-all-artifacts!)) :frontend (fn [{:keys [edition]}] (build-frontend! edition)) + :licenses (fn [{:keys [edition]}] + (build-licenses! edition)) :drivers (fn [{:keys [edition]}] (build-drivers/build-drivers! edition)) - :backend-licenses (fn [{:keys [edition]}] - (build-backend-licenses-file! edition)) - :frontend-licenses (fn [{:keys []}] - (build-frontend-licenses-file!)) :uberjar (fn [{:keys [edition]}] (build-uberjar! edition)))) @@ -142,14 +139,11 @@ (when-let [steps (not-empty steps)] {:steps steps}))))) +;; useful to call from command line `cd bin/build-mb && clojure -X build/list-without-license` (defn list-without-license [{:keys []}] - (let [classpath-and-logs (u/sh {:dir u/project-root-directory - :quiet? true} - "lein" - "with-profile" - "-dev,+ee,+include-all-drivers" - "classpath") - classpath (last classpath-and-logs) + (let [[classpath] (u/sh {:dir u/project-root-directory + :quiet? true} + "clojure" "-A:ee" "-Spath") classpath-entries (license/jar-entries classpath) {:keys [without-license]} (license/process* {:classpath-entries classpath-entries diff --git a/bin/build-mb/src/build/licenses.clj b/bin/build-mb/src/build/licenses.clj index ba656beb3e..7acc5699a1 100644 --- a/bin/build-mb/src/build/licenses.clj +++ b/bin/build-mb/src/build/licenses.clj @@ -192,9 +192,10 @@ (filter jar-file?))) (defn generate - "Process a classpath, creating a file of all license information, writing to `:output-filename`. Backfill is a clojure - data structure or a filename of an edn file of a clojure datastructure providing for backfilling license information - if it is not discernable from the jar. Should be of the form (note keys are strings not symbols) + "Process a classpath, creating a file of all license information, writing to `:output-filename`. `classpath-entries` + should be a seq of classpath roots. Split a classpath on the classpath separator. Backfill is a clojure data + structure or a filename of an edn file of a clojure datastructure providing for backfilling license information if + it is not discernable from the jar. Should be of the form (note keys are strings not symbols) {\"group\" {\"artifact\" \"license text\"} \"group\" {\"artifact\" {:resource \"filename-of-license\"}} @@ -216,24 +217,22 @@ :without-license [ [jar-filename {:coords {:group :artifact :version} :error }] ... ]}" [{:keys [classpath backfill output-filename report?] :or {report? true}}] (let [backfill (if (string? backfill) - (edn/read-string (slurp backfill)) + (edn/read-string (slurp (io/resource backfill))) (or backfill {})) - entries (jar-entries classpath)] - (let [{:keys [with-license without-license] :as license-info} - (process* {:classpath-entries entries - :backfill backfill})] + entries (jar-entries classpath) + {:keys [with-license without-license] :as license-info} + (process* {:classpath-entries entries + :backfill backfill})] + (when (seq with-license) + (with-open [os (io/writer output-filename)] + (run! #(write-license os %) with-license))) + (when report? + (when (seq without-license) + (run! #(report-missing *err* %) without-license)) (when (seq with-license) - (with-open [os (io/writer output-filename)] - (run! #(write-license os %) with-license))) - (when report? - (when (seq without-license) - (run! #(report-missing *err* %) without-license)) - (when (seq with-license) - (println "License information for" (count with-license) "libraries written to " - output-filename) - ;; we call this from the build script. if we switch to the shell we can reenable this and figure out the - ;; best defaults. Want to make sure we never kill our build script - #_(System/exit (if (seq without-license) 1 0)))) - license-info))) - -;; clj -X build.licenses/generate :classpath \"$(cd ../.. && lein with-profile -dev,+ee,+include-all-drivers classpath | tail -n1)\" :backfill "\"resources/overrides.edn\"" :output-filename "\"backend-licenses-ee.txt\"" + (println "License information for" (count with-license) "libraries written to " + output-filename) + ;; we call this from the build script. if we switch to the shell we can reenable this and figure out the + ;; best defaults. Want to make sure we never kill our build script + #_(System/exit (if (seq without-license) 1 0)))) + license-info)) diff --git a/bin/build-mb/test/build/licenses_test.clj b/bin/build-mb/test/build/licenses_test.clj index 1388af4c35..6d44655863 100644 --- a/bin/build-mb/test/build/licenses_test.clj +++ b/bin/build-mb/test/build/licenses_test.clj @@ -207,28 +207,26 @@ (deftest all-deps-have-licenses (testing "All deps on the classpath have licenses" - (loop-until-success #(u/sh {:dir u/project-root-directory} "lein" "with-profile" "+include-all-drivers,+oss,+ee" "deps") 3 "download deps") - (doseq [edition [:oss :ee]] - (let [classpath (u/sh {:dir u/project-root-directory - :quiet? true} - "lein" - "with-profile" (str \- "dev" - (str \, \+ (name edition)) - \,"+include-all-drivers") - "classpath") - classpath-entries (->> (str/split (last classpath) (re-pattern lic/classpath-separator)) - (filter lic/jar-file?))] - (let [results (lic/process* {:classpath-entries classpath-entries - :backfill (edn/read-string - (slurp (io/resource "overrides.edn")))})] - (is (nil? (:without-license results)) - (str "Deps without license information:\n" - (str/join "\n" (map first (:without-license results))))) - (is (= (set classpath-entries) - (into #{} (->> results :with-license (map first)))))) - (is (some? (:without-license - (lic/process* {:classpath-entries classpath-entries - :backfill {}})))))))) + (loop-until-success #(u/sh {:dir u/project-root-directory} "clojure" "-A:ee" "-P") 3 "download deps") + (let [edition :ee + classpath (u/sh {:dir u/project-root-directory + :quiet? true} + "clojure" + "-A:ee" + "-Spath") + classpath-entries (->> (str/split (last classpath) (re-pattern lic/classpath-separator)) + (filter lic/jar-file?))] + (let [results (lic/process* {:classpath-entries classpath-entries + :backfill (edn/read-string + (slurp (io/resource "overrides.edn")))})] + (is (nil? (:without-license results)) + (str "Deps without license information:\n" + (str/join "\n" (map first (:without-license results))))) + (is (= (set classpath-entries) + (into #{} (->> results :with-license (map first)))))) + (is (some? (:without-license + (lic/process* {:classpath-entries classpath-entries + :backfill {}}))))))) (comment (run-tests) (binding [clojure.test/*test-out* *out*] (run-tests)) diff --git a/bin/check-clojure-cli.sh b/bin/check-clojure-cli.sh index 0fdb8cd5df..d78b774713 100755 --- a/bin/check-clojure-cli.sh +++ b/bin/check-clojure-cli.sh @@ -1,9 +1,7 @@ #! /usr/bin/env bash -set -eou pipefail - you_need_to_upgrade() { - echo "Clojure CLI must be at least version 1.10.1.708. Your version is $version." + echo "Clojure CLI must be at least version 1.10.3.905. Your version is $version." echo "See https://www.clojure.org/guides/getting_started for upgrade instructions." exit -3 } @@ -24,8 +22,8 @@ check_clojure_cli() { elif [ "$minor_version" -eq "10" ]; then if [ "$patch_version" -lt "1" ]; then you_need_to_upgrade - elif [ "$patch_version" -eq "1" ]; then - if [ "$build_version" -lt "708" ]; then + elif [ "$patch_version" -eq "3" ]; then + if [ "$build_version" -lt "905" ]; then you_need_to_upgrade fi fi diff --git a/bin/clear-outdated-cpcaches.sh b/bin/clear-outdated-cpcaches.sh new file mode 100755 index 0000000000..89c2e43cd3 --- /dev/null +++ b/bin/clear-outdated-cpcaches.sh @@ -0,0 +1,51 @@ +#! /usr/bin/env bash + +set -euo pipefail + +script_directory=`dirname "${BASH_SOURCE[0]}"` + +# This function will clear all the .cpcache directories if any deps.edn file is newer than any of them. +clear_outdated_cpcaches() { + echo "Clearing outdated .cpcache directories if needed..." + + # switch to project root directory if we're not already there + cd "$script_directory/.." + project_root=`pwd` + + cpcaches=`find bin java modules -type d -name .cpcache` + if [ -d .cpcache ]; then + cpcaches=".cpcache $cpcaches" + fi + if [ -z "$cpcaches" ]; then + echo "No .cpcache directories found; nothing to do" + return 0 + fi + + deps_edns="deps.edn $(find bin java modules -type f -name deps.edn)" + + # find the OLDEST cpcache and NEWEST deps.edn files. + oldest_cpcache="" + for cpcache in $cpcaches; do + if [ -z "$oldest_cpcache" ] || [ "$cpcache" -ot "$oldest_cpcache" ]; then + oldest_cpcache="$cpcache" + fi + done + + newest_deps_edn="" + for deps_edn in $deps_edns; do + if [ -z "$newest_deps_edn" ] || [ "$deps_edn" -nt "$newest_deps_edn" ]; then + newest_deps_edn="$deps_edn" + fi + done + + # if the newest deps.edn is newer than the *ANY* of the cpcaches, clear all the cpcaches. + if [ "$newest_deps_edn" -nt "$oldest_cpcache" ]; then + echo "$newest_deps_edn is newer than $oldest_cpcache; deleting all .cpcache directories" + for cpcache in $cpcaches; do + echo "rm -rf $cpcache" + rm -rf "$cpcache" + done + else + echo ".cpcache directories are up to date." + fi +} diff --git a/bin/common/src/metabuild_common/core.clj b/bin/common/src/metabuild_common/core.clj index 08b592c67b..a58be58dbd 100644 --- a/bin/common/src/metabuild_common/core.clj +++ b/bin/common/src/metabuild_common/core.clj @@ -61,7 +61,6 @@ [output announce error - format-bytes pretty-print-exception safe-println] diff --git a/bin/common/src/metabuild_common/files.clj b/bin/common/src/metabuild_common/files.clj index cff9f9407d..4a1d9e3456 100644 --- a/bin/common/src/metabuild_common/files.clj +++ b/bin/common/src/metabuild_common/files.clj @@ -98,17 +98,17 @@ (def ^String project-root-directory "Root directory of the Metabase repo, e.g. `/users/cam/metabase`. Determined by finding the directory that has - `project.clj` in it." + `.git` in it." (loop [^File dir (File. ^String (env/env :user-dir))] (cond - (file-exists? (filename (.getAbsolutePath dir) "project.clj")) + (file-exists? (filename (.getAbsolutePath dir) "package.json")) (.getAbsolutePath dir) (.getParentFile dir) (recur (.getParentFile dir)) :else - (throw (ex-info (format "Can't find project root directory: no parent directory of %s has a project.clj file" + (throw (ex-info (format "Can't find project root directory: no parent directory of %s has a .git directory" (env/env :user-dir)) {:dir (env/env :user-dir)}))))) diff --git a/bin/common/src/metabuild_common/output.clj b/bin/common/src/metabuild_common/output.clj index 3e70771d03..2e49b694be 100644 --- a/bin/common/src/metabuild_common/output.clj +++ b/bin/common/src/metabuild_common/output.clj @@ -40,12 +40,3 @@ (println (colorize/red (str "Step failed: " (.getMessage e)))) (binding [pprint/*print-right-margin* 120] (pprint/pprint e-map)))) - -(defn format-bytes - "Nicely format `num-bytes` in a human-readable way (e.g. KB/MB/etc.)" - [num-bytes] - (loop [n num-bytes [suffix & more] ["B" "KB" "MB" "GB"]] - (if (and (seq more) - (>= n 1024)) - (recur (/ n 1024.0) more) - (format "%.1f %s" n suffix)))) diff --git a/bin/i18n/update-translation-template b/bin/i18n/update-translation-template index 8f48a8f1a9..e327cda372 100755 --- a/bin/i18n/update-translation-template +++ b/bin/i18n/update-translation-template @@ -1,6 +1,6 @@ -#!/bin/sh +#! /usr/bin/env bash -set -eu +set -euo pipefail # gettext installed via homebrew is "keg-only", add it to the PATH if [ -d "/usr/local/opt/gettext/bin" ]; then @@ -13,6 +13,16 @@ if ! command -v xgettext > /dev/null; then exit 1 fi +# switch to project root directory if we're not already there +script_directory=`dirname "${BASH_SOURCE[0]}"` +cd "$script_directory/../.." + +source "./bin/check-clojure-cli.sh" +check_clojure_cli + +source "./bin/prep.sh" +prep_deps + POT_NAME="locales/metabase.pot" POT_BACKEND_NAME="locales/metabase-backend.pot" # NOTE: hardcoded in .babelrc @@ -68,7 +78,7 @@ rm "$POT_BACKEND_NAME.bak" # update auto dash pot # ######################## -lein generate-automagic-dashboards-pot +clojure -M:generate-automagic-dashboards-pot ################## # merge all pots # diff --git a/bin/lint-migrations-file.sh b/bin/lint-migrations-file.sh index 419ea7540d..e649dcdf9a 100755 --- a/bin/lint-migrations-file.sh +++ b/bin/lint-migrations-file.sh @@ -9,5 +9,11 @@ cd "$script_directory/.." source "./bin/check-clojure-cli.sh" check_clojure_cli +source "./bin/clear-outdated-cpcaches.sh" +clear_outdated_cpcaches + +source "./bin/prep.sh" +prep_deps + cd bin/lint-migrations-file clojure -M -m lint-migrations-file $@ diff --git a/bin/prep.sh b/bin/prep.sh new file mode 100755 index 0000000000..0962c281d8 --- /dev/null +++ b/bin/prep.sh @@ -0,0 +1,60 @@ +#! /usr/bin/env bash + +# functions for running prep steps to compile Java and AOT source files, needed before running other stuff. + +script_directory=`dirname "${BASH_SOURCE[0]}"` +cd "$script_directory/.." +project_root=`pwd` + +clear_cpcaches() { + cd "$project_root" + for file in `find . -type d -name .cpcache`; do + rm -rf "$file" + done +} + +compile_java_sources() { + cd "$project_root" + + echo "Compile Java source files in $project_root/java if needed..." + if [ ! -d "$project_root/java/target/classes" ]; then + echo 'Compile Java source files' + cd "$project_root" + clojure -Sforce -X:deps prep + else + echo 'Java source files are already compiled' + fi +} + +compile_spark_sql_aot_sources() { + cd "$project_root" + + echo "Compile Spark SQL AOT source files in $project_root/modules/drivers/sparksql if needed..." + if [ ! -d "$project_root/modules/drivers/sparksql/target/classes" ]; then + echo 'Compile Spark SQL AOT source files' + cd "$project_root/modules/drivers" + clojure -Sforce -X:deps prep + else + echo 'Spark SQL AOT source files are already compiled' + fi +} + +prep_deps() { + if compile_java_sources; then + echo "Java sources => OK" + else + echo 'Compilation failed (WHY?!); clearing classpath caches and trying again...' + clear_cpcaches + compile_java_sources + fi + + if compile_spark_sql_aot_sources; then + echo "Spark SQL AOT sources => OK" + else + echo 'Compilation failed (WHY?!); clearing classpath caches and trying again...' + clear_cpcaches + compile_spark_sql_aot_sources + fi + + cd "$project_root" +} diff --git a/bin/release.sh b/bin/release.sh index 19dd88914e..b8fc3c68bb 100755 --- a/bin/release.sh +++ b/bin/release.sh @@ -2,8 +2,18 @@ set -euo pipefail +# switch to project root directory if we're not already there +script_directory=`dirname "${BASH_SOURCE[0]}"` +cd "$script_directory/.." + source "./bin/check-clojure-cli.sh" check_clojure_cli +source "./bin/clear-outdated-cpcaches.sh" +clear_outdated_cpcaches + +source "./bin/prep.sh" +prep_deps + cd bin/release clojure -M -m release $@ diff --git a/bin/release/deps.edn b/bin/release/deps.edn index b9e0e5d236..cffa79de3f 100644 --- a/bin/release/deps.edn +++ b/bin/release/deps.edn @@ -14,8 +14,4 @@ :extra-deps {com.cognitect/test-runner {:git/url "https://github.com/cognitect-labs/test-runner.git" :sha "209b64504cb3bd3b99ecfec7937b358a879f55c1"} org.clojure/data.json {:mvn/version "2.0.2"}} - :main-opts ["-m" "cognitect.test-runner"]} - :nREPL {:extra-paths ["test"] - :extra-deps {nrepl/nrepl {:mvn/version "0.8.3"} - org.clojure/data.json {:mvn/version "2.0.2"}} - :main-opts ["-m" "nrepl.cmdline" "-i"]}}} + :main-opts ["-m" "cognitect.test-runner"]}}} diff --git a/build.clj b/build.clj new file mode 100644 index 0000000000..2b52bdf298 --- /dev/null +++ b/build.clj @@ -0,0 +1,160 @@ +(ns build + (:require [clojure.java.io :as io] + [clojure.string :as str] + [clojure.tools.build.api :as b] + [clojure.tools.build.util.zip :as b.zip] + [clojure.tools.namespace.dependency :as ns.deps] + [clojure.tools.namespace.find :as ns.find] + [clojure.tools.namespace.parse :as ns.parse] + [hf.depstar.api :as d] + [metabuild-common.core :as c]) + (:import java.io.OutputStream + java.net.URI + [java.nio.file Files FileSystems OpenOption StandardOpenOption] + java.util.Collections + java.util.jar.Manifest)) + +(def class-dir "target/classes") +(def uberjar-filename "target/uberjar/metabase.jar") + +(defn do-with-duration-ms [thunk f] + (let [start-time-ms (System/currentTimeMillis) + result (thunk) + duration (- (System/currentTimeMillis) start-time-ms)] + (f duration) + result)) + +(defmacro with-duration-ms [[duration-ms-binding] & body] + (let [[butlast-forms last-form] ((juxt butlast last) body)] + `(do-with-duration-ms + (fn [] ~@butlast-forms) + (fn [~duration-ms-binding] + ~last-form)))) + +(defn create-basis [edition] + {:pre [(#{:ee :oss} edition)]} + (b/create-basis {:project "deps.edn", :aliases #{edition}})) + +(defn all-paths [basis] + (concat (:paths basis) + (get-in basis [:classpath-args :extra-paths]))) + +(defn clean! [] + (c/step "Clean" + (c/step (format "Delete %s" class-dir) + (b/delete {:path class-dir})) + (c/step (format "Delete %s" uberjar-filename) + (b/delete {:path uberjar-filename})))) + +;; this topo sort order stuff is required for stuff to work correctly... I copied it from my Cloverage PR +;; https://github.com/cloverage/cloverage/pull/303 +(defn- dependencies-graph + "Return a `clojure.tools.namespace` dependency graph of namespaces named by `ns-symbol`." + [ns-decls] + (reduce + (fn [graph ns-decl] + (let [ns-symbol (ns.parse/name-from-ns-decl ns-decl)] + (reduce + (fn [graph dep] + (ns.deps/depend graph ns-symbol dep)) + graph + (ns.parse/deps-from-ns-decl ns-decl)))) + (ns.deps/graph) + ns-decls)) + +(defn metabase-namespaces-in-topo-order [basis] + (let [ns-decls (mapcat + (comp ns.find/find-ns-decls-in-dir io/file) + (all-paths basis)) + ns-symbols (set (map ns.parse/name-from-ns-decl ns-decls))] + (->> (dependencies-graph ns-decls) + ns.deps/topo-sort + (filter ns-symbols)))) + +(defn compile-sources! [basis] + (c/step "Compile Clojure source files" + (let [paths (all-paths basis) + _ (c/announce "Compiling Clojure files in %s" (pr-str paths)) + ns-decls (c/step "Determine compilation order for Metabase files" + (metabase-namespaces-in-topo-order basis))] + (with-duration-ms [duration-ms] + (b/compile-clj {:basis basis + :src-dirs paths + :class-dir class-dir + :ns-compile ns-decls}) + (c/announce "Finished compilation in %.1f seconds." (/ duration-ms 1000.0)))))) + +(defn copy-resources! [edition basis] + (c/step "Copy resources" + ;; technically we don't NEED to copy the Clojure source files but it doesn't really hurt anything IMO. + (doseq [path (all-paths basis)] + (c/step (format "Copy %s" path) + (b/copy-dir {:target-dir class-dir, :src-dirs [path]}))))) + +(defn create-uberjar! [basis] + (c/step "Create uberjar" + (with-duration-ms [duration-ms] + (d/uber {:class-dir class-dir + :uber-file uberjar-filename + :basis basis}) + (c/announce "Created uberjar in %.1f seconds." (/ duration-ms 1000.0))))) + +(def manifest-entries + {"Manifest-Version" "1.0" + "Created-By" "Metabase build.clj" + "Build-Jdk-Spec" (System/getProperty "java.specification.version") + "Main-Class" "metabase.core" + "Liquibase-Package" (str/join "," + ["liquibase.change" + "liquibase.changelog" + "liquibase.database" + "liquibase.datatype" + "liquibase.diff" + "liquibase.executor" + "liquibase.ext" + "liquibase.lockservice" + "liquibase.logging" + "liquibase.parser" + "liquibase.precondition" + "liquibase.sdk" + "liquibase.serializer" + "liquibase.snapshot" + "liquibase.sqlgenerator" + "liquibase.structure" + "liquibase.structurecompare"])}) + +(defn manifest ^Manifest [] + (doto (Manifest.) + (b.zip/fill-manifest! manifest-entries))) + +(defn write-manifest! [^OutputStream os] + (.write (manifest) os) + (.flush os)) + +;; the customizations we need to make are not currently supported by tools.build -- see +;; https://ask.clojure.org/index.php/10827/ability-customize-manifest-created-clojure-tools-build-uber -- so we need +;; to do it by hand for the time being. +(defn update-manifest! [] + (c/step "Update META-INF/MANIFEST.MF" + (with-open [fs (FileSystems/newFileSystem (URI. (str "jar:file:" (.getAbsolutePath (io/file "target/uberjar/metabase.jar")))) + Collections/EMPTY_MAP)] + (let [manifest-path (.getPath fs "META-INF" (into-array String ["MANIFEST.MF"]))] + (with-open [os (Files/newOutputStream manifest-path (into-array OpenOption [StandardOpenOption/WRITE + StandardOpenOption/TRUNCATE_EXISTING]))] + (write-manifest! os)))))) + +;; clojure -T:build uberjar :edition +(defn uberjar [{:keys [edition], :or {edition :oss}}] + (c/step (format "Build %s uberjar" edition) + (with-duration-ms [duration-ms] + (clean!) + (let [basis (create-basis edition)] + (compile-sources! basis) + (copy-resources! edition basis) + (create-uberjar! basis) + (update-manifest!)) + (c/announce "Built target/uberjar/metabase.jar in %.1f seconds." + (/ duration-ms 1000.0))))) + +;; TODO -- add `jar` and `install` commands to install Metabase to the local Maven repo (?) could make it easier to +;; build 3rd-party drivers the old way diff --git a/deps.edn b/deps.edn new file mode 100644 index 0000000000..e72fb27401 --- /dev/null +++ b/deps.edn @@ -0,0 +1,383 @@ +;; -*- comment-column: 80; -*- +{:deps + ;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ;; !! PLEASE KEEP THESE ORGANIZED ALPHABETICALLY !! + ;; !! AND ADD A COMMENT EXPLAINING THEIR PURPOSE !! + ;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + {aleph/aleph {:mvn/version "0.4.6" ; Async HTTP library; WebSockets + :exclusions [org.clojure/tools.logging]} + amalloy/ring-buffer {:mvn/version "1.2.2" ; fixed length queue implementation, used in log buffering + :exclusions [org.clojure/clojure + org.clojure/clojurescript]} + amalloy/ring-gzip-middleware {:mvn/version "0.1.4"} ; Ring middleware to GZIP responses if client can handle it + bigml/histogram {:mvn/version "4.1.3"} ; Histogram data structure + buddy/buddy-core {:mvn/version "1.5.0" ; various cryptograhpic functions + :exclusions [commons-codec/commons-codec + org.bouncycastle/bcpkix-jdk15on + org.bouncycastle/bcprov-jdk15on]} + buddy/buddy-sign {:mvn/version "3.0.0"} ; JSON Web Tokens; High-Level message signing library + cheshire/cheshire {:mvn/version "5.10.0"} ; fast JSON encoding (used by Ring JSON middleware) + clj-http/clj-http {:mvn/version "3.10.3" ; HTTP client + :exclusions [commons-codec/commons-codec + commons-io/commons-io + slingshot/slingshot]} + clojurewerkz/quartzite {:mvn/version "2.1.0" ; scheduling library + :exclusions [c3p0/c3p0]} + colorize/colorize {:mvn/version "0.1.1" ; string output with ANSI color codes (for logging) + :exclusions [org.clojure/clojure]} + com.cemerick/friend {:mvn/version "0.2.3" ; auth library + :exclusions [commons-codec/commons-codec + net.sourceforge.nekohtml/nekohtml + org.apache.httpcomponents/httpclient + ring/ring-core + slingshot/slingshot]} + com.clearspring.analytics/stream {:mvn/version "2.9.6" ; Various sketching algorithms + :exclusions [it.unimi.dsi/fastutil + org.slf4j/slf4j-api]} + com.draines/postal {:mvn/version "2.0.3"} ; SMTP library + com.google.guava/guava {:mvn/version "28.2-jre"} ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697 + com.h2database/h2 {:mvn/version "1.4.197"} ; embedded SQL database + com.taoensso/nippy {:mvn/version "2.14.0"} ; Fast serialization (i.e., GZIP) library for Clojure + commons-codec/commons-codec {:mvn/version "1.15"} ; Apache Commons -- useful codec util fns + commons-io/commons-io {:mvn/version "2.8.0"} ; Apache Commons -- useful IO util fns + commons-validator/commons-validator {:mvn/version "1.6" ; Apache Commons -- useful validation util fns + :exclusions [commons-beanutils/commons-beanutils + commons-digester/commons-digester + commons-logging/commons-logging]} + compojure/compojure {:mvn/version "1.6.1" ; HTTP Routing library built on Ring + :exclusions [ring/ring-codec]} + crypto-random/crypto-random {:mvn/version "1.2.0"} ; library for generating cryptographically secure random bytes and strings + dk.ative/docjure {:mvn/version "1.14.0" ; excel export + :exclusions [org.apache.poi/poi + org.apache.poi/poi-ooxml]} + environ/environ {:mvn/version "1.2.0"} ; env vars/Java properties abstraction + hiccup/hiccup {:mvn/version "1.0.5"} ; HTML templating + honeysql/honeysql {:mvn/version "1.0.461" ; Transform Clojure data structures to SQL + :exclusions [org.clojure/clojurescript]} + instaparse/instaparse {:mvn/version "1.4.10"} ; Make your own parser + io.forward/yaml {:mvn/version "1.0.9" ; Clojure wrapper for YAML library SnakeYAML (which we already use for liquibase) + :exclusions [org.clojure/clojure + org.flatland/ordered + org.yaml/snakeyaml]} + javax.xml.bind/jaxb-api {:mvn/version "2.4.0-b180830.0359"} ; add the `javax.xml.bind` classes which we're still using but were removed in Java 11 + joda-time/joda-time {:mvn/version "2.10.8"} + kixi/stats {:mvn/version "0.4.4" ; Various statistic measures implemented as transducers + :exclusions [org.clojure/data.avl]} + me.raynes/fs {:mvn/version "1.4.6" ; Filesystem tools + :exclusions [org.apache.commons/commons-compress]} + medley/medley {:mvn/version "1.3.0"} ; lightweight lib of useful functions + metabase/connection-pool {:mvn/version "1.1.1"} ; simple wrapper around C3P0. JDBC connection pools + metabase/saml20-clj {:mvn/version "2.0.0"} ; EE SAML integration + metabase/throttle {:mvn/version "1.0.2"} ; Tools for throttling access to API endpoints and other code pathways + net.cgrand/macrovich {:mvn/version "0.2.1"} ; utils for writing macros for both Clojure & ClojureScript + net.redhogs.cronparser/cron-parser-core {:mvn/version "3.4" ; describe Cron schedule in human-readable language + :exclusions [joda-time/joda-time ; exclude joda time 2.3 which has outdated timezone information + org.slf4j/slf4j-api]} + net.sf.cssbox/cssbox {:mvn/version "4.12" ; HTML / CSS rendering + :exclusions [org.slf4j/slf4j-api]} + org.apache.commons/commons-compress {:mvn/version "1.20"} ; compression utils + org.apache.commons/commons-lang3 {:mvn/version "3.10"} ; helper methods for working with java.lang stuff + org.apache.logging.log4j/log4j-1.2-api {:mvn/version "2.13.3"} ; apache logging framework + org.apache.logging.log4j/log4j-api {:mvn/version "2.13.3"} ; add compatibility with log4j 1.2 + org.apache.logging.log4j/log4j-core {:mvn/version "2.13.3"} ; apache logging framework + org.apache.logging.log4j/log4j-jcl {:mvn/version "2.13.3"} ; allows the commons-logging API to work with log4j 2 + org.apache.logging.log4j/log4j-liquibase {:mvn/version "2.13.3"} ; liquibase logging via log4j 2 + org.apache.logging.log4j/log4j-slf4j-impl {:mvn/version "2.13.3"} ; allows the slf4j API to work with log4j 2 + org.apache.poi/poi {:mvn/version "5.0.0"} ; Work with Office documents (e.g. Excel spreadsheets) -- newer version than one specified by Docjure + org.apache.poi/poi-ooxml {:mvn/version "5.0.0" + :exclusions [org.bouncycastle/bcpkix-jdk15on + org.bouncycastle/bcprov-jdk15on]} + org.apache.sshd/sshd-core {:mvn/version "2.4.0"} ; ssh tunneling and test server + org.bouncycastle/bcpkix-jdk15on {:mvn/version "1.68"} ; Bouncy Castle crypto library -- explicit version of BC specified to resolve illegal reflective access errors + org.bouncycastle/bcprov-jdk15on {:mvn/version "1.68"} + org.clojars.pntblnk/clj-ldap {:mvn/version "0.0.16"} ; LDAP client + org.clojure/clojure {:mvn/version "1.10.3"} + org.clojure/core.async {:mvn/version "0.4.500" + :exclusions [org.clojure/tools.reader]} + org.clojure/core.logic {:mvn/version "1.0.0"} ; optimized pattern matching library for Clojure + org.clojure/core.match {:mvn/version "0.3.0"} + org.clojure/core.memoize {:mvn/version "1.0.236"} ; useful FIFO, LRU, etc. caching mechanisms + org.clojure/data.csv {:mvn/version "0.1.4"} ; CSV parsing / generation + org.clojure/java.classpath {:mvn/version "1.0.0"} ; examine the Java classpath from Clojure programs + org.clojure/java.jdbc {:mvn/version "0.7.11"} ; basic JDBC access from Clojure + org.clojure/java.jmx {:mvn/version "1.0.0"} ; JMX bean library, for exporting diagnostic info + org.clojure/math.combinatorics {:mvn/version "0.1.4"} ; combinatorics functions + org.clojure/math.numeric-tower {:mvn/version "0.0.4"} ; math functions like `ceil` + org.clojure/tools.logging {:mvn/version "1.1.0"} ; logging framework + org.clojure/tools.namespace {:mvn/version "1.0.0"} + org.clojure/tools.reader {:mvn/version "1.3.6"} + org.clojure/tools.trace {:mvn/version "0.7.10"} ; function tracing + org.eclipse.jetty/jetty-server {:mvn/version "9.4.32.v20200930"} ; We require JDK 8 which allows us to run Jetty 9.4, ring-jetty-adapter runs on 1.7 which forces an older version + org.flatland/ordered {:mvn/version "1.5.9"} ; ordered maps & sets + org.graalvm.js/js {:mvn/version "21.0.0.2"} ; JavaScript engine + org.graalvm.js/js-scriptengine {:mvn/version "21.0.0.2"} + org.liquibase/liquibase-core {:mvn/version "3.6.3" ; migration management (Java lib) + :exclusions [ch.qos.logback/logback-classic]} + org.mariadb.jdbc/mariadb-java-client {:mvn/version "2.6.2"} ; MySQL/MariaDB driver + org.postgresql/postgresql {:mvn/version "42.2.18"} ; Postgres driver + org.slf4j/slf4j-api {:mvn/version "1.7.30"} ; abstraction for logging frameworks -- allows end user to plug in desired logging framework at deployment time + org.tcrawley/dynapath {:mvn/version "1.1.0"} ; Dynamically add Jars (e.g. Oracle or Vertica) to classpath + org.threeten/threeten-extra {:mvn/version "1.5.0"} ; extra Java 8 java.time classes like DayOfMonth and Quarter + org.yaml/snakeyaml {:mvn/version "1.23"} ; YAML parser (required by liquibase) + potemkin/potemkin {:mvn/version "0.4.5" ; utility macros & fns + :exclusions [riddley/riddley]} + pretty/pretty {:mvn/version "1.0.5"} ; protocol for defining how custom types should be pretty printed + prismatic/schema {:mvn/version "1.1.12"} ; Data schema declaration and validation library + redux/redux {:mvn/version "0.1.4"} ; Utility functions for building and composing transducers + riddley/riddley {:mvn/version "0.2.0"} ; code walking lib -- used interally by Potemkin, manifold, etc. + ring/ring-core {:mvn/version "1.8.1"} ; web server (Jetty wrapper) + ring/ring-jetty-adapter {:mvn/version "1.8.1"} ; Ring adapter using Jetty webserver + ring/ring-json {:mvn/version "0.5.0"} ; Ring middleware for reading/writing JSON automatically + robdaemon/clojure.java-time {:mvn/version "0.3.3-SNAPSHOT"} ; Java 8 java.time wrapper. Fork to address #13102 - see upstream PR: https://github.com/dm3/clojure.java-time/pull/60 + slingshot/slingshot {:mvn/version "0.12.2"} ; enhanced throw/catch, used by other deps + stencil/stencil {:mvn/version "0.5.0"} ; Mustache templates for Clojure + toucan/toucan {:mvn/version "1.15.3" ; Model layer, hydration, and DB utilities + :exclusions [honeysql/honeysql + org.clojure/java.jdbc + org.clojure/tools.logging + org.clojure/tools.namespace]} + user-agent/user-agent {:mvn/version "0.1.0"} ; User-Agent string parser, for Login History page & elsewhere + weavejester/dependency {:mvn/version "0.2.1"} ; Dependency graphs and topological sorting + + ;; dummy dependency for the Java source file(s) + metabase/java-deps {:local/root "java"}} + ;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + ;; !! PLEASE KEEP NEW DEPENDENCIES ABOVE ALPHABETICALLY ORGANIZED AND ADD COMMENTS EXPLAINING THEM. !! + ;; !! *PLEASE DO NOT* ADD NEW ONES TO THE BOTTOM OF THE LIST. !! + ;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + + + :paths + ["src" "shared/src" "resources" "java/target/classes"] + + :aliases + { +;;; Local Dev & test profiles + + ;; for local development: start a REPL with + ;; + ;; clojure -A:dev (basic dev REPL that includes test namespaces) + ;; clojure -A:dev:drivers:drivers-dev (dev REPL w/ drivers + tests) + ;; clojure -A:dev:ee:ee-dev (dev REPL w/ EE code including tests) + ;; + ;; You can start a web server from this REPL with + ;; + ;; (require 'dev) + ;; (dev/start!) + :dev + {:extra-deps + {clj-http-fake/clj-http-fake {:mvn/version "1.0.3" + :exclusions [slingshot/slingshot]} + cloverage/cloverage {:mvn/version "1.2.2"} + eftest/eftest {:mvn/version "0.5.9"} + jonase/eastwood {:mvn/version "0.9.4"} + lein-check-namespace-decls/lein-check-namespace-decls {:mvn/version "1.0.4"} ; misnomer since this works on Clojure CLI now too + pjstadig/humane-test-output {:mvn/version "0.11.0"} + reifyhealth/specmonstah {:mvn/version "2.0.0"} + ring/ring-mock {:mvn/version "0.4.0"} + talltale/talltale {:mvn/version "0.5.4"}} + + :extra-paths ["dev/src" "local/src" "test" "shared/test" "test_resources"] + :jvm-opts ["-Dmb.run.mode=dev" + "-Dmb.field.filter.operators.enabled=true" + "-Dmb.test.env.setting=ABCDEFG" + "-Duser.timezone=UTC" + "-Duser.language=en" + ;; Exceptions that get thrown repeatedly are created without stacktraces as a performance + ;; optimization in newer Java versions. This makes debugging pretty hard when working on stuff + ;; locally -- prefer debuggability over performance for local dev work. + "-XX:-OmitStackTraceInFastThrow" + ;; prevent Java icon from randomly popping up in macOS dock + "-Djava.awt.headless=true"]} + + ;; includes test code as source paths. Run tests with clojure -X:dev:test + :test + {:extra-paths ["test_config"] + :exec-fn test-runner/run-tests + :jvm-opts ["-Dmb.run.mode=test" + "-Dmb.db.in.memory=true" + "-Dmb.jetty.join=false" + "-Dmb.field.filter.operators.enabled=true" + "-Dmb.api.key=test-api-key" + ;; Different port from normal `:dev` so you can run tests on a different server. + ;; TODO -- figure out how to do a random port like in the old project.clj? + "-Dmb.jetty.port=3001"]} + + ;; run the dev server with + ;; clojure -M:run + ;; clojure -M:run:drivers (include all drivers) + ;; clojure -M:run:ee (include EE code) + :run + {:main-opts ["-m" "metabase.core"] + :jvm-opts ["-Djava.awt.headless=true" ; prevent Java icon from randomly popping up in macOS dock + "-Dmb.jetty.port=3000"]} + + ;; alias for CI-specific options. + :ci + {:jvm-opts ["-Xmx2g" + ;; normally CircleCI sets `CI` as an env var, so this is mostly to replicate that locally. + "-Dci=TRUE"]} + + ;; include EE source code. + :ee + {:extra-paths ["enterprise/backend/src"]} + + ;; Include EE tests. + ;; for ee dev: :dev:ee:ee-dev + ;; for ee tests: clojure -X:dev:ee:ee-dev:test + :ee-dev + {:extra-paths ["enterprise/backend/test"]} + + ;; these aliases exist for symmetry with the ee aliases. Empty for now. + :oss + {} + + :oss-dev + {} + + ;; for local dev -- include the drivers locally with :dev:drivers + :drivers + {:extra-deps + {metabase/driver-modules {:local/root "modules/drivers"}}} + + ;; for local dev: include drivers as well as their tests. + ;; + ;; clojure -X:dev:drivers:drivers-dev:test + ;; + ;; or + ;; + ;; clojure -X:dev:ee:ee-dev:drivers:drivers-dev:test (for EE) + :drivers-dev + {:extra-paths + ["modules/drivers/bigquery/test" + "modules/drivers/druid/test" + "modules/drivers/google/test" + "modules/drivers/googleanalytics/test" + "modules/drivers/mongo/test" + "modules/drivers/oracle/test" + "modules/drivers/presto/test" + "modules/drivers/presto-common/test" + "modules/drivers/presto-jdbc/test" + "modules/drivers/redshift/test" + "modules/drivers/snowflake/test" + "modules/drivers/sparksql/test" + "modules/drivers/sqlite/test" + "modules/drivers/sqlserver/test" + "modules/drivers/vertica/test"]} + +;;; Linters + + ;; clojure -X:dev:ee:ee-dev:drivers:drivers-dev:namespace-checker + :namespace-checker + {:exec-fn metabase.linters.namespace-checker/check-namespace-decls + :exec-args {:prefix-rewriting false}} + + ;; clojure -M:dev:ee:ee-dev:drivers:drivers-dev:check + :check + {:extra-deps {athos/clj-check {:git/url "https://github.com/athos/clj-check.git" + :sha "518d5a1cbfcd7c952f548e6dbfcb9a4a5faf9062"}} + :main-opts ["-m" "clj-check.check"]} + + ;; clojure -X:dev:ee:ee-dev:drivers:drivers-dev:eastwood + :eastwood + {:exec-fn metabase.linters.eastwood/eastwood + :exec-args {;; manually specify the source paths for the time being (exclude test paths) until we fix Eastwood + ;; errors in the test paths (once PR #17193 is merged) + :source-paths ["src" + "shared/src" + "enterprise/backend/src" + "modules/drivers/bigquery/src" + "modules/drivers/druid/src" + "modules/drivers/google/src" + "modules/drivers/googleanalytics/src" + "modules/drivers/mongo/src" + "modules/drivers/oracle/src" + "modules/drivers/presto/src" + "modules/drivers/presto-common/src" + "modules/drivers/presto-jdbc/src" + "modules/drivers/redshift/src" + "modules/drivers/snowflake/src" + "modules/drivers/sparksql/src" + "modules/drivers/sqlite/src" + "modules/drivers/sqlserver/src" + "modules/drivers/vertica/src"] + :add-linters [:unused-private-vars + ;; These linters are pretty useful but give a few false + ;; positives and can't be selectively disabled (yet) + ;; + ;; For example see https://github.com/jonase/eastwood/issues/193 + ;; + ;; It's still useful to re-enable them and run them every once + ;; in a while because they catch a lot of actual errors too. + ;; Keep an eye on the issue above and re-enable them if we can + ;; get them to work + #_:unused-fn-args + #_:unused-locals] + :exclude-linters [ ;; Turn this off temporarily until we finish removing + ;; self-deprecated functions & macros + :deprecations + ;; this has a fit in libs that use Potemkin `import-vars` such + ;; as `java-time` + :implicit-dependencies + ;; too many false positives for now + :unused-ret-vals]}} + + ;; clojure -X:dev:ee:ee-dev:test:cloverage + :cloverage + {:exec-fn cloverage/run-project + :exec-args {:fail-threshold 69 + :codecov? true + ;; don't instrument logging forms, since they won't get executed as part of tests anyway + ;; log calls expand to these + :exclude-call + [clojure.tools.logging/logf + clojure.tools.logging/logp] + + :source-ns-path + ["src" "enterprise/backend/src" "shared/src"] + + :test-ns-path + ["test" "enterprise/backend/test" "shared/test"] + + :ns-regex + ["^metabase.*" "^metabase-enterprise.*"] + + ;; don't instrument Postgres/MySQL driver namespaces, because we don't current run tests for them + ;; as part of recording test coverage, which means they can give us false positives. + ;; + ;; regex literals aren't allowed in EDN. We parse them in `./test/cloverage.clj` + :ns-exclude-regex + ["metabase\\.driver\\.mysql" "metabase\\.driver\\.postgres"]} + ;; different port from `:test` so you can run it at the same time as `:test`. + :jvm-opts ["-Dmb.jetty.port=3002"]} + +;;; building Uberjar + + ;; clojure -T:build uberjar + ;; clojure -T:build uberjar :edition :ee + :build + {:deps {io.github.clojure/tools.build {:git/tag "v0.1.6", :git/sha "5636e61"} + com.github.seancorfield/depstar {:tag "v2.1.267", :sha "1a45f79"} + metabase/build.common {:local/root "bin/common"} + metabase/buid-mb {:local/root "bin/build-mb"}} + :ns-default build} + +;;; Other misc convenience aliases + + ;; Profile Metabase start time with clojure -M:profile + :profile + {:main-opts ["-m" "metabase.core" "profile"] + :jvm-opts ["-XX:+CITime" ; print time spent in JIT compiler + "-XX:+PrintGC"]} + + ;; get the H2 shell with clojure -M:h2 + :h2 + {:main-opts ["-m" "org.h2.tools.Shell"]} + + ;; clojure -M:generate-automagic-dashboards-pot + :generate-automagic-dashboards-pot + {:main-opts ["-m" "metabase.automagic-dashboards.rules"]} + + ;; TODO -- consider creating an alias that includes the `./bin` build-drivers & release code as well so we can work + ;; on them all from a single REPL process. + }} diff --git a/docs/api-documentation.md b/docs/api-documentation.md index 0239548b98..d84047117e 100644 --- a/docs/api-documentation.md +++ b/docs/api-documentation.md @@ -1,6 +1,6 @@ # API Documentation for Metabase -_This file was generated from source comments by `lein run api-documentation`_. +_This file was generated from source comments by `clojure -M:run api-documentation`_. Check out an introduction to the [Metabase API](https://www.metabase.com/learn/developing-applications/advanced-metabase/metabase-api.html). diff --git a/docs/developers-guide.md b/docs/developers-guide.md index 4380e16486..ad3ffe4051 100644 --- a/docs/developers-guide.md +++ b/docs/developers-guide.md @@ -28,12 +28,11 @@ These are the tools which are required in order to complete any build of the Met 2. [Java Development Kit JDK (https://adoptopenjdk.net/releases.html)](https://adoptopenjdk.net/releases.html) - you need to install JDK 11 ([more info on Java versions](./operations-guide/java-versions.md)) 3. [Node.js (http://nodejs.org/)](http://nodejs.org/) - latest LTS release 4. [Yarn package manager for Node.js](https://yarnpkg.com/) - latest release of version 1.x - you can install it in any OS by doing `npm install --global yarn` -5. [Leiningen (http://leiningen.org/)](http://leiningen.org/) - latest release On a most recent stable Ubuntu/Debian, all the tools above, with the exception of Clojure, can be installed by using: ``` -sudo apt install openjdk-11-jdk nodejs leiningen && sudo npm install --global yarn +sudo apt install openjdk-11-jdk nodejs && sudo npm install --global yarn ``` If you have multiple JDK versions installed in your machine, be sure to switch your JDK before building by doing `sudo update-alternatives --config java` and selecting Java 11 in the menu @@ -79,7 +78,7 @@ $ yarn Run your backend development server with - lein run + clojure -M:run Start the frontend build process with @@ -150,12 +149,13 @@ Leiningen and your REPL are the main development tools for the backend. There ar And of course your Jetty development server is available via - lein run + clojure -M:run ### Building drivers -Most of the drivers Metabase uses to connect to external data warehouse databases are separate Leiningen projects under the `modules/` subdirectory. When running Metabase via `lein`, you'll -need to build these drivers in order to have access to them. You can build drivers as follows: +Most of the drivers Metabase uses to connect to external data warehouse databases are separate projects under the +`modules/` subdirectory. When running Metabase via `clojure`, you'll need to build these drivers in order to have access +to them. You can build drivers as follows: ``` # Build the 'mongo' driver @@ -169,41 +169,58 @@ need to build these drivers in order to have access to them. You can build drive ./bin/build-drivers.sh ``` -The first time you build a driver, it will be a bit slow, because Metabase needs to build the core project a couple of times so the driver can use it as a dependency; you can take comfort in the -fact that you won't need to build the driver again after that. Alternatively, running Metabase 1.0+ from the uberjar will unpack all of the pre-built drivers into your plugins directory; you can -do this instead if you already have a Metabase uberjar (just make sure `plugins` is in the root directory of the Metabase source, i.e. the same directory as `project.clj`). +### Including driver source paths for development or other tasks -### Including driver source paths for development or other Leiningen tasks - -For development when running various Leiningen tasks you can add the `include-all-drivers` profile to merge the drivers' dependencies and source paths into the Metabase -project: +For development when running various Leiningen tasks you can add the `drivers` and `drivers-dev` aliases to merge the +drivers' dependencies and source paths into the Metabase project: ``` -# Install dependencies -lein with-profiles +include-all-drivers deps +# Install dependencies, including for drivers +clojure -P -X:dev:ci:drivers:drivers-dev ``` -This profile is added by default when running `lein repl`, tests, and linters. - #### Unit Tests / Linting Run unit tests with - lein test + # OSS tests only + clojure -X:dev:test + + # OSS + EE tests + clojure -X:dev:ee:ee-dev:test + +or a specific test (or test namespace) with -or a specific test with + # run tests in only one namespace (pass in a symbol) + clojure -X:dev:test :only metabase.api.session-test - lein test metabase.api.session-test + # run one specific test (pass in a qualified symbol) + clojure -X:dev:test :only metabase.api.session-test/my-test + + # run tests in one specific folder (test/metabase/util in this example) + # pass arg in double-quotes so Clojure CLI interprets it as a string; + # our test runner treats strings as directories + clojure -X:dev:test :only '"test/metabase/util"' By default, the tests only run against the `h2` driver. You can specify which drivers to run tests against with the env var `DRIVERS`: - DRIVERS=h2,postgres,mysql,mongo lein test + DRIVERS=h2,postgres,mysql,mongo clojure -X:dev:drivers:drivers-dev:test Some drivers require additional environment variables when testing since they are impossible to run locally (such as Redshift and Bigquery). The tests will fail on launch and let you know what parameters to supply if needed. ##### Run the linters: - lein eastwood && lein bikeshed && lein docstring-checker && lein check-namespace-decls && ./bin/reflection-linter +`clj-kondo` must be installed separately; see https://github.com/clj-kondo/clj-kondo/blob/master/doc/install.md for +instructions. + + # Run Eastwood + clojure -X:dev:ee:ee-dev:drivers:drivers-dev:eastwood + + # Run the namespace checker + clojure -X:dev:ee:ee-dev:drivers:drivers-dev:namespace-checker + + # Run clj-kondo + clj-kondo --parallel --lint src shared/src enterprise/backend/src --config lint-config.edn #### Developing with Emacs diff --git a/enterprise/README.md b/enterprise/README.md index 272bec096a..6295a227d0 100644 --- a/enterprise/README.md +++ b/enterprise/README.md @@ -19,28 +19,15 @@ MB_EDITION=ee yarn build-hot ### Back-end -You need to add the `:ee` profile to the leiningen command to run Metabase Enterprise Edition. +You need to add the `:ee` alias to the Clojure CLI command to run Metabase Enterprise Edition. ```clj -lein with-profile +ee run -``` - -```clj -lein with-profile +ee uberjar -``` - -```clj -lein with-profile +ee repl -``` - -In Emacs/CIDER you can customize the `lein repl` command used to start the REPL by passing a prefix argument, e.g. - -```emacs-lisp -C-u M-x cider-jack-in -``` +# Start a local Metabase server that includes EE sources +clojure -M:ee:run -or, programatically: +# start a REPL that includes EE sources. +clojure -A:ee -```emacs-lisp -(cider-jack-in '(4)) +# start a REPL that includes EE sources & test namespaces. +clojure -A:dev:ee:ee-dev ``` diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/api/routes.clj b/enterprise/backend/src/metabase_enterprise/sandbox/api/routes.clj index 52e4f34380..5e1898527b 100644 --- a/enterprise/backend/src/metabase_enterprise/sandbox/api/routes.clj +++ b/enterprise/backend/src/metabase_enterprise/sandbox/api/routes.clj @@ -6,8 +6,7 @@ [metabase-enterprise.sandbox.api.user :as user] [metabase.server.middleware.auth :as middleware.auth])) -;; this is copied from `metabase.api.routes` because if we require that above we will destroy startup times for `lein -;; ring server` +;; Duplicated with `metabase.api.routes` but prevents circular deps between the two namespaces. (def ^:private +auth "Wrap `routes` so they may only be accessed with proper authentiaction credentials." middleware.auth/enforce-authentication) diff --git a/enterprise/backend/src/metabase_enterprise/sandbox/models/group_table_access_policy.clj b/enterprise/backend/src/metabase_enterprise/sandbox/models/group_table_access_policy.clj index 111fafcd2d..916a936e76 100644 --- a/enterprise/backend/src/metabase_enterprise/sandbox/models/group_table_access_policy.clj +++ b/enterprise/backend/src/metabase_enterprise/sandbox/models/group_table_access_policy.clj @@ -20,6 +20,19 @@ (models/defmodel GroupTableAccessPolicy :group_table_access_policy) +;; This guard is to make sure this file doesn't get compiled twice when building the uberjar -- that will totally +;; screw things up because Toucan models use Potemkin `defrecord+` under the hood. +(when *compile-files* + (defonce previous-compilation-trace (atom nil)) + (when @previous-compilation-trace + (println "THIS FILE HAS ALREADY BEEN COMPILED!!!!!") + (println "This compilation trace:") + ((requiring-resolve 'clojure.pprint/pprint) (vec (.getStackTrace (Thread/currentThread)))) + (println "Previous compilation trace:") + ((requiring-resolve 'clojure.pprint/pprint) @previous-compilation-trace) + (throw (ex-info "THIS FILE HAS ALREADY BEEN COMPILED!!!!!" {}))) + (reset! previous-compilation-trace (vec (.getStackTrace (Thread/currentThread))))) + (defn- normalize-attribute-remapping-targets [attribute-remappings] (m/map-vals normalize/normalize diff --git a/java/build.clj b/java/build.clj new file mode 100644 index 0000000000..a037e64fb6 --- /dev/null +++ b/java/build.clj @@ -0,0 +1,9 @@ +(ns build + (:require [clojure.tools.build.api :as b])) + +(defn compile-java [_] + (b/javac + {:src-dirs ["."] + :class-dir "target/classes" + :basis (b/create-basis {:aliases #{:compilation-basis}}) + :javac-opts ["-source" "8", "-target" "8"]})) diff --git a/java/deps.edn b/java/deps.edn new file mode 100644 index 0000000000..1c1934d771 --- /dev/null +++ b/java/deps.edn @@ -0,0 +1,15 @@ +{:deps/prep-lib + {:ensure "target/classes" + :alias :build + :fn compile-java} + + :paths ["target/classes"] + + :aliases + {:build + {:deps {io.github.clojure/tools.build {:git/tag "v0.1.6" :git/sha "5636e61"}} + :ns-default build} + + ;; dependencies needed for compiling the Java files. + :compilation-basis + {:extra-deps {org.liquibase/liquibase-core {:mvn/version "3.6.3"}}}}} diff --git a/lein-plugins/include-drivers/.lein-failures b/lein-plugins/include-drivers/.lein-failures deleted file mode 100644 index 9e26dfeeb6..0000000000 --- a/lein-plugins/include-drivers/.lein-failures +++ /dev/null @@ -1 +0,0 @@ -{} \ No newline at end of file diff --git a/lein-plugins/include-drivers/project.clj b/lein-plugins/include-drivers/project.clj deleted file mode 100644 index 149577d8aa..0000000000 --- a/lein-plugins/include-drivers/project.clj +++ /dev/null @@ -1,5 +0,0 @@ -(defproject metabase/lein-include-drivers "1.0.9" - :min-lein-version "2.5.0" - :eval-in-leiningen true - :deploy-repositories [["clojars" {:sign-releases false}]] - :dependencies [[colorize "0.1.1" :exclusions [org.clojure/clojure]]]) diff --git a/lein-plugins/include-drivers/src/leiningen/include_drivers.clj b/lein-plugins/include-drivers/src/leiningen/include_drivers.clj deleted file mode 100644 index 94160c0826..0000000000 --- a/lein-plugins/include-drivers/src/leiningen/include_drivers.clj +++ /dev/null @@ -1,147 +0,0 @@ -(ns leiningen.include-drivers - (:require [clojure.string :as str] - [colorize.core :as colorize] - [leiningen.core.project :as p]) - (:import java.io.File)) - -(defonce ^:private ^{:arglists '([s]), :doc "Log a message `s` the first time we see it. This middleware might run - multiple times, and we only really need to log a message the first time we see it."} - log-once - (comp (memoize println) str)) - -(defn- file-exists? [^String filename] - (.exists (File. filename))) - -(defn- driver-parents - "As listed in a 'parents' file in the module source directory. Add to the list of drivers to add sources for" - [driver] - (let [parents-file (File. (format "modules/drivers/%s/parents" driver))] - (when (.exists parents-file) - (str/split-lines (slurp parents-file))))) - -(defn- driver->project [driver] - (let [project-filename (format "modules/drivers/%s/project.clj" driver)] - (when (file-exists? project-filename) - (p/read project-filename)))) - -(defn- plugins-file-exists? [filename-pattern] - (some - (fn [filename] - (re-matches filename-pattern filename)) - (.list (File. "plugins")))) - -(defn- driver-dependencies-satisfied? - "If a driver's project specifies a list of dependency filenames like - - {:include-drivers-dependencies [#\"^ojdbc[78]\\.jar$\"]} - - Make sure a file matching that name pattern exists in the `/plugins` directory." - [driver] - {:pre [(string? driver) (seq driver)]} - (if-let [{:keys [include-drivers-dependencies]} (driver->project driver)] - (or (every? plugins-file-exists? include-drivers-dependencies) - (log-once - (colorize/color - :red - (format "[include-drivers middleware] Not including %s because not all dependencies matching %s found in /plugins" - driver (set include-drivers-dependencies))))) - (log-once - (colorize/color - :red - (format "[include-drivers middleware] Not including %s because we could not its project.clj" driver))))) - -;; if :include-drivers is specified in the project, and its value is `:all`, include all drivers; if it's a collection -;; of driver names, include the specified drivers; otherwise include whatever was set in the `DRIVERS` env var -(defn- test-drivers [{:keys [include-drivers]}] - (let [drivers - (cond - (= include-drivers :all) - (.list (File. "modules/drivers")) - - (coll? include-drivers) - include-drivers - - :else - (some-> (System/getenv "DRIVERS") (str/split #",") set (disj "h2" "postgres" "mysql"))) - - _ (log-once - (colorize/color - :magenta - (format "[include-drivers middleware] Attempting to include these drivers: %s" (set drivers)))) - - available-drivers - (for [driver drivers - :when (driver-dependencies-satisfied? driver)] - driver)] - (concat - available-drivers - (set (mapcat driver-parents available-drivers))))) - -(defn- test-drivers-source-paths [test-drivers] - (vec - (for [driver test-drivers - :let [source-path (format "modules/drivers/%s/src" driver)] - :when (file-exists? source-path)] - source-path))) - -(defn- test-drivers-test-paths [test-drivers] - (vec - (for [driver test-drivers - :let [test-path (format "modules/drivers/%s/test" driver)] - :when (file-exists? test-path)] - test-path))) - -(defn- test-drivers-test-paths [test-drivers] - (vec - (for [driver test-drivers - :let [test-path (format "modules/drivers/%s/test" driver)] - :when (file-exists? test-path)] - test-path))) - -(defn- test-drivers-projects [test-drivers] - (filter some? (map driver->project test-drivers))) - -(defn- test-drivers-dependencies [test-projects] - (vec - (for [{:keys [dependencies]} test-projects - dep dependencies - :when (not= :provided (keyword (:scope (apply array-map dep))))] - dep))) - -(defn- test-drivers-repositories [test-projects] - (vec - (for [{:keys [repositories]} test-projects - repo repositories] - repo))) - -(defn- test-drivers-aot [test-projects] - (vec - (for [{:keys [aot]} test-projects - ;; if aot is something like all we don't want to merge it into the MB project - :when (sequential? aot) - klass aot] - klass))) - -(defn- test-drivers-profile [project] - (let [test-drivers (test-drivers project) - test-projects (test-drivers-projects test-drivers)] - (log-once - (colorize/color - :magenta - (format "[include-drivers middleware] including these drivers: %s" (set test-drivers)))) - {:repositories (test-drivers-repositories test-projects) - :dependencies (test-drivers-dependencies test-projects) - :aot (test-drivers-aot test-projects) - :source-paths (test-drivers-source-paths test-drivers) - :test-paths (test-drivers-test-paths test-drivers)})) - -(defn middleware - "Add dependencies, source paths, and test paths for to Metabase drivers that are packaged as separate projects and - specified by the `DRIVERS` env var." - [project] - ;; When we merge a new profile into the project Leiningen will reload the project, which will cause our middleware - ;; to run a second time. Make sure we don't add the profile a second time or we'll be stuck in an infinite loop of - ;; adding a new profile and reloading. - (if (::has-driver-profiles? project) - project - (p/merge-profiles project [(test-drivers-profile project) {::has-driver-profiles? true}]))) diff --git a/modules/drivers/bigquery/deps.edn b/modules/drivers/bigquery/deps.edn new file mode 100644 index 0000000000..a37ecd0083 --- /dev/null +++ b/modules/drivers/bigquery/deps.edn @@ -0,0 +1,7 @@ +{:paths + ["src" "resources"] + + :deps + {com.google.apis/google-api-services-bigquery {:mvn/version "v2-rev20200523-1.30.9"}} + + :metabase.build-driver/parents #{:google}} diff --git a/modules/drivers/bigquery/project.clj b/modules/drivers/bigquery/project.clj deleted file mode 100644 index c7173597d6..0000000000 --- a/modules/drivers/bigquery/project.clj +++ /dev/null @@ -1,19 +0,0 @@ -(defproject metabase/bigquery-driver "1.0.0-SNAPSHOT-1.30.9" - :min-lein-version "2.5.0" - - :dependencies - [[com.google.apis/google-api-services-bigquery "v2-rev20200523-1.30.9"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"] - [metabase/google-driver "1.0.0-SNAPSHOT-1.30.7"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "bigquery.metabase-driver.jar"}}) diff --git a/modules/drivers/deps.edn b/modules/drivers/deps.edn new file mode 100644 index 0000000000..f189b15e42 --- /dev/null +++ b/modules/drivers/deps.edn @@ -0,0 +1,16 @@ +{:deps + {metabase/bigquery {:local/root "bigquery"} + metabase/druid {:local/root "druid"} + metabase/google {:local/root "google"} + metabase/googleanalytics {:local/root "googleanalytics"} + metabase/mongo {:local/root "mongo"} + metabase/oracle {:local/root "oracle"} + metabase/presto {:local/root "presto"} + metabase/presto-common {:local/root "presto-common"} + metabase/presto-jdbc {:local/root "presto-jdbc"} + metabase/redshift {:local/root "redshift"} + metabase/snowflake {:local/root "snowflake"} + metabase/sparksql {:local/root "sparksql"} + metabase/sqlite {:local/root "sqlite"} + metabase/sqlserver {:local/root "sqlserver"} + metabase/vertica {:local/root "vertica"}}} diff --git a/modules/drivers/druid/deps.edn b/modules/drivers/druid/deps.edn new file mode 100644 index 0000000000..aa7725e18a --- /dev/null +++ b/modules/drivers/druid/deps.edn @@ -0,0 +1 @@ +{:paths ["src" "resources"]} diff --git a/modules/drivers/druid/project.clj b/modules/drivers/druid/project.clj deleted file mode 100644 index 016a5898dc..0000000000 --- a/modules/drivers/druid/project.clj +++ /dev/null @@ -1,14 +0,0 @@ -(defproject metabase/druid-driver "1.0.0-SNAPSHOT" - :min-lein-version "2.5.0" - - :profiles - {:provided - {:dependencies [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "druid.metabase-driver.jar"}}) diff --git a/modules/drivers/google/deps.edn b/modules/drivers/google/deps.edn new file mode 100644 index 0000000000..16d6ab9145 --- /dev/null +++ b/modules/drivers/google/deps.edn @@ -0,0 +1,5 @@ +{:paths + ["src" "resources"] + + :deps + {com.google.api-client/google-api-client {:mvn/version "1.30.7"}}} diff --git a/modules/drivers/google/project.clj b/modules/drivers/google/project.clj deleted file mode 100644 index a7152cd0d0..0000000000 --- a/modules/drivers/google/project.clj +++ /dev/null @@ -1,26 +0,0 @@ -(defproject metabase/google-driver "1.0.0-SNAPSHOT-1.30.7" - :min-lein-version "2.5.0" - - :aliases - {"install-for-building-drivers" ["with-profile" "+install-for-building-drivers" "install"]} - - :dependencies - [[com.google.api-client/google-api-client "1.30.7"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [com.fasterxml.jackson.core/jackson-core "2.10.2"] ; Not sure why this is needed -- this is a dep of metabase-core - [metabase-core "1.0.0-SNAPSHOT"]]} - - :install-for-building-drivers - {:auto-clean true - :aot :all} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "google.metabase-driver.jar"}}) diff --git a/modules/drivers/googleanalytics/deps.edn b/modules/drivers/googleanalytics/deps.edn new file mode 100644 index 0000000000..301ac47e3a --- /dev/null +++ b/modules/drivers/googleanalytics/deps.edn @@ -0,0 +1,6 @@ +{:paths ["src" "resources"] + + :deps + {com.google.apis/google-api-services-analytics {:mvn/version "v3-rev20180622-1.27.0"}} + + :metabase.build-driver/parents #{:google}} diff --git a/modules/drivers/googleanalytics/project.clj b/modules/drivers/googleanalytics/project.clj deleted file mode 100644 index 5f0a229d77..0000000000 --- a/modules/drivers/googleanalytics/project.clj +++ /dev/null @@ -1,19 +0,0 @@ -(defproject metabase/googleanalytics-driver "1.0.0-SNAPSHOT-1.27.0" - :min-lein-version "2.5.0" - - :dependencies - [[com.google.apis/google-api-services-analytics "v3-rev20180622-1.27.0"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"] - [metabase/google-driver "1.0.0-SNAPSHOT-1.30.7"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "googleanalytics.metabase-driver.jar"}}) diff --git a/modules/drivers/mongo/deps.edn b/modules/drivers/mongo/deps.edn new file mode 100644 index 0000000000..bce9971be2 --- /dev/null +++ b/modules/drivers/mongo/deps.edn @@ -0,0 +1,5 @@ +{:paths + ["src" "resources"] + + :deps + {com.novemberain/monger {:mvn/version "3.5.0"}}} diff --git a/modules/drivers/mongo/project.clj b/modules/drivers/mongo/project.clj deleted file mode 100644 index 04181d7ce7..0000000000 --- a/modules/drivers/mongo/project.clj +++ /dev/null @@ -1,17 +0,0 @@ -(defproject metabase/mongo-driver "1.0.0-SNAPSHOT-3.9.0" - :min-lein-version "2.5.0" - - :dependencies - [[com.novemberain/monger "3.5.0"]] - - :profiles - {:provided - {:dependencies [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "mongo.metabase-driver.jar"}}) diff --git a/modules/drivers/mongo/src/metabase/driver/mongo.clj b/modules/drivers/mongo/src/metabase/driver/mongo.clj index 3cd980c382..78e4833164 100644 --- a/modules/drivers/mongo/src/metabase/driver/mongo.clj +++ b/modules/drivers/mongo/src/metabase/driver/mongo.clj @@ -12,7 +12,6 @@ [metabase.driver.mongo.parameters :as parameters] [metabase.driver.mongo.query-processor :as qp] [metabase.driver.mongo.util :refer [with-mongo-connection]] - [metabase.plugins.classloader :as classloader] [metabase.query-processor.store :as qp.store] [metabase.query-processor.timezone :as qp.timezone] [metabase.util :as u] @@ -20,6 +19,7 @@ [monger.command :as cmd] [monger.conversion :as m.conversion] [monger.db :as mdb] + monger.json [schema.core :as s] [taoensso.nippy :as nippy]) (:import com.mongodb.DB @@ -29,7 +29,7 @@ ;; See http://clojuremongodb.info/articles/integration.html Loading this namespace will load appropriate Monger ;; integrations with Cheshire. -(classloader/require 'monger.json) +(comment monger.json/keep-me) ;; JSON Encoding (etc.) diff --git a/modules/drivers/oracle/deps.edn b/modules/drivers/oracle/deps.edn new file mode 100644 index 0000000000..dacb8de8ed --- /dev/null +++ b/modules/drivers/oracle/deps.edn @@ -0,0 +1,14 @@ +{:paths + ["src" "resources-ee"] + + :deps + {com.oracle.ojdbc/ojdbc8 {:mvn/version "19.3.0.0"}} + + :aliases + {:oss + ;; JDBC driver isn't GPL-compatible + {:replace-deps {} + :replace-paths ["src" "resources"]} + + :ee + {}}} diff --git a/modules/drivers/oracle/project.clj b/modules/drivers/oracle/project.clj deleted file mode 100644 index d587bf9772..0000000000 --- a/modules/drivers/oracle/project.clj +++ /dev/null @@ -1,26 +0,0 @@ -(defproject metabase/oracle-driver "1.1.0" - :min-lein-version "2.5.0" - - :include-drivers-dependencies [#"^ojdbc\d+\.jar$"] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - ;; can't ship it as part of MB! - [com.oracle.ojdbc/ojdbc8 "19.3.0.0"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :ee - {:dependencies - [[com.oracle.ojdbc/ojdbc8 "19.3.0.0"]] - - :resource-paths - ^:replace ["resources-ee"]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "oracle.metabase-driver.jar"}}) diff --git a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj index 4b8618e04b..efb0213480 100644 --- a/modules/drivers/oracle/test/metabase/driver/oracle_test.clj +++ b/modules/drivers/oracle/test/metabase/driver/oracle_test.clj @@ -115,15 +115,16 @@ (tu/db-timezone-id))))) (deftest insert-rows-ddl-test - (is (= [[(str "INSERT ALL" - " INTO \"my_db\".\"my_table\" (\"col1\", \"col2\") VALUES (?, 1)" - " INTO \"my_db\".\"my_table\" (\"col1\", \"col2\") VALUES (?, 2) " - "SELECT * FROM dual") - "A" - "B"]] - (ddl/insert-rows-ddl-statements :oracle (hx/identifier :table "my_db" "my_table") [{:col1 "A", :col2 1} - {:col1 "B", :col2 2}])) - "Make sure we're generating correct DDL for Oracle to insert all rows at once.")) + (mt/test-driver :oracle + (testing "Make sure we're generating correct DDL for Oracle to insert all rows at once." + (is (= [[(str "INSERT ALL" + " INTO \"my_db\".\"my_table\" (\"col1\", \"col2\") VALUES (?, 1)" + " INTO \"my_db\".\"my_table\" (\"col1\", \"col2\") VALUES (?, 2) " + "SELECT * FROM dual") + "A" + "B"]] + (ddl/insert-rows-ddl-statements :oracle (hx/identifier :table "my_db" "my_table") [{:col1 "A", :col2 1} + {:col1 "B", :col2 2}])))))) (defn- do-with-temp-user [username f] (let [username (or username (tu/random-name))] diff --git a/modules/drivers/presto-common/deps.edn b/modules/drivers/presto-common/deps.edn new file mode 100644 index 0000000000..aa7725e18a --- /dev/null +++ b/modules/drivers/presto-common/deps.edn @@ -0,0 +1 @@ +{:paths ["src" "resources"]} diff --git a/modules/drivers/presto-common/project.clj b/modules/drivers/presto-common/project.clj deleted file mode 100644 index 6fd0ab15bb..0000000000 --- a/modules/drivers/presto-common/project.clj +++ /dev/null @@ -1,24 +0,0 @@ -(defproject metabase/presto-common-driver "1.0.0-SNAPSHOT" - :min-lein-version "2.5.0" - - :description "Common code for all Presto drivers. Defines HoneySQL behavior, query generation, etc." - - :aliases - {"install-for-building-drivers" ["with-profile" "+install-for-building-drivers" "install"]} - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :install-for-building-drivers - {:auto-clean true - :aot :all} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "presto-common.metabase-driver.jar"}}) diff --git a/modules/drivers/presto-jdbc/deps.edn b/modules/drivers/presto-jdbc/deps.edn new file mode 100644 index 0000000000..e457681728 --- /dev/null +++ b/modules/drivers/presto-jdbc/deps.edn @@ -0,0 +1,8 @@ +{:paths + ["src" "resources"] + + :deps + {com.facebook.presto/presto-jdbc {:mvn/version "0.254"}} + + :metabase.build-driver/parents + #{:presto-common}} diff --git a/modules/drivers/presto-jdbc/project.clj b/modules/drivers/presto-jdbc/project.clj deleted file mode 100644 index 9bd5b358a6..0000000000 --- a/modules/drivers/presto-jdbc/project.clj +++ /dev/null @@ -1,19 +0,0 @@ -(defproject metabase/presto-jdbc-driver "1.0.0-0.254-SNAPSHOT" - :min-lein-version "2.5.0" - - :dependencies - [[com.facebook.presto/presto-jdbc "0.254"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"] - [metabase/presto-common-driver "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "presto-jdbc.metabase-driver.jar"}}) diff --git a/modules/drivers/presto/deps.edn b/modules/drivers/presto/deps.edn new file mode 100644 index 0000000000..6f38a27789 --- /dev/null +++ b/modules/drivers/presto/deps.edn @@ -0,0 +1,2 @@ +{:paths ["src" "resources"] + :metabase.build-driver/parents #{:presto-common}} diff --git a/modules/drivers/presto/project.clj b/modules/drivers/presto/project.clj deleted file mode 100644 index 33c8f12c62..0000000000 --- a/modules/drivers/presto/project.clj +++ /dev/null @@ -1,16 +0,0 @@ -(defproject metabase/presto-driver "1.0.0-SNAPSHOT" - :min-lein-version "2.5.0" - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"] - [metabase/presto-common-driver "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "presto.metabase-driver.jar"}}) diff --git a/modules/drivers/presto/test/metabase/test/data/presto.clj b/modules/drivers/presto/test/metabase/test/data/presto.clj index 2b833a5ee2..31ae04f62c 100644 --- a/modules/drivers/presto/test/metabase/test/data/presto.clj +++ b/modules/drivers/presto/test/metabase/test/data/presto.clj @@ -11,7 +11,6 @@ [metabase.driver.sql.util :as sql.u] [metabase.driver.sql.util.unprepare :as unprepare] [metabase.test.data.interface :as tx] - [metabase.test.data.presto-common] [metabase.test.data.sql :as sql.tx])) (sql.tx/add-test-extensions! :presto) diff --git a/modules/drivers/redshift/deps.edn b/modules/drivers/redshift/deps.edn new file mode 100644 index 0000000000..3149e0d4db --- /dev/null +++ b/modules/drivers/redshift/deps.edn @@ -0,0 +1,8 @@ +{:paths + ["src" "resources"] + + :mvn/repos + {"redshift" {:url "https://s3.amazonaws.com/redshift-maven-repository/release"}} + + :deps + {com.amazon.redshift/redshift-jdbc42 {:mvn/version "2.0.0.3"}}} diff --git a/modules/drivers/redshift/project.clj b/modules/drivers/redshift/project.clj deleted file mode 100644 index 38c3bf833b..0000000000 --- a/modules/drivers/redshift/project.clj +++ /dev/null @@ -1,22 +0,0 @@ -(defproject metabase/redshift-driver "1.1.0-SNAPSHOT-2.0.0.3" - :min-lein-version "2.5.0" - - :repositories - [["redshift" "https://s3.amazonaws.com/redshift-maven-repository/release"]] - - - :dependencies - [[com.amazon.redshift/redshift-jdbc42 "2.0.0.3"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "redshift.metabase-driver.jar"}}) diff --git a/modules/drivers/snowflake/deps.edn b/modules/drivers/snowflake/deps.edn new file mode 100644 index 0000000000..cd559ec4d8 --- /dev/null +++ b/modules/drivers/snowflake/deps.edn @@ -0,0 +1,5 @@ +{:paths + ["src" "resources"] + + :deps + {net.snowflake/snowflake-jdbc {:mvn/version "3.12.13"}}} diff --git a/modules/drivers/snowflake/project.clj b/modules/drivers/snowflake/project.clj deleted file mode 100644 index 21392a3b9f..0000000000 --- a/modules/drivers/snowflake/project.clj +++ /dev/null @@ -1,18 +0,0 @@ -(defproject metabase/snowflake-driver "1.0.0-SNAPSHOT-3.12.7" - :min-lein-version "2.5.0" - - :dependencies - [[net.snowflake/snowflake-jdbc "3.12.13"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "snowflake.metabase-driver.jar"}}) diff --git a/modules/drivers/sparksql/build.clj b/modules/drivers/sparksql/build.clj new file mode 100644 index 0000000000..e4e15578c9 --- /dev/null +++ b/modules/drivers/sparksql/build.clj @@ -0,0 +1,16 @@ +(ns build + (:require [clojure.pprint :as pprint] + [clojure.tools.build.api :as b])) + +(defn aot [_] + (try + (b/compile-clj + {:src-dirs ["src"] + :class-dir "target/classes" + :basis (b/create-basis {:aliases #{:compilation-basis}}) + :ns-compile '[metabase.driver.FixedHiveConnection + metabase.driver.FixedHiveDriver]}) + (catch Throwable e + (println "Error AOT compiling Spark SQL namespaces:" (ex-message e)) + (pprint/pprint (Throwable->map e)) + (throw e)))) diff --git a/modules/drivers/sparksql/deps.edn b/modules/drivers/sparksql/deps.edn new file mode 100644 index 0000000000..03a35fe5a3 --- /dev/null +++ b/modules/drivers/sparksql/deps.edn @@ -0,0 +1,45 @@ +{:paths + ["src" "resources" "target/classes"] + + ;; Exclusions below are all either things that are already part of metabase-core, or provide conflicting + ;; implementations of things like log4j <-> slf4j, or are part of both hadoop-common and hive-jdbc; + :deps + {org.apache.hadoop/hadoop-common + {:mvn/version "3.1.1" + :exclusions [com.fasterxml.jackson.core/jackson-core + com.google.guava/guava + commons-logging/commons-logging + org.apache.httpcomponents/httpcore + org.codehaus.jackson/jackson-core-asl + org.codehaus.jackson/jackson-mapper-asl + org.eclipse.jetty/jetty-http + org.eclipse.jetty/jetty-io + org.eclipse.jetty/jetty-server + org.eclipse.jetty/jetty-util + org.slf4j/slf4j-log4j12 + org.tukaani/xz]} + + org.apache.hive/hive-jdbc + {:mvn/version "1.2.2" + :exclusions [commons-logging/commons-logging + org.apache.curator/curator-framework + org.codehaus.jackson/jackson-jaxrs + org.codehaus.jackson/jackson-xc + org.slf4j/slf4j-log4j12 + org.eclipse.jetty.aggregate/jetty-all]}} + + :deps/prep-lib + {:ensure "target/classes" + :alias :aot + :fn aot} + + :aliases + {:aot + {:deps {io.github.clojure/tools.build {:git/tag "v0.1.6" :git/sha "5636e61"}} + :ns-default build} + + ;; dependencies needed for AOT compilation. Same as deps above but without all the exclusions. + :compilation-basis + {:replace-deps {org.apache.hadoop/hadoop-common {:mvn/version "3.1.1"} + org.apache.hive/hive-jdbc {:mvn/version "1.2.2"} + org.clojure/clojure {:mvn/version "1.10.3"}}}}} diff --git a/modules/drivers/sparksql/project.clj b/modules/drivers/sparksql/project.clj deleted file mode 100644 index 6c9e52ab52..0000000000 --- a/modules/drivers/sparksql/project.clj +++ /dev/null @@ -1,45 +0,0 @@ -(defproject metabase/sparksql-driver "1.1.0-SNAPSHOT-1.2.2" - :min-lein-version "2.5.0" - - :dependencies - [ - ;; Exclusions below are all either things that are already part of metabase-core, or provide conflicting - ;; implementations of things like log4j <-> slf4j, or are part of both hadoop-common and hive-jdbc; - [org.apache.hadoop/hadoop-common "3.1.1" - :exclusions [com.fasterxml.jackson.core/jackson-core - com.google.guava/guava - commons-logging - org.apache.httpcomponents/httpcore - org.codehaus.jackson/jackson-core-asl - org.codehaus.jackson/jackson-mapper-asl - org.eclipse.jetty/jetty-http - org.eclipse.jetty/jetty-io - org.eclipse.jetty/jetty-server - org.eclipse.jetty/jetty-util - org.slf4j/slf4j-log4j12 - org.tukaani/xz]] - [org.apache.hive/hive-jdbc "1.2.2" - :exclusions - [commons-logging - org.apache.curator/curator-framework - org.codehaus.jackson/jackson-jaxrs - org.codehaus.jackson/jackson-xc - org.slf4j/slf4j-log4j12 - org.eclipse.jetty.aggregate/jetty-all]]] - - ;; only used for the lein with-drivers stuff (i.e. tests and REPL) - :aot [metabase.driver.FixedHiveConnection - metabase.driver.FixedHiveDriver] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "sparksql.metabase-driver.jar"}}) diff --git a/modules/drivers/sqlite/deps.edn b/modules/drivers/sqlite/deps.edn new file mode 100644 index 0000000000..e4fd0f6391 --- /dev/null +++ b/modules/drivers/sqlite/deps.edn @@ -0,0 +1,5 @@ +{:paths + ["src" "resources"] + + :deps + {org.xerial/sqlite-jdbc {:mvn/version "3.25.2"}}} diff --git a/modules/drivers/sqlite/project.clj b/modules/drivers/sqlite/project.clj deleted file mode 100644 index 2a78e44a53..0000000000 --- a/modules/drivers/sqlite/project.clj +++ /dev/null @@ -1,18 +0,0 @@ -(defproject metabase/sqlite-driver "1.0.0-SNAPSHOT-3.25.2" - :min-lein-version "2.5.0" - - :dependencies - [[org.xerial/sqlite-jdbc "3.25.2"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "sqlite.metabase-driver.jar"}}) diff --git a/modules/drivers/sqlserver/deps.edn b/modules/drivers/sqlserver/deps.edn new file mode 100644 index 0000000000..6f0f1649ce --- /dev/null +++ b/modules/drivers/sqlserver/deps.edn @@ -0,0 +1,5 @@ +{:paths + ["src" "resources"] + + :deps + {com.microsoft.sqlserver/mssql-jdbc {:mvn/version "9.2.1.jre8"}}} diff --git a/modules/drivers/sqlserver/project.clj b/modules/drivers/sqlserver/project.clj deleted file mode 100644 index 6edcf61449..0000000000 --- a/modules/drivers/sqlserver/project.clj +++ /dev/null @@ -1,18 +0,0 @@ -(defproject metabase/sqlserver-driver "1.1.0-SNAPSHOT-9.2.1.jre8" - :min-lein-version "2.5.0" - - :dependencies - [[com.microsoft.sqlserver/mssql-jdbc "9.2.1.jre8"]] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"]]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "sqlserver.metabase-driver.jar"}}) diff --git a/modules/drivers/vertica/deps.edn b/modules/drivers/vertica/deps.edn new file mode 100644 index 0000000000..67f110c912 --- /dev/null +++ b/modules/drivers/vertica/deps.edn @@ -0,0 +1,14 @@ +{:paths + ["src" "resources-ee"] + + :deps + {com.vertica.jdbc/vertica-jdbc {:mvn/version "10.0.1-0"}} + + :aliases + {:oss + ;; JDBC driver isn't GPL-compatible + {:replace-deps {} + :replace-paths ["src" "resources"]} + + :ee + {}}} diff --git a/modules/drivers/vertica/project.clj b/modules/drivers/vertica/project.clj deleted file mode 100644 index 7233a896c1..0000000000 --- a/modules/drivers/vertica/project.clj +++ /dev/null @@ -1,25 +0,0 @@ -(defproject metabase/vertica-driver "1.0.0-SNAPSHOT" - :min-lein-version "2.5.0" - - :include-drivers-dependencies [#"^vertica-jdbc-.*\.jar$"] - - :profiles - {:provided - {:dependencies - [[org.clojure/clojure "1.10.1"] - [metabase-core "1.0.0-SNAPSHOT"] - [com.vertica.jdbc/vertica-jdbc "10.0.1-0"]]} - - :ee - {:dependencies - [[com.vertica.jdbc/vertica-jdbc "10.0.1-0"]] - - :resource-paths - ^:replace ["resources-ee"]} - - :uberjar - {:auto-clean true - :aot :all - :javac-options ["-target" "1.8", "-source" "1.8"] - :target-path "target/%s" - :uberjar-name "vertica.metabase-driver.jar"}}) diff --git a/modules/drivers/vertica/test/metabase/test/data/vertica.clj b/modules/drivers/vertica/test/metabase/test/data/vertica.clj index a9b8bde156..b73bddf9a5 100644 --- a/modules/drivers/vertica/test/metabase/test/data/vertica.clj +++ b/modules/drivers/vertica/test/metabase/test/data/vertica.clj @@ -7,12 +7,12 @@ [java-time :as t] [medley.core :as m] [metabase.driver.sql-jdbc.connection :as sql-jdbc.conn] + [metabase.test :as mt] [metabase.test.data.interface :as tx] [metabase.test.data.sql :as sql.tx] [metabase.test.data.sql-jdbc :as sql-jdbc.tx] [metabase.test.data.sql-jdbc.execute :as execute] [metabase.test.data.sql-jdbc.load-data :as load-data] - [metabase.test :as mt] [metabase.util :as u] [metabase.util.files :as files])) diff --git a/project.clj b/project.clj deleted file mode 100644 index 4be94fdacb..0000000000 --- a/project.clj +++ /dev/null @@ -1,433 +0,0 @@ -;; -*- comment-column: 70; -*- -;; full set of options are here .. https://github.com/technomancy/leiningen/blob/master/sample.project.clj - -(defproject metabase-core "1.0.0-SNAPSHOT" - :description "Metabase" - :url "https://metabase.com/" - :min-lein-version "2.5.0" - - :aliases - {"profile" ["with-profile" "+profile" "run" "profile"] - "h2" ["with-profile" "+h2-shell" "run" "-url" "jdbc:h2:./metabase.db" - "-user" "" "-password" "" "-driver" "org.h2.Driver"] - "generate-automagic-dashboards-pot" ["with-profile" "+generate-automagic-dashboards-pot" "run"] - "install" ["with-profile" "+install" "install"] - "install-ee" ["with-profile" "+install,+ee" "install"] - "install-for-building-drivers" ["with-profile" "install-for-building-drivers" "install"] - "install-for-building-drivers-ee" ["with-profile" "install-for-building-drivers,+ee" "install"] - "run" ["with-profile" "+run" "run"] - "run-ee" ["with-profile" "+run,+ee" "run"] - "run-with-repl" ["with-profile" "+run-with-repl" "repl"] - "run-with-repl-ee" ["with-profile" "+run-with-repl,+ee" "repl"] - ;; "ring" ["with-profile" "+ring" "ring"] - ;; "ring-ee" ["with-profile" "+ring,+ee" "ring"] - "test" ["with-profile" "+test" "test"] - "test-ee" ["with-profile" "+test,+ee" "test"] - "check-namespace-decls" ["with-profile" "+check-namespace-decls" "check-namespace-decls"] - "eastwood" ["with-profile" "+eastwood" "eastwood"] - "cloverage" ["with-profile" "+cloverage" "cloverage"] - ;; `lein lint` will run all linters - "lint" ["do" ["eastwood"] ["check-namespace-decls"] ["cloverage"]] - "repl" ["with-profile" "+repl" "repl"] - "repl-ee" ["with-profile" "+repl,+ee" "repl"] - "uberjar" ["uberjar"] - "uberjar-ee" ["with-profile" "+ee" "uberjar"]} - - ;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - ;; !! PLEASE KEEP THESE ORGANIZED ALPHABETICALLY !! - ;; !! AND ADD A COMMENT EXPLAINING THEIR PURPOSE !! - ;; !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! - :dependencies - [[org.clojure/clojure "1.10.1"] - [org.clojure/core.async "0.4.500" - :exclusions [org.clojure/tools.reader]] - [joda-time/joda-time "2.10.8"] - [org.clojure/core.logic "1.0.0"] - [org.clojure/core.match "0.3.0"] ; optimized pattern matching library for Clojure - [org.clojure/core.memoize "1.0.236"] ; needed by core.match and search; has useful FIFO, LRU, etc. caching mechanisms - [org.clojure/data.csv "0.1.4"] ; CSV parsing / generation - [org.clojure/java.classpath "1.0.0"] ; examine the Java classpath from Clojure programs - [org.clojure/java.jdbc "0.7.11"] ; basic JDBC access from Clojure - [org.clojure/math.combinatorics "0.1.4"] ; combinatorics functions - [org.clojure/math.numeric-tower "0.0.4"] ; math functions like `ceil` - [org.clojure/tools.logging "1.1.0"] ; logging framework - [org.clojure/tools.namespace "1.0.0"] - [org.clojure/tools.trace "0.7.10"] ; function tracing - [amalloy/ring-buffer "1.2.2" - :exclusions [org.clojure/clojure - org.clojure/clojurescript]] ; fixed length queue implementation, used in log buffering - [amalloy/ring-gzip-middleware "0.1.4"] ; Ring middleware to GZIP responses if client can handle it - [aleph "0.4.6" :exclusions [org.clojure/tools.logging]] ; Async HTTP library; WebSockets - [bigml/histogram "4.1.3"] ; Histogram data structure - [buddy/buddy-core "1.5.0" ; various cryptograhpic functions - :exclusions [commons-codec - org.bouncycastle/bcprov-jdk15on - org.bouncycastle/bcpkix-jdk15on]] - [buddy/buddy-sign "3.0.0"] ; JSON Web Tokens; High-Level message signing library - [cheshire "5.10.0"] ; fast JSON encoding (used by Ring JSON middleware) - [clj-http "3.10.3" ; HTTP client - :exclusions [commons-codec - commons-io - slingshot]] - ;; fork to address #13102 - see upstream PR: https://github.com/dm3/clojure.java-time/pull/60 - ;; TODO: switch back to the upstream once a version is released with the above patch - [robdaemon/clojure.java-time "0.3.3-SNAPSHOT"] ; Java 8 java.time wrapper - [clojurewerkz/quartzite "2.1.0" ; scheduling library - :exclusions [c3p0]] - [colorize "0.1.1" :exclusions [org.clojure/clojure]] ; string output with ANSI color codes (for logging) - [com.cemerick/friend "0.2.3" ; auth library - :exclusions [commons-codec - org.apache.httpcomponents/httpclient - net.sourceforge.nekohtml/nekohtml - ring/ring-core - slingshot]] - [com.clearspring.analytics/stream "2.9.6" ; Various sketching algorithms - :exclusions [org.slf4j/slf4j-api - it.unimi.dsi/fastutil]] - [com.draines/postal "2.0.3"] ; SMTP library - [com.google.guava/guava "28.2-jre"] ; dep for BigQuery, Spark, and GA. Require here rather than letting different dep versions stomp on each other — see comments on #9697 - [com.h2database/h2 "1.4.197"] ; embedded SQL database - [com.taoensso/nippy "2.14.0"] ; Fast serialization (i.e., GZIP) library for Clojure - [commons-codec/commons-codec "1.15"] ; Apache Commons -- useful codec util fns - [commons-io/commons-io "2.8.0"] ; Apache Commons -- useful IO util fns - [commons-validator/commons-validator "1.6" ; Apache Commons -- useful validation util fns - :exclusions [commons-beanutils - commons-digester - commons-logging]] - [compojure "1.6.1" :exclusions [ring/ring-codec]] ; HTTP Routing library built on Ring - [crypto-random "1.2.0"] ; library for generating cryptographically secure random bytes and strings - [dk.ative/docjure "1.14.0" :exclusions [org.apache.poi/poi - org.apache.poi/poi-ooxml]] ; excel export - [environ "1.2.0"] ; easy environment management - [hiccup "1.0.5"] ; HTML templating - [honeysql "1.0.461" :exclusions [org.clojure/clojurescript]] ; Transform Clojure data structures to SQL - [instaparse "1.4.10"] ; Make your own parser - [io.forward/yaml "1.0.9" ; Clojure wrapper for YAML library SnakeYAML (which we already use for liquibase) - :exclusions [org.clojure/clojure - org.flatland/ordered - org.yaml/snakeyaml]] - [javax.xml.bind/jaxb-api "2.4.0-b180830.0359"] ; add the `javax.xml.bind` classes which we're still using but were removed in Java 11 - [kixi/stats "0.4.4" :exclusions [org.clojure/data.avl]] ; Various statistic measures implemented as transducers - [me.raynes/fs "1.4.6" ; Filesystem tools - :exclusions [org.apache.commons/commons-compress]] - [medley "1.3.0"] ; lightweight lib of useful functions - [metabase/connection-pool "1.1.1"] ; simple wrapper around C3P0. JDBC connection pools - [metabase/saml20-clj "2.0.0"] ; EE SAML integration - [metabase/throttle "1.0.2"] ; Tools for throttling access to API endpoints and other code pathways - [net.cgrand/macrovich "0.2.1"] ; utils for writing macros for both Clojure & ClojureScript - [net.redhogs.cronparser/cron-parser-core "3.4" ; describe Cron schedule in human-readable language - :exclusions [org.slf4j/slf4j-api joda-time]] ; exclude joda time 2.3 which has outdated timezone information - [net.sf.cssbox/cssbox "4.12" :exclusions [org.slf4j/slf4j-api]] ; HTML / CSS rendering - [org.apache.commons/commons-compress "1.20"] ; compression utils - [org.apache.commons/commons-lang3 "3.10"] ; helper methods for working with java.lang stuff - [org.apache.logging.log4j/log4j-api "2.13.3"] ; apache logging framework - [org.apache.logging.log4j/log4j-1.2-api "2.13.3"] ; add compatibility with log4j 1.2 - [org.apache.logging.log4j/log4j-core "2.13.3"] ; apache logging framework - [org.apache.logging.log4j/log4j-jcl "2.13.3"] ; allows the commons-logging API to work with log4j 2 - [org.apache.logging.log4j/log4j-liquibase "2.13.3"] ; liquibase logging via log4j 2 - [org.apache.logging.log4j/log4j-slf4j-impl "2.13.3"] ; allows the slf4j API to work with log4j 2 - [org.apache.poi/poi "5.0.0"] ; Work with Office documents (e.g. Excel spreadsheets) -- newer version than one specified by Docjure - [org.apache.poi/poi-ooxml "5.0.0" - :exclusions [org.bouncycastle/bcprov-jdk15on - org.bouncycastle/bcpkix-jdk15on]] - [org.apache.sshd/sshd-core "2.4.0"] ; ssh tunneling and test server - [org.bouncycastle/bcprov-jdk15on "1.68"] ; Bouncy Castle crypto library -- explicit version of BC specified to resolve illegal reflective access errors - [org.bouncycastle/bcpkix-jdk15on "1.68"] - [org.clojars.pntblnk/clj-ldap "0.0.16"] ; LDAP client - [org.eclipse.jetty/jetty-server "9.4.32.v20200930"] ; We require JDK 8 which allows us to run Jetty 9.4, ring-jetty-adapter runs on 1.7 which forces an older version - [org.flatland/ordered "1.5.9"] ; ordered maps & sets - [org.graalvm.js/js "21.0.0.2"] ; JavaScript engine - [org.graalvm.js/js-scriptengine "21.0.0.2"] - [org.liquibase/liquibase-core "3.6.3" ; migration management (Java lib) - :exclusions [ch.qos.logback/logback-classic]] - [org.mariadb.jdbc/mariadb-java-client "2.6.2"] ; MySQL/MariaDB driver - [org.postgresql/postgresql "42.2.18"] ; Postgres driver - [org.slf4j/slf4j-api "1.7.30"] ; abstraction for logging frameworks -- allows end user to plug in desired logging framework at deployment time - [org.tcrawley/dynapath "1.1.0"] ; Dynamically add Jars (e.g. Oracle or Vertica) to classpath - [org.threeten/threeten-extra "1.5.0"] ; extra Java 8 java.time classes like DayOfMonth and Quarter - [org.yaml/snakeyaml "1.23"] ; YAML parser (required by liquibase) - [potemkin "0.4.5" :exclusions [riddley]] ; utility macros & fns - [pretty "1.0.5"] ; protocol for defining how custom types should be pretty printed - [prismatic/schema "1.1.12"] ; Data schema declaration and validation library - [redux "0.1.4"] ; Utility functions for building and composing transducers - [riddley "0.2.0"] ; code walking lib -- used interally by Potemkin, manifold, etc. - [ring/ring-core "1.8.1"] - [ring/ring-jetty-adapter "1.8.1"] ; Ring adapter using Jetty webserver (used to run a Ring server for unit tests) - [ring/ring-json "0.5.0"] ; Ring middleware for reading/writing JSON automatically - [slingshot "0.12.2"] ; enhanced throw/catch, used by other deps - [stencil "0.5.0"] ; Mustache templates for Clojure - [toucan "1.15.3" :exclusions [org.clojure/java.jdbc ; Model layer, hydration, and DB utilities - org.clojure/tools.logging - org.clojure/tools.namespace - honeysql]] - [user-agent "0.1.0"] ; User-Agent string parser, for Login History page & elsewhere - [weavejester/dependency "0.2.1"] ; Dependency graphs and topological sorting - [org.clojure/java.jmx "1.0.0"]] ; JMX bean library, for exporting diagnostic info - - :main ^:skip-aot metabase.core - - :manifest - {;; Liquibase uses this manifest parameter to dynamically find extensions at startup (via classpath scanning, etc) - "Liquibase-Package" - #= (eval - (str "liquibase.change,liquibase.changelog,liquibase.database,liquibase.parser,liquibase.precondition," - "liquibase.datatype,liquibase.serializer,liquibase.sqlgenerator,liquibase.executor," - "liquibase.snapshot,liquibase.logging,liquibase.diff,liquibase.structure," - "liquibase.structurecompare,liquibase.lockservice,liquibase.sdk,liquibase.ext"))} - - :jvm-opts - ["-XX:+IgnoreUnrecognizedVMOptions" ; ignore things not recognized for our Java version instead of refusing to start - "-Djava.awt.headless=true"] ; prevent Java icon from randomly popping up in dock when running `lein ring server` - - :target-path "target/%s" - - :javac-options - ["-target" "1.8", "-source" "1.8"] - - :source-paths - ["src" "backend/mbql/src" "shared/src"] - - :java-source-paths - ["java"] - - :uberjar-name - "metabase.jar" - - :profiles - {:oss ; exists for symmetry with the ee profile - {} - - :ee - {:source-paths ["enterprise/backend/src"] - :test-paths ["enterprise/backend/test"]} - - :socket - {:dependencies - [[vlaaad/reveal "1.3.196"]] - :jvm-opts - ["-Dclojure.server.repl={:port 5555 :accept clojure.core.server/repl}"]} - - :dev - {:source-paths ["dev/src" "local/src"] - :test-paths ["test" "backend/mbql/test" "shared/test"] - - :dependencies - [[clj-http-fake "1.0.3" :exclusions [slingshot]] ; Library to mock clj-http responses - [jonase/eastwood "0.9.4" :exclusions [org.clojure/clojure]] ; to run Eastwood - [methodical "0.9.4-alpha"] - [pjstadig/humane-test-output "0.10.0"] - [reifyhealth/specmonstah "2.0.0"] ; Generate fixtures to test huge databases - [ring/ring-mock "0.4.0"] - [talltale "0.5.4"]] ; Generate realistic data for fixtures - - - :plugins - [[lein-environ "1.1.0"] ; easy access to environment variables - [lein-licenses "LATEST"]] - - :injections - [(require 'pjstadig.humane-test-output) - (pjstadig.humane-test-output/activate!) - ;; redefs lives in the `test/` directory; it's only relevant to tests, so if it's not there (e.g. when running - ;; `lein ring server` or the like) it doesn't matter - (try (require 'metabase.test.redefs) - (catch Throwable _))] - - :env - {:mb-run-mode "dev" - :mb-field-filter-operators-enabled "true" - :mb-test-env-setting "ABCDEFG"} - - :jvm-opts - ["-Dlogfile.path=target/log"] - - :repl-options - {:init-ns user ; starting in the user namespace is a lot faster than metabase.core since it has less deps - :timeout 240000}} - - ;; output test results in JUnit XML format - :junit - {:dependencies - [[pjstadig/humane-test-output "0.10.0"] - [test-report-junit-xml "0.2.0"]] - - :plugins - [[lein-test-report-junit-xml "0.2.0"]] - - ;; the custom JUnit formatting logic lives in `backend/junit/test/metabase/junit.clj` - :test-paths ["backend/junit/test"] - - :injections - [(require 'metabase.junit)] - - :test-report-junit-xml - {:output-dir "target/junit" - :format-result metabase.junit/format-result}} - - :ci - {} - - :install - {} - - :install-for-building-drivers - {:auto-clean true - :aot :all} - - :exclude-tests - {:test-paths ^:replace []} - - :run - [:include-all-drivers - :exclude-tests {}] - - :run-with-repl - [:exclude-tests - :include-all-drivers - - {:env - {:mb-jetty-join "false"} - - :repl-options - {:init (do (require 'metabase.core) - (metabase.core/-main)) - :timeout 240000}}] - - ;; DISABLED FOR NOW SINCE IT'S BROKEN -- SEE #12181 - ;; start the dev HTTP server with 'lein ring server' - ;; :ring - ;; [:exclude-tests - ;; :include-all-drivers - ;; {:dependencies - ;; ;; used internally by lein ring to track namespace changes. Newer version contains fix by yours truly with 1000x - ;; ;; faster launch time - ;; [[ns-tracker "0.4.0"]] - - ;; :plugins - ;; [[lein-ring "0.12.5" :exclusions [org.clojure/clojure]]] - - ;; :ring - ;; {:handler metabase.server.handler/app - ;; :init metabase.core/init! - ;; :async? true - ;; :destroy metabase.core/destroy - ;; :reload-paths ["src"]}}] - - :with-include-drivers-middleware - {:plugins - [[metabase/lein-include-drivers "1.0.9"]] - - :middleware - [leiningen.include-drivers/middleware]} - - ;; shared config used by various commands that run tests (lein test and lein cloverage) - :test-common - {:resource-paths - ["test_resources"] - - :env - {:mb-run-mode "test" - :mb-db-in-memory "true" - :mb-jetty-join "false" - :mb-field-filter-operators-enabled "true" - :mb-api-key "test-api-key" - ;; use a random port between 3001 and 3501. That way if you run multiple sets of tests at the same time locally - ;; they won't stomp on each other - :mb-jetty-port #= (eval (str (+ 3001 (rand-int 500))))} - - :jvm-opts - ["-Duser.timezone=UTC" - "-Duser.language=en"]} - - :test - [:with-include-drivers-middleware :test-common] - - :include-all-drivers - [:with-include-drivers-middleware - {:include-drivers :all - :injections - [(require 'metabase.plugins) - (metabase.plugins/load-plugins!)]}] - - :repl - [:include-all-drivers - ;; so running the tests doesn't give you different answers - {:jvm-opts ["-Duser.timezone=UTC"]}] - - ;; shared stuff between all linter profiles. - :linters-common - [:include-all-drivers - :ee - :test-common - ;; always use in-memory H2 database for linters - {:env {:mb-db-type "h2"}}] - - :eastwood - [:linters-common - {:plugins - [[jonase/eastwood "0.9.4" :exclusions [org.clojure/clojure]]] - - :eastwood - {:exclude-namespaces [dev dev.test dev.debug-qp] - :config-files ["./test_resources/eastwood-config.clj"] - :add-linters [:unused-private-vars - ;; These linters are pretty useful but give a few false positives and can't be selectively - ;; disabled (yet) - ;; - ;; For example see https://github.com/jonase/eastwood/issues/193 - ;; - ;; It's still useful to re-enable them and run them every once in a while because they catch - ;; a lot of actual errors too. Keep an eye on the issue above and re-enable them if we can - ;; get them to work - #_:unused-fn-args - #_:unused-locals] - :exclude-linters [:deprecations - ;; this has a fit in libs that use Potemkin `import-vars` such as `java-time` - :implicit-dependencies - ;; too many false positives for now - :unused-ret-vals - :unused-ret-vals-in-try - :def-in-def - :wrong-ns-form]}}] - - :check-namespace-decls - [:linters-common - {:plugins [[lein-check-namespace-decls "1.0.3"]] - :check-namespace-decls {:prefix-rewriting false} - ;; include test namespaces - :source-paths ["test" "enterprise/backend/test" "shared/test"]}] - - :cloverage - [:test-common - {:dependencies [[camsaul/cloverage "1.2.1.1" :exclusions [riddley]]] - :plugins [[camsaul/lein-cloverage "1.2.1.1"]] - :source-paths ^:replace ["src" "backend/mbql/src" "enterprise/backend/src" "shared/src"] - :test-paths ^:replace ["test" "backend/mbql/test" "enterprise/backend/test" "shared/test"] - :cloverage {:fail-threshold 69 - :exclude-call - [;; don't instrument logging forms, since they won't get executed as part of tests anyway - ;; log calls expand to these - clojure.tools.logging/logf - clojure.tools.logging/logp] - ;; don't instrument Postgres/MySQL driver namespaces, because we don't current run tests for them - ;; as part of recording test coverage, which means they can give us false positives. - :ns-exclude-regex - [#"metabase\.driver\.mysql" - #"metabase\.driver\.postgres"]}}] - - ;; build the uberjar with `lein uberjar` - :uberjar - {:auto-clean true - :aot :all} - - ;; Profile Metabase start time with `lein profile` - :profile - {:jvm-opts ["-XX:+CITime" ; print time spent in JIT compiler - "-XX:+PrintGC"]} ; print a message when garbage collection takes place - - ;; get the H2 shell with 'lein h2' - :h2-shell - {:main org.h2.tools.Shell} - - :generate-automagic-dashboards-pot - {:main metabase.automagic-dashboards.rules}}) diff --git a/src/metabase/api/activity.clj b/src/metabase/api/activity.clj index 6f127ed903..d794cf1f3c 100644 --- a/src/metabase/api/activity.clj +++ b/src/metabase/api/activity.clj @@ -7,6 +7,7 @@ [metabase.models.dashboard :refer [Dashboard]] [metabase.models.interface :as mi] [metabase.models.view-log :refer [ViewLog]] + [metabase.server.middleware.offset-paging :as offset-paging] [toucan.db :as db] [toucan.hydrate :refer [hydrate]])) @@ -61,9 +62,11 @@ (defendpoint GET "/" "Get recent activity." [] - (filter mi/can-read? (-> (db/select Activity, {:order-by [[:timestamp :desc]], :limit 40}) - (hydrate :user :table :database) - add-model-exists-info))) + (filter mi/can-read? (-> (db/select Activity {:order-by [[:timestamp :desc]] + :limit offset-paging/*limit* + :offset offset-paging/*offset*}) + (hydrate :user :table :database) + add-model-exists-info))) (defn- view-log-entry->matching-object [{:keys [model model_id]}] (when (contains? #{"card" "dashboard"} model) diff --git a/src/metabase/api/email.clj b/src/metabase/api/email.clj index 4b61e1b7b2..085b1beac5 100644 --- a/src/metabase/api/email.clj +++ b/src/metabase/api/email.clj @@ -6,7 +6,6 @@ [clojure.tools.logging :as log] [compojure.core :refer [DELETE POST PUT]] [metabase.api.common :as api] - [metabase.config :as config] [metabase.email :as email] [metabase.models.setting :as setting] [metabase.util.i18n :refer [tru]] @@ -69,24 +68,18 @@ [:as {settings :body}] {settings su/Map} (api/check-superuser) - (let [email-settings (select-keys settings (keys mb-to-smtp-settings)) - smtp-settings (-> (set/rename-keys email-settings mb-to-smtp-settings) - (assoc :port (some-> (:email-smtp-port settings) Integer/parseInt))) - ;; TODO - this is :thumbs_down`, we should just use `with-redefs` for `email/test-smtp-connection` where - ;; needed in the tests. FIXME! - response (if-not config/is-test? - ;; in normal conditions, validate connection - (email/test-smtp-connection smtp-settings) - ;; for unit testing just respond with a success message - {:error :SUCCESS}) - tested-settings (merge settings (select-keys response [:port :security])) - [_ corrections _] (data/diff settings tested-settings) + (let [email-settings (select-keys settings (keys mb-to-smtp-settings)) + smtp-settings (-> (set/rename-keys email-settings mb-to-smtp-settings) + (assoc :port (some-> (:email-smtp-port settings) Integer/parseInt))) + response (email/test-smtp-connection smtp-settings) + tested-settings (merge settings (select-keys response [:port :security])) + [_ corrections _] (data/diff settings tested-settings) properly-named-corrections (set/rename-keys corrections (set/map-invert mb-to-smtp-settings)) - corrected-settings (merge email-settings properly-named-corrections)] + corrected-settings (merge email-settings properly-named-corrections)] (if (= :SUCCESS (:error response)) ;; test was good, save our settings (assoc (setting/set-many! corrected-settings) - :with-corrections (humanize-email-corrections properly-named-corrections)) + :with-corrections (humanize-email-corrections properly-named-corrections)) ;; test failed, return response message {:status 500 :body (humanize-error-messages response)}))) diff --git a/src/metabase/async/streaming_response.clj b/src/metabase/async/streaming_response.clj index f2087ad25f..69c9d39ac9 100644 --- a/src/metabase/async/streaming_response.clj +++ b/src/metabase/async/streaming_response.clj @@ -206,7 +206,7 @@ (p.types/deftype+ StreamingResponse [f options donechan] pretty/PrettyPrintable (pretty [_] - (list (symbol (str (.getCanonicalName StreamingResponse) \.)) f options)) + (list (pretty/qualify-symbol-for-*ns* `->StreamingResponse) f options donechan)) server.protocols/Respond (respond [this context] @@ -222,12 +222,6 @@ (send* [this request respond* _] (respond* (compojure.response/render this request)))) - ;; TODO -- if we want this to work when running via `lein ring server` we need to add an impl for - ;; `ring.core.protocols/StreamableResponseBody`. Not sure if we want to do that because it would result in different - ;; behavior when running via `lein ring server` vs `lein run`/uberjar. Maybe better just to take `lein ring server` - ;; out and replace it with an auto-reload version of `lein run` - - ;; TODO -- don't think any of this is needed any mo (defn- render [^StreamingResponse streaming-response gzip?] (let [{:keys [headers content-type], :as options} (.options streaming-response)] diff --git a/src/metabase/automagic_dashboards/rules.clj b/src/metabase/automagic_dashboards/rules.clj index 7d5a39fd32..eae634df4f 100644 --- a/src/metabase/automagic_dashboards/rules.clj +++ b/src/metabase/automagic_dashboards/rules.clj @@ -367,7 +367,9 @@ rules)))) (defn -main - "Entry point for lein task `generate-automagic-dashboards-pot`" + "Entry point for Clojure CLI task `generate-automagic-dashboards-pot`. Run it with + + clojure -M:generate-automagic-dashboards-pot" [& _] (->> (all-rules) (mapcat extract-localized-strings) diff --git a/src/metabase/cmd.clj b/src/metabase/cmd.clj index decada5d8d..0e5cc98a7b 100644 --- a/src/metabase/cmd.clj +++ b/src/metabase/cmd.clj @@ -1,15 +1,14 @@ (ns metabase.cmd - "Functions for commands that can be ran from the command-line with `lein` or the Metabase JAR. These are ran as - follows: + "Functions for commands that can be ran from the command-line with the Clojure CLI or the Metabase JAR. These are ran + as follows: for example, running the `migrate` command and passing it `force` can be done using one of the following ways: - lein run migrate force + clojure -M:run migrate force java -jar metabase.jar migrate force - Logic below translates resolves the command itself to a function marked with `^:command` metadata and calls the function with arguments as appropriate. @@ -18,6 +17,7 @@ (:refer-clojure :exclude [load]) (:require [clojure.string :as str] [clojure.tools.logging :as log] + [environ.core :as env] [medley.core :as m] [metabase.config :as config] [metabase.mbql.util :as mbql.u] @@ -64,8 +64,7 @@ "Start Metabase the usual way and exit. Useful for profiling Metabase launch time." [] ;; override env var that would normally make Jetty block forever - (classloader/require 'environ.core 'metabase.core) - (alter-var-root #'environ.core/env assoc :mb-jetty-join "false") + (alter-var-root #'env/env assoc :mb-jetty-join "false") (u/profile "start-normally" ((resolve 'metabase.core/start-normally)))) (defn ^:command reset-password @@ -185,7 +184,7 @@ (System/exit 1)))) (defn run-cmd - "Run `cmd` with `args`. This is a function above. e.g. `lein run metabase migrate force` becomes + "Run `cmd` with `args`. This is a function above. e.g. `clojure -M:run metabase migrate force` becomes `(migrate \"force\")`." [cmd args] (try (apply (cmd->fn cmd) args) diff --git a/src/metabase/cmd/dump_to_h2.clj b/src/metabase/cmd/dump_to_h2.clj index fc29de397e..542648eafa 100644 --- a/src/metabase/cmd/dump_to_h2.clj +++ b/src/metabase/cmd/dump_to_h2.clj @@ -3,7 +3,7 @@ Run this as follows (h2 filename is optional): - lein run dump-to-h2 '/path/to/metabase.db/' + clojure -M:run dump-to-h2 '/path/to/metabase.db/' or @@ -11,7 +11,7 @@ Validate with: - lein run load-from-h2 '/path/to/metabase.db'" + clojure -M:run load-from-h2 '\"/path/to/metabase.db\"'" (:require [clojure.tools.logging :as log] [metabase.cmd.copy :as copy] [metabase.cmd.copy.h2 :as copy.h2] diff --git a/src/metabase/cmd/endpoint_dox.clj b/src/metabase/cmd/endpoint_dox.clj index ee172c3a37..bb8e4c6ec3 100644 --- a/src/metabase/cmd/endpoint_dox.clj +++ b/src/metabase/cmd/endpoint_dox.clj @@ -10,7 +10,7 @@ [] (str "# API Documentation for Metabase" "\n\n" - "_This file was generated from source comments by `lein run api-documentation`_." + "_This file was generated from source comments by `clojure -M:run api-documentation`_." "\n\n" "Check out an introduction to the " "[Metabase API](https://www.metabase.com/learn/developing-applications/advanced-metabase/metabase-api.html)." diff --git a/src/metabase/cmd/load_from_h2.clj b/src/metabase/cmd/load_from_h2.clj index 1e69add93f..444f315614 100644 --- a/src/metabase/cmd/load_from_h2.clj +++ b/src/metabase/cmd/load_from_h2.clj @@ -1,7 +1,7 @@ (ns metabase.cmd.load-from-h2 "Commands for loading data from an H2 file into another database. Run this with - lein run load-from-h2 + clojure -M:run load-from-h2 or @@ -12,11 +12,11 @@ # Postgres psql -c 'DROP DATABASE IF EXISTS metabase;' psql -c 'CREATE DATABASE metabase;' - MB_DB_TYPE=postgres MB_DB_HOST=localhost MB_DB_PORT=5432 MB_DB_USER=camsaul MB_DB_DBNAME=metabase lein run load-from-h2 + MB_DB_TYPE=postgres MB_DB_HOST=localhost MB_DB_PORT=5432 MB_DB_USER=camsaul MB_DB_DBNAME=metabase clojure -M:run load-from-h2 # MySQL mysql -u root -e 'DROP DATABASE IF EXISTS metabase; CREATE DATABASE metabase;' - MB_DB_TYPE=mysql MB_DB_HOST=localhost MB_DB_PORT=3305 MB_DB_USER=root MB_DB_DBNAME=metabase lein run load-from-h2" + MB_DB_TYPE=mysql MB_DB_HOST=localhost MB_DB_PORT=3305 MB_DB_USER=root MB_DB_DBNAME=metabase clojure -M:run load-from-h2" (:require [metabase.cmd.copy :as copy] [metabase.cmd.copy.h2 :as copy.h2] [metabase.db.connection :as mdb.conn] diff --git a/src/metabase/cmd/refresh_integration_test_db_metadata.clj b/src/metabase/cmd/refresh_integration_test_db_metadata.clj index 8e847b8c3f..fb2de11796 100644 --- a/src/metabase/cmd/refresh_integration_test_db_metadata.clj +++ b/src/metabase/cmd/refresh_integration_test_db_metadata.clj @@ -1,6 +1,6 @@ (ns metabase.cmd.refresh-integration-test-db-metadata (:require [clojure.java.io :as io] - [environ.core :refer [env]] + [environ.core :as env] [metabase.db :as mdb] [metabase.models.database :refer [Database]] [metabase.models.field :refer [Field]] @@ -22,7 +22,7 @@ [] (let [db-path (test-fixture-db-path)] ;; now set the path at MB_DB_FILE - (alter-var-root #'environ.core/env assoc :mb-db-type "h2", :mb-db-file db-path) + (alter-var-root #'env/env assoc :mb-db-type "h2", :mb-db-file db-path) ;; set up the DB, make sure sample dataset is added (mdb/setup-db!) (sample-data/add-sample-dataset!) diff --git a/src/metabase/config.clj b/src/metabase/config.clj index 7baff9c5f6..fa27943410 100644 --- a/src/metabase/config.clj +++ b/src/metabase/config.clj @@ -70,9 +70,9 @@ (defn ^Boolean config-bool "Fetch a configuration key and parse it as a boolean." [k] (some-> k config-str Boolean/parseBoolean)) (defn ^Keyword config-kw "Fetch a configuration key and parse it as a keyword." [k] (some-> k config-str keyword)) -(def ^Boolean is-dev? "Are we running in `dev` mode (i.e. in a REPL or via `lein ring server`)?" (= :dev (config-kw :mb-run-mode))) -(def ^Boolean is-prod? "Are we running in `prod` mode (i.e. from a JAR)?" (= :prod (config-kw :mb-run-mode))) -(def ^Boolean is-test? "Are we running in `test` mode (i.e. via `lein test`)?" (= :test (config-kw :mb-run-mode))) +(def ^Boolean is-dev? "Are we running in `dev` mode (i.e. in a REPL or via `clojure -M:run`)?" (= :dev (config-kw :mb-run-mode))) +(def ^Boolean is-prod? "Are we running in `prod` mode (i.e. from a JAR)?" (= :prod (config-kw :mb-run-mode))) +(def ^Boolean is-test? "Are we running in `test` mode (i.e. via `clojure -X:test`)?" (= :test (config-kw :mb-run-mode))) ;;; Version stuff @@ -127,14 +127,3 @@ (def ^Keyword mb-session-cookie-samesite "Value for session cookie's `SameSite` directive. Must be one of \"none\", \"lax\", or \"strict\" (case insensitive)." (mb-session-cookie-samesite*)) - - -;; This only affects dev: -;; -;; If for some wacky reason the test namespaces are getting loaded (e.g. when running via -;; `lein ring` or `lein ring sever`, DO NOT RUN THE EXPECTATIONS TESTS AT SHUTDOWN! THIS WILL NUKE YOUR APPLICATION DB -(try - (classloader/require 'expectations) - ((resolve 'expectations/disable-run-on-shutdown)) - ;; This will fail if the test dependencies aren't present (e.g. in a JAR situation) which is totally fine - (catch Throwable _)) diff --git a/src/metabase/core.clj b/src/metabase/core.clj index d8e5d679b5..9b04eef9a1 100644 --- a/src/metabase/core.clj +++ b/src/metabase/core.clj @@ -160,5 +160,5 @@ [& [cmd & args]] (maybe-enable-tracing) (if cmd - (run-cmd cmd args) ; run a command like `java -jar metabase.jar migrate release-locks` or `lein run migrate release-locks` + (run-cmd cmd args) ; run a command like `java -jar metabase.jar migrate release-locks` or `clojure -M:run migrate release-locks` (start-normally))) ; with no command line args just start Metabase normally diff --git a/src/metabase/models/card.clj b/src/metabase/models/card.clj index 47a8d97bf1..a2d6963357 100644 --- a/src/metabase/models/card.clj +++ b/src/metabase/models/card.clj @@ -166,7 +166,7 @@ For the OSS edition, there is no implementation for this function -- it is a no-op. For Metabase Enterprise Edition, the implementation of this function is - `metabase-enterprise.sandbox.models.group-table-access-policy/update-card-check-gtaps` and is installed by that + [[metabase-enterprise.sandbox.models.group-table-access-policy/update-card-check-gtaps]] and is installed by that namespace."} pre-update-check-sandbox-constraints (atom identity)) diff --git a/src/metabase/models/setting.clj b/src/metabase/models/setting.clj index 7a790f8d6d..4947029810 100644 --- a/src/metabase/models/setting.clj +++ b/src/metabase/models/setting.clj @@ -242,7 +242,13 @@ (defn get-json "Get the string value of `setting-definition-or-name` and parse it as JSON." [setting-definition-or-name] - (json/parse-string (get-string setting-definition-or-name) keyword)) + (try + (json/parse-string (get-string setting-definition-or-name) keyword) + (catch Throwable e + (let [{setting-name :name} (resolve-setting setting-definition-or-name)] + (throw (ex-info (tru "Error parsing JSON setting {0}: {1}" setting-name (ex-message e)) + {:setting setting-name} + e)))))) (defn get-timestamp "Get the string value of `setting-definition-or-name` and parse it as an ISO-8601-formatted string, returning a diff --git a/src/metabase/plugins.clj b/src/metabase/plugins.clj index f92312d52a..e6965e9092 100644 --- a/src/metabase/plugins.clj +++ b/src/metabase/plugins.clj @@ -153,7 +153,7 @@ * Metabase creates the plugins directory if it does not already exist. * Any plugins that are shipped as part of Metabase itself are extracted from the Metabase uberjar (or `resources` - directory when running with `lein`) into the plugins directory. + directory when running with the Clojure CLI) into the plugins directory. * Each JAR in the plugins directory that *does not* include a Metabase plugin manifest is added to the classpath. * For JARs that include a Metabase plugin manifest (a `metabase-plugin.yaml` file), a lazy-loading Metabase driver is registered; when the driver is initialized (automatically, when certain methods are called) the JAR is added diff --git a/src/metabase/plugins/classloader.clj b/src/metabase/plugins/classloader.clj index a7f3c953a1..281bfad44b 100644 --- a/src/metabase/plugins/classloader.clj +++ b/src/metabase/plugins/classloader.clj @@ -23,9 +23,9 @@ (delay ;; If the Clojure runtime base loader is already an instance of DynamicClassLoader (e.g. it is something like ;; `clojure.lang.Compiler/LOADER` we can go ahead and use that in the future. This is usually the case when doing - ;; REPL-based development or running via `lein`; when running from the UberJAR `clojure.lang.Compiler/LOADER` is - ;; not set and thus this will return the current thread's context classloader, which is usually just the System - ;; classloader. + ;; REPL-based development or running via the Clojure CLI; when running from the UberJAR + ;; `clojure.lang.Compiler/LOADER` is not set and thus this will return the current thread's context classloader, + ;; which is usually just the System classloader. ;; ;; The base loader is what Clojure ultimately uses to loading namespaces with `require` so adding URLs to it is ;; they way to go, if we can) @@ -116,21 +116,24 @@ Added benefit -- this is also thread-safe, unlike vanilla require." [& args] - ;; done for side-effects to ensure context classloader is the right one - (the-classloader) - ;; as elsewhere make sure Clojure is using our context classloader (which should normally be true anyway) because - ;; that's the one that will have access to the JARs we've added to the classpath at runtime - (try - (binding [*use-context-classloader* true] - ;; serialize requires - (locking clojure.lang.RT/REQUIRE_LOCK - (apply clojure.core/require args))) - (catch Throwable e - (throw (ex-info (.getMessage e) - {:classloader (the-classloader) - :classpath-urls (map str (dynapath/all-classpath-urls (the-classloader))) - :system-classpath (sort (str/split (System/getProperty "java.class.path") #"[:;]"))} - e))))) + ;; during compilation, don't load any namespaces. This is going to totally screw up our compilation because + ;; namespaces can end up being compiled twice + (when-not *compile-files* + ;; done for side-effects to ensure context classloader is the right one + (the-classloader) + ;; as elsewhere make sure Clojure is using our context classloader (which should normally be true anyway) because + ;; that's the one that will have access to the JARs we've added to the classpath at runtime + (try + (binding [*use-context-classloader* true] + ;; serialize requires + (locking clojure.lang.RT/REQUIRE_LOCK + (apply clojure.core/require args))) + (catch Throwable e + (throw (ex-info (.getMessage e) + {:classloader (the-classloader) + :classpath-urls (map str (dynapath/all-classpath-urls (the-classloader))) + :system-classpath (sort (str/split (System/getProperty "java.class.path") #"[:;]"))} + e)))))) (defonce ^:private already-added (atom #{})) diff --git a/src/metabase/sample_data.clj b/src/metabase/sample_data.clj index 4b27b6fc4a..240680db91 100644 --- a/src/metabase/sample_data.clj +++ b/src/metabase/sample_data.clj @@ -16,7 +16,7 @@ (throw (Exception. (trs "Sample dataset DB file ''{0}'' cannot be found." sample-dataset-filename)))) {:db (-> (.getPath resource) - (str/replace #"^file:" "zip:") ; to connect to an H2 DB inside a JAR just replace file: with zip: (this doesn't do anything when running from `lein`, which has no `file:` prefix) + (str/replace #"^file:" "zip:") ; to connect to an H2 DB inside a JAR just replace file: with zip: (this doesn't do anything when running from the Clojure CLI, which has no `file:` prefix) (str/replace #"\.mv\.db$" "") ; strip the .mv.db suffix from the path (str/replace #"%20" " ") ; for some reason the path can get URL-encoded and replace spaces with `%20`; this breaks things so switch them back to spaces (str ";USER=GUEST;PASSWORD=guest"))})) ; specify the GUEST user account created for the DB diff --git a/test/cloverage.clj b/test/cloverage.clj new file mode 100644 index 0000000000..0a97ff1a7e --- /dev/null +++ b/test/cloverage.clj @@ -0,0 +1,12 @@ +(ns cloverage + (:require cloverage.coverage)) + +(defn run-project [options] + ;; parse regex lists into actual regex Patterns since regex literals aren't allowed in EDN. + (let [options (reduce + (fn [options k] + (cond-> options + (seq (k options)) (update k (partial map re-pattern)))) + options + [:ns-regex :ns-exclude-regex])] + (cloverage.coverage/run-project options))) diff --git a/test/metabase/api/activity_test.clj b/test/metabase/api/activity_test.clj index eaa8b0b498..5061a15fa5 100644 --- a/test/metabase/api/activity_test.clj +++ b/test/metabase/api/activity_test.clj @@ -72,7 +72,7 @@ :user nil})] ;; remove other activities from the API response just in case -- we're not interested in those (let [these-activity-ids (set (map u/the-id [activity1 activity2 activity3]))] - (for [activity (mt/user-http-request :crowberto :get 200 "activity") + (for [activity (mt/user-http-request :crowberto :get 200 "activity" :limit Integer/MAX_VALUE) :when (contains? these-activity-ids (u/the-id activity))] (dissoc activity :timestamp))))))))) @@ -167,7 +167,6 @@ "user" #{90}} (#'activity-api/activities->referenced-objects fake-activities)))) - (deftest referenced-objects->existing-objects-test (mt/with-temp Dashboard [{dashboard-id :id}] (is (= {"dashboard" #{dashboard-id}, "card" nil} diff --git a/test/metabase/api/collection_test.clj b/test/metabase/api/collection_test.clj index 939a22db6c..478681dc9f 100644 --- a/test/metabase/api/collection_test.clj +++ b/test/metabase/api/collection_test.clj @@ -52,9 +52,8 @@ :authority_level nil :id "root"} (assoc (into {} collection) :can_write true)] - (for [collection (mt/user-http-request :crowberto :get 200 "collection") - :when (not (:personal_owner_id collection))] - collection))))) + (filter #(#{(:id collection) "root"} (:id %)) + (mt/user-http-request :crowberto :get 200 "collection")))))) (testing "We should only see our own Personal Collections!" (is (= ["Lucky Pigeon's Personal Collection"] @@ -80,21 +79,32 @@ (is (= ["Our analytics" "Collection 1" "Rasta Toucan's Personal Collection"] - (map :name (mt/user-http-request :rasta :get 200 "collection"))))))) - - (testing "check that we don't see collections if they're archived" - (mt/with-temp* [Collection [collection-1 {:name "Archived Collection", :archived true}] - Collection [collection-2 {:name "Regular Collection"}]] - (is (= ["Our analytics" - "Rasta Toucan's Personal Collection" - "Regular Collection"] - (map :name (mt/user-http-request :rasta :get 200 "collection")))))) + (->> (mt/user-http-request :rasta :get 200 "collection") + (filter (fn [{collection-name :name}] + (or (#{"Our analytics" "Collection 1" "Collection 2"} collection-name) + (str/includes? collection-name "Personal Collection")))) + (map :name))))))) + + (mt/with-temp* [Collection [collection-1 {:name "Archived Collection", :archived true}] + Collection [collection-2 {:name "Regular Collection"}]] + (letfn [(remove-other-collections [collections] + (filter (fn [{collection-name :name}] + (or (#{"Our analytics" "Archived Collection" "Regular Collection"} collection-name) + (str/includes? collection-name "Personal Collection"))) + collections))] + (testing "check that we don't see collections if they're archived" + (is (= ["Our analytics" + "Rasta Toucan's Personal Collection" + "Regular Collection"] + (->> (mt/user-http-request :rasta :get 200 "collection") + remove-other-collections + (map :name))))) - (testing "Check that if we pass `?archived=true` we instead see archived Collections" - (mt/with-temp* [Collection [collection-1 {:name "Archived Collection", :archived true}] - Collection [collection-2 {:name "Regular Collection"}]] - (is (= ["Archived Collection"] - (map :name (mt/user-http-request :rasta :get 200 "collection" :archived :true)))))) + (testing "Check that if we pass `?archived=true` we instead see archived Collections" + (is (= ["Archived Collection"] + (->> (mt/user-http-request :rasta :get 200 "collection" :archived :true) + remove-other-collections + (map :name))))))) (testing "?namespace= parameter" (mt/with-temp* [Collection [{normal-id :id} {:name "Normal Collection"}] @@ -140,6 +150,8 @@ (deftest collection-tree-test (testing "GET /api/collection/tree" (let [personal-collection (collection/user->personal-collection (mt/user->id :rasta))] + (testing "sanity check" + (is (some? personal-collection))) (with-collection-hierarchy [a b c d e f g] (let [ids (set (map :id (cons personal-collection [a b c d e f g]))) response (mt/user-http-request :rasta :get 200 "collection/tree")] @@ -306,6 +318,13 @@ true)) items)) +(defn- remove-non-personal-collections + [items] + (remove (fn [{:keys [model name]}] + (when (= model "collection") + (not (str/includes? name "Personal Collection")))) + items)) + (defn- default-item [{:keys [model] :as item-map}] (merge {:id true, :collection_position nil} (when (= model "collection") @@ -793,32 +812,41 @@ (testing "Make sure you can see everything for Users that can see everything" (is (= [(default-item {:name "Birthday Card", :description nil, :favorite false, :model "card", :display "table"}) (collection-item "Crowberto Corv's Personal Collection") - (default-item {:name "Dine & Dashboard", - :favorite false, :description nil, :model "dashboard"}) + (default-item {:name "Dine & Dashboard", :favorite false, :description nil, :model "dashboard"}) (default-item {:name "Electro-Magnetic Pulse", :model "pulse"}) ] (with-some-children-of-collection nil (-> (:data (mt/user-http-request :crowberto :get 200 "collection/root/items")) (remove-non-test-items &ids) + remove-non-personal-collections mt/boolean-ids-and-timestamps)))) (testing "... with limits and offsets" - (is (= [(default-item {:name "Birthday Card", - :favorite false, :display "table" :description nil, :model "card"}) - (collection-item "Crowberto Corv's Personal Collection")] - (with-some-children-of-collection nil - (-> (:data (mt/user-http-request :crowberto :get 200 "collection/root/items" :limit "2" :offset "1")) - (remove-non-test-items &ids) - mt/boolean-ids-and-timestamps))))) + (with-some-children-of-collection nil + (letfn [(items [limit offset] + (:data (mt/user-http-request :crowberto :get 200 "collection/root/items" + :limit (str limit), :offset (str offset))))] + (let [[a-1 b-1 :as items-1] (items 2 0)] + (is (= 2 + (count items-1))) + (let [[a-2 b-2 :as items-2] (items 2 1)] + (is (= 2 + (count items-2))) + (is (= b-1 a-2)) + (is (not= items-1 items-2))))))) (testing "... with a total back, too, even with limit and offset" - (is (= 5 (with-some-children-of-collection nil - (:total (mt/user-http-request :crowberto :get 200 "collection/root/items" :limit "2" :offset "1")))))) + ;; `:total` should be at least 5 items based on `with-some-children-of-collection`. Might be a bit more if + ;; other stuff was created + (is (<= 5 (with-some-children-of-collection nil + (:total (mt/user-http-request :crowberto :get 200 "collection/root/items" :limit "2" :offset "1")))))) (testing "...but we don't let you see stuff you wouldn't otherwise be allowed to see" (is (= [(collection-item "Rasta Toucan's Personal Collection")] ;; if a User doesn't have perms for the Root Collection then they don't get to see things with no collection_id (with-some-children-of-collection nil - (mt/boolean-ids-and-timestamps (:data (mt/user-http-request :rasta :get 200 "collection/root/items")))))) + (-> (:data (mt/user-http-request :rasta :get 200 "collection/root/items")) + remove-non-personal-collections + mt/boolean-ids-and-timestamps)))) (testing "...but if they have read perms for the Root Collection they should get to see them" (with-some-children-of-collection nil @@ -831,6 +859,7 @@ (collection-item "Rasta Toucan's Personal Collection")] (-> (:data (mt/user-http-request :rasta :get 200 "collection/root/items")) (remove-non-test-items &ids) + remove-non-personal-collections mt/boolean-ids-and-timestamps )))))))) (testing "So I suppose my Personal Collection should show up when I fetch the Root Collection, shouldn't it..." @@ -888,8 +917,17 @@ (defn- api-get-root-collection-children [& additional-get-params] - (mt/boolean-ids-and-timestamps (:data (apply mt/user-http-request :rasta :get 200 "collection/root/items" additional-get-params))) ) + (mt/boolean-ids-and-timestamps (:data (apply mt/user-http-request :rasta :get 200 "collection/root/items" additional-get-params)))) + +(defn- remove-non-test-collections [items] + (filter (fn [{collection-name :name}] + (or (str/includes? collection-name "Personal Collection") + (#{"A" "B" "C" "D" "E" "F" "G"} collection-name))) + items)) + (deftest fetch-root-collection-items-test + (testing "sanity check" + (is (collection/user->personal-collection (mt/user->id :rasta)))) (testing "GET /api/collection/root/items" (testing "Do top-level collections show up as children of the Root Collection?" (with-collection-hierarchy [a b c d e f g] @@ -899,7 +937,7 @@ (api-get-root-collection-ancestors)))) (testing "children" (is (= (map collection-item ["A" "Rasta Toucan's Personal Collection"]) - (api-get-root-collection-children)))))) + (remove-non-test-collections (api-get-root-collection-children))))))) (testing "...and collapsing children should work for the Root Collection as well" (with-collection-hierarchy [b d e f g] @@ -909,7 +947,7 @@ (api-get-root-collection-ancestors)))) (testing "children" (is (= (map collection-item ["B" "D" "F" "Rasta Toucan's Personal Collection"]) - (api-get-root-collection-children)))))) + (remove-non-test-collections (api-get-root-collection-children))))))) (testing "does `archived` work on Collections as well?" (with-collection-hierarchy [a b d e f g] @@ -920,7 +958,7 @@ (api-get-root-collection-ancestors :archived true)))) (testing "children" (is (= [(collection-item "A")] - (api-get-root-collection-children :archived true)))))) + (remove-non-test-collections (api-get-root-collection-children :archived true))))))) (testing "\n?namespace= parameter" (mt/with-temp* [Collection [{normal-id :id} {:name "Normal Collection"}] diff --git a/test/metabase/api/database_test.clj b/test/metabase/api/database_test.clj index 083242efd2..c2279008d6 100644 --- a/test/metabase/api/database_test.clj +++ b/test/metabase/api/database_test.clj @@ -62,7 +62,7 @@ (defn- expected-tables [db-or-id] (map table-details (db/select Table - :db_id (u/the-id db-or-id), :active true + :db_id (u/the-id db-or-id), :active true, :visibility_type nil {:order-by [[:%lower.schema :asc] [:%lower.display_name :asc]]}))) (defn- field-details [field] @@ -807,9 +807,9 @@ ;; run the cards to populate their result_metadata columns (doseq [card [card-1 card-2]] ((mt/user->client :crowberto) :post 202 (format "card/%d/query" (u/the-id card)))) - (is (= ["Everything else" - "My Collection"] - ((mt/user->client :lucky) :get 200 (format "database/%d/schemas" mbql.s/saved-questions-virtual-database-id)))))) + (let [schemas (set ((mt/user->client :lucky) :get 200 (format "database/%d/schemas" mbql.s/saved-questions-virtual-database-id)))] + (is (contains? schemas "Everything else")) + (is (contains? schemas "My Collection"))))) (testing "null and empty schemas should both come back as blank strings" (mt/with-temp* [Database [{db-id :id}] diff --git a/test/metabase/api/email_test.clj b/test/metabase/api/email_test.clj index c238b04f31..127c9e0211 100644 --- a/test/metabase/api/email_test.clj +++ b/test/metabase/api/email_test.clj @@ -1,7 +1,8 @@ (ns metabase.api.email-test (:require [clojure.test :refer :all] + [metabase.email :as email] [metabase.models.setting :as setting] - [metabase.test.data.users :refer [user->client]] + [metabase.test :as mt] [metabase.test.util :as tu])) (defn- email-settings @@ -26,24 +27,29 @@ (testing "PUT /api/email" (tu/discard-setting-changes [email-smtp-host email-smtp-port email-smtp-security email-smtp-username email-smtp-password email-from-address] - ((user->client :crowberto) :put 200 "email" default-email-settings) - (is (= default-email-settings - (email-settings)))))) + (with-redefs [email/test-smtp-connection (constantly {:error :SUCCESS})] + (is (= (assoc default-email-settings + :with-corrections {}) + (mt/user-http-request :crowberto :put 200 "email" default-email-settings))) + (is (= default-email-settings + (email-settings))))))) -;; (deftest clear-email-settings-test (testing "DELETE /api/email" (tu/discard-setting-changes [email-smtp-host email-smtp-port email-smtp-security email-smtp-username email-smtp-password email-from-address] - ((user->client :crowberto) :put 200 "email" default-email-settings) - (let [new-email-settings (email-settings)] - ((user->client :crowberto) :delete 204 "email") - (is (= default-email-settings - new-email-settings)) - (is (= {:email-smtp-host nil - :email-smtp-port nil - :email-smtp-security "none" - :email-smtp-username nil - :email-smtp-password nil - :email-from-address "notifications@metabase.com"} - (email-settings))))))) + (with-redefs [email/test-smtp-connection (constantly {:error :SUCCESS})] + (is (= (assoc default-email-settings + :with-corrections {}) + (mt/user-http-request :crowberto :put 200 "email" default-email-settings))) + (let [new-email-settings (email-settings)] + (is (nil? (mt/user-http-request :crowberto :delete 204 "email"))) + (is (= default-email-settings + new-email-settings)) + (is (= {:email-smtp-host nil + :email-smtp-port nil + :email-smtp-security "none" + :email-smtp-username nil + :email-smtp-password nil + :email-from-address "notifications@metabase.com"} + (email-settings)))))))) diff --git a/test/metabase/api/field_test.clj b/test/metabase/api/field_test.clj index 33da18d958..53543135c8 100644 --- a/test/metabase/api/field_test.clj +++ b/test/metabase/api/field_test.clj @@ -62,9 +62,9 @@ :has_field_values "list" :dimensions [] :name_field nil}) - (m/dissoc-in [:table :db :updated_at] [:table :db :created_at])) + (m/dissoc-in [:table :db :updated_at] [:table :db :created_at] [:table :db :timezone])) (-> (mt/user-http-request :rasta :get 200 (format "field/%d" (mt/id :users :name))) - (m/dissoc-in [:table :db :updated_at] [:table :db :created_at])))))) + (m/dissoc-in [:table :db :updated_at] [:table :db :created_at] [:table :db :timezone])))))) (deftest get-field-summary-test (testing "GET /api/field/:id/summary" diff --git a/test/metabase/api/search_test.clj b/test/metabase/api/search_test.clj index fe6691e0b4..1b4f077635 100644 --- a/test/metabase/api/search_test.clj +++ b/test/metabase/api/search_test.clj @@ -221,7 +221,7 @@ (is (= 2 (count (search-request-data :crowberto :q "test" :limit "2" :offset "0")))))) (testing "It offsets matches properly" (with-search-items-in-root-collection "test" - (is (= 4 (count (search-request-data :crowberto :q "test" :limit "100" :offset "2")))))) + (is (<= 4 (count (search-request-data :crowberto :q "test" :limit "100" :offset "2")))))) (testing "It subsets matches for model" (with-search-items-in-root-collection "test" (is (= 0 (count (search-request-data :crowberto :q "test" :models "database")))) diff --git a/test/metabase/api/user_test.clj b/test/metabase/api/user_test.clj index 39343d2390..b1becacc8d 100644 --- a/test/metabase/api/user_test.clj +++ b/test/metabase/api/user_test.clj @@ -579,8 +579,8 @@ {:email "adifferentemail@metabase.com"})))))))) (defn- do-with-preserved-rasta-personal-collection-name [thunk] - (let [{collection-name :name, collection-id :id} (collection/user->personal-collection (mt/user->id :rasta))] - (mt/with-temp-vals-in-db Collection collection-id {:name collection-name} + (let [{collection-name :name, :keys [slug id]} (collection/user->personal-collection (mt/user->id :rasta))] + (mt/with-temp-vals-in-db Collection id {:name collection-name, :slug slug} (thunk)))) (defmacro ^:private with-preserved-rasta-personal-collection-name @@ -669,7 +669,14 @@ {:group_ids [(u/the-id (group/all-users)) (u/the-id (group/admin))]}) (is (= {:is-superuser? true, :pgm-exists? true} - (superuser-and-admin-pgm-info email))))))) + (superuser-and-admin-pgm-info email)))))) + + (testing "Double-check that the test cleaned up after itself" + (is (= "Rasta" + (db/select-one-field :first_name User :id (mt/user->id :rasta)))) + (is (= {:name "Rasta Toucan's Personal Collection" + :slug "rasta_toucan_s_personal_collection"} + (mt/derecordize (db/select-one [Collection :name :slug] :personal_owner_id (mt/user->id :rasta))))))) (deftest update-locale-test (testing "PUT /api/user/:id\n" diff --git a/test/metabase/linters/common.clj b/test/metabase/linters/common.clj new file mode 100644 index 0000000000..c96cb17409 --- /dev/null +++ b/test/metabase/linters/common.clj @@ -0,0 +1,9 @@ +(ns metabase.linters.common + (:require [clojure.java.classpath :as classpath] + [clojure.string :as str])) + +(defn source-paths [] + (filter (fn [^java.io.File file] + (and (.isDirectory ^java.io.File file) + (not (str/ends-with? (.getName file) "resources")))) + (classpath/system-classpath))) diff --git a/test/metabase/linters/eastwood.clj b/test/metabase/linters/eastwood.clj new file mode 100644 index 0000000000..18af4c381d --- /dev/null +++ b/test/metabase/linters/eastwood.clj @@ -0,0 +1,9 @@ +(ns metabase.linters.eastwood + (:require [eastwood.lint :as eastwood] + [metabase.linters.common :as common])) + +(defn eastwood [options] + (eastwood/eastwood-from-cmdline + (merge + {:source-paths (common/source-paths)} + options))) diff --git a/test/metabase/linters/namespace_checker.clj b/test/metabase/linters/namespace_checker.clj new file mode 100644 index 0000000000..5ffb38bda7 --- /dev/null +++ b/test/metabase/linters/namespace_checker.clj @@ -0,0 +1,9 @@ +(ns metabase.linters.namespace-checker + (:require [check-namespace-decls.core :as check-ns] + [metabase.linters.common :as common])) + +(defn check-namespace-decls [options] + (check-ns/check-namespace-decls + (merge + {:source-paths (common/source-paths)} + options))) diff --git a/test/metabase/models/dashboard_test.clj b/test/metabase/models/dashboard_test.clj index f942b6e735..e22d638456 100644 --- a/test/metabase/models/dashboard_test.clj +++ b/test/metabase/models/dashboard_test.clj @@ -3,7 +3,7 @@ [metabase.api.common :as api] [metabase.automagic-dashboards.core :as magic] [metabase.models.card :refer [Card]] - [metabase.models.collection :refer [Collection]] + [metabase.models.collection :as collection :refer [Collection]] [metabase.models.dashboard :as dashboard :refer :all] [metabase.models.dashboard-card :as dashboard-card :refer [DashboardCard]] [metabase.models.dashboard-card-series :refer [DashboardCardSeries]] @@ -235,17 +235,13 @@ (deftest transient-dashboards-test (testing "test that we save a transient dashboard" (tu/with-model-cleanup [Card Dashboard DashboardCard Collection] - (binding [api/*current-user-id* (users/user->id :rasta) - api/*current-user-permissions-set* (-> :rasta - users/user->id - user/permissions-set - atom)] - (let [dashboard (magic/automagic-analysis (Table (id :venues)) {}) - rastas-personal-collection (db/select-one-field :id 'Collection - :personal_owner_id api/*current-user-id*) - saved-dashboard (save-transient-dashboard! dashboard rastas-personal-collection)] - (is (= (db/count 'DashboardCard :dashboard_id (:id saved-dashboard)) - (-> dashboard :ordered_cards count)))))))) + (let [rastas-personal-collection (collection/user->personal-collection (users/user->id :rasta))] + (binding [api/*current-user-id* (users/user->id :rasta) + api/*current-user-permissions-set* (-> :rasta users/user->id user/permissions-set atom)] + (let [dashboard (magic/automagic-analysis (Table (id :venues)) {}) + saved-dashboard (save-transient-dashboard! dashboard (u/the-id rastas-personal-collection))] + (is (= (db/count DashboardCard :dashboard_id (u/the-id saved-dashboard)) + (-> dashboard :ordered_cards count))))))))) (deftest validate-collection-namespace-test (mt/with-temp Collection [{collection-id :id} {:namespace "currency"}] diff --git a/test/metabase/models/database_test.clj b/test/metabase/models/database_test.clj index fcd4dc0d2b..79fc5f73ed 100644 --- a/test/metabase/models/database_test.clj +++ b/test/metabase/models/database_test.clj @@ -8,9 +8,9 @@ [metabase.models.database :as mdb] [metabase.models.permissions :as perms] [metabase.models.user :as user] - [metabase.plugins.classloader :as classloader] [metabase.server.middleware.session :as mw.session] [metabase.task :as task] + [metabase.task.sync-databases :as task.sync-databases] [metabase.test :as mt] [schema.core :as s] [toucan.db :as db])) @@ -31,8 +31,10 @@ (deftest tasks-test (testing "Sync tasks should get scheduled for a newly created Database" (mt/with-temp-scheduler - (classloader/require 'metabase.task.sync-databases) - (task/init! :metabase.task.sync-databases/SyncDatabases) + ;; temporarily disable the `maybe-update-db-schedules` behavior that normally happens when the sync databases + ;; task gets initialized so we don't end up getting all of our database sync schedules randomized. + (with-redefs [task.sync-databases/maybe-update-db-schedules identity] + (task/init! ::task.sync-databases/SyncDatabases)) (mt/with-temp Database [{db-id :id}] (is (schema= {:description (s/eq (format "sync-and-analyze Database %d" db-id)) :key (s/eq (format "metabase.task.sync-and-analyze.trigger.%d" db-id)) diff --git a/test/metabase/models/setting_test.clj b/test/metabase/models/setting_test.clj index eef79ff538..5abf870d85 100644 --- a/test/metabase/models/setting_test.clj +++ b/test/metabase/models/setting_test.clj @@ -15,9 +15,6 @@ (use-fixtures :once (fixtures/initialize :db)) ;; ## TEST SETTINGS DEFINITIONS -;; TODO! These don't get loaded by `lein ring server` unless this file is touched -;; so if you run unit tests while `lein ring server` is running (i.e., no Jetty server is started) -;; these tests will fail. FIXME (defsetting test-setting-1 (deferred-tru "Test setting - this only shows up in dev (1)")) @@ -419,20 +416,21 @@ (deftest previously-encrypted-settings-test (testing "Make sure settings that were encrypted don't cause `user-facing-info` to blow up if encyrption key changed" - (encryption-test/with-secret-key "0B9cD6++AME+A7/oR7Y2xvPRHX3cHA2z7w+LbObd/9Y=" - (test-json-setting {:abc 123}) - (is (not= "{\"abc\":123}" - (actual-value-in-db :test-json-setting)))) - (testing (str "If fetching the Setting fails (e.g. because key changed) `user-facing-info` should return `nil` " - "rather than failing entirely") - (encryption-test/with-secret-key nil - (is (= {:key :test-json-setting - :value nil - :is_env_setting false - :env_name "MB_TEST_JSON_SETTING" - :description "Test setting - this only shows up in dev (4)" - :default nil} - (#'setting/user-facing-info (setting/resolve-setting :test-json-setting)))))))) + (mt/discard-setting-changes [test-json-setting] + (encryption-test/with-secret-key "0B9cD6++AME+A7/oR7Y2xvPRHX3cHA2z7w+LbObd/9Y=" + (test-json-setting {:abc 123}) + (is (not= "{\"abc\":123}" + (actual-value-in-db :test-json-setting)))) + (testing (str "If fetching the Setting fails (e.g. because key changed) `user-facing-info` should return `nil` " + "rather than failing entirely") + (encryption-test/with-secret-key nil + (is (= {:key :test-json-setting + :value nil + :is_env_setting false + :env_name "MB_TEST_JSON_SETTING" + :description "Test setting - this only shows up in dev (4)" + :default nil} + (#'setting/user-facing-info (setting/resolve-setting :test-json-setting))))))))) ;;; ----------------------------------------------- TIMESTAMP SETTINGS ----------------------------------------------- diff --git a/test/metabase/query_processor_test/breakout_test.clj b/test/metabase/query_processor_test/breakout_test.clj index 65efe7d940..c9563adf0c 100644 --- a/test/metabase/query_processor_test/breakout_test.clj +++ b/test/metabase/query_processor_test/breakout_test.clj @@ -68,11 +68,11 @@ (mt/test-drivers (mt/normal-drivers) (mt/with-column-remappings [venues.category_id (values-of categories.name)] (let [{:keys [rows cols]} (qp.test/rows-and-cols - (mt/format-rows-by [int int str] - (mt/run-mbql-query venues - {:aggregation [[:count]] - :breakout [$category_id] - :limit 5})))] + (mt/format-rows-by [int int str] + (mt/run-mbql-query venues + {:aggregation [[:count]] + :breakout [$category_id] + :limit 5})))] (is (= [(assoc (qp.test/breakout-col :venues :category_id) :remapped_to "Category ID") (qp.test/aggregate-col :count) (#'add-dim-projections/create-remapped-col "Category ID" (mt/format-name "category_id") :type/Text)] diff --git a/test/metabase/query_processor_test/expressions_test.clj b/test/metabase/query_processor_test/expressions_test.clj index fb7ce7db82..a3722aa531 100644 --- a/test/metabase/query_processor_test/expressions_test.clj +++ b/test/metabase/query_processor_test/expressions_test.clj @@ -7,10 +7,10 @@ [medley.core :as m] [metabase.driver :as driver] [metabase.query-processor :as qp] - [metabase.query-processor-test :as qp.test] [metabase.sync :as sync] [metabase.test :as mt] [metabase.test.data.one-off-dbs :as one-off-dbs] + [metabase.util :as u] [metabase.util.date-2 :as u.date])) (deftest basic-test @@ -143,14 +143,20 @@ (deftest expressions-should-include-type-test (mt/test-drivers (mt/normal-drivers-with-feature :expressions) (testing "Custom aggregation expressions should include their type" - (is (= (conj #{{:name "x" :base_type (:base_type (qp.test/aggregate-col :sum :venues :price))}} - {:name (mt/format-name "category_id") - :base_type (:base_type (qp.test/breakout-col :venues :category_id))}) - (set (map #(select-keys % [:name :base_type]) - (mt/cols - (mt/run-mbql-query venues - {:aggregation [[:aggregation-options [:sum [:* $price -1]] {:name "x"}]] - :breakout [$category_id]}))))))))) + (let [cols (mt/cols + (mt/run-mbql-query venues + {:aggregation [[:aggregation-options [:sum [:* $price -1]] {:name "x"}]] + :breakout [$category_id]}))] + (testing (format "cols = %s" (u/pprint-to-str cols)) + (is (= #{"x" (mt/format-name "category_id")} + (set (map :name cols)))) + (let [name->base-type (into {} (map (juxt :name :base_type) cols))] + (testing "x" + (is (isa? (name->base-type "x") + :type/Number))) + (testing "category_id" + (is (isa? (name->base-type (mt/format-name "category_id")) + :type/Number))))))))) ;;; +----------------------------------------------------------------------------------------------------------------+ diff --git a/test/metabase/query_processor_test/nested_queries_test.clj b/test/metabase/query_processor_test/nested_queries_test.clj index 2753fb124a..36f5863fb1 100644 --- a/test/metabase/query_processor_test/nested_queries_test.clj +++ b/test/metabase/query_processor_test/nested_queries_test.clj @@ -174,7 +174,6 @@ [& additional-clauses] (apply mbql-card-def :source-table (mt/id :venues) additional-clauses)) - (defn- query-with-source-card {:style/indent 1} ([card] diff --git a/test/metabase/related_test.clj b/test/metabase/related_test.clj index 24a5c04de8..1e5c043b58 100644 --- a/test/metabase/related_test.clj +++ b/test/metabase/related_test.clj @@ -101,10 +101,11 @@ [k (if (sequential? v) (sort (map :id v)) (:id v))]))] - ;; filter out Cards not created as part of `with-world` so these tests can be ran from the REPL. - (m/update-existing m - :similar-questions - (partial filter (set ((juxt :card-id-a :card-id-b :card-id-c) *world*)))))) + (-> m + ;; filter out Cards not created as part of `with-world` so these tests can be ran from the REPL. + (m/update-existing :similar-questions (partial filter (set ((juxt :card-id-a :card-id-b :card-id-c) *world*)))) + ;; do the same for Collections. + (m/update-existing :collections (partial filter (partial = (:collection-id *world*))))))) (deftest related-cards-test (with-world diff --git a/test/metabase/test.clj b/test/metabase/test.clj index 033d174d33..eb1e9efce0 100644 --- a/test/metabase/test.clj +++ b/test/metabase/test.clj @@ -27,16 +27,21 @@ [metabase.test.data.interface :as tx] [metabase.test.data.users :as test-users] [metabase.test.initialize :as initialize] + metabase.test.redefs [metabase.test.util :as tu] [metabase.test.util.async :as tu.async] [metabase.test.util.i18n :as i18n.tu] [metabase.test.util.log :as tu.log] [metabase.test.util.timezone :as tu.tz] [metabase.util :as u] + [pjstadig.humane-test-output :as humane-test-output] [potemkin :as p] + test-runner [toucan.db :as db] [toucan.util.test :as tt])) +(humane-test-output/activate!) + ;; Fool the linters into thinking these namespaces are used! See discussion on ;; https://github.com/clojure-emacs/refactor-nrepl/pull/270 (comment @@ -48,6 +53,7 @@ http/keep-me i18n.tu/keep-me initialize/keep-me + metabase.test.redefs/keep-me mt.tu/keep-me mw.session/keep-me qp/keep-me @@ -180,9 +186,6 @@ with-discarded-collections-perms-changes with-env-keys-renamed-by with-locale - with-log-level - with-log-messages - with-log-messages-for-level with-model-cleanup with-non-admin-groups-no-root-collection-for-namespace-perms with-non-admin-groups-no-root-collection-perms @@ -238,6 +241,7 @@ ;; TODO -- move this stuff into some other namespace and refer to it here (defn do-with-clock [clock thunk] + (test-runner/assert-test-is-not-parallel "with-clock") (testing (format "\nsystem clock = %s" (pr-str clock)) (let [clock (cond (t/clock? clock) clock diff --git a/test/metabase/test/data/env.clj b/test/metabase/test/data/env.clj index cabfe8cd5e..fbe50b69db 100644 --- a/test/metabase/test/data/env.clj +++ b/test/metabase/test/data/env.clj @@ -52,7 +52,7 @@ * by setting env var `DRIVERS` when running tests from the command line or CI. - DRIVERS=h2,mongo lein test + DRIVERS=h2,mongo clojure -X:dev:test * temporarily from the REPL, by using the `with-test-drivers-macro` diff --git a/test/metabase/test/data/impl.clj b/test/metabase/test/data/impl.clj index 3008603d1d..c3987eed28 100644 --- a/test/metabase/test/data/impl.clj +++ b/test/metabase/test/data/impl.clj @@ -3,6 +3,7 @@ (:require [clojure.string :as str] [clojure.tools.logging :as log] [clojure.tools.reader.edn :as edn] + [metabase.api.common :as api] [metabase.config :as config] [metabase.driver :as driver] [metabase.models :refer [Database Field FieldValues Table]] @@ -152,9 +153,15 @@ ;; code may run inside of some other block that sets report timezone ;; ;; require/resolve used here to avoid circular refs - ((requiring-resolve 'metabase.test.util/do-with-temporary-setting-value) - :report-timezone nil - #(create-database! driver dbdef))))))) + (letfn [(thunk [] + (binding [api/*current-user-id* nil + api/*current-user-permissions-set* nil] + (create-database! driver dbdef)))] + (if (driver/report-timezone) + ((requiring-resolve 'metabase.test.util/do-with-temporary-setting-value) + :report-timezone nil + thunk) + (thunk)))))))) (defn- get-or-create-test-data-db! "Get or create the Test Data database for `driver`, which defaults to `driver/*driver*`, or `:h2` if that is unbound." @@ -331,5 +338,7 @@ (assert db) (assert (db/exists? Database :id (u/the-id db))) db))))] - (binding [*get-db* #(get-db-for-driver (tx/driver))] + (binding [*get-db* (fn [] + (locking do-with-dataset + (get-db-for-driver (tx/driver))))] (f)))) diff --git a/test/metabase/test/redefs.clj b/test/metabase/test/redefs.clj index cbafe460f5..2aee1e7b97 100644 --- a/test/metabase/test/redefs.clj +++ b/test/metabase/test/redefs.clj @@ -3,14 +3,20 @@ [toucan.util.test :as tt])) ;; wrap `do-with-temp` so it initializes the DB before doing the other stuff it usually does -(when-not (::wrapped? (meta #'tt/do-with-temp)) - (alter-var-root #'tt/do-with-temp (fn [f] - (fn [& args] - (classloader/require 'metabase.test.initialize) - ((resolve 'metabase.test.initialize/initialize-if-needed!) :db) - (classloader/require 'metabase.test.util) ; so with-temp-defaults are loaded - (apply f args)))) - (alter-meta! #'tt/do-with-temp assoc ::wrapped? true)) -;; mark `expect-with-temp` as deprecated -- it's not needed for `deftest`-style tests -(alter-meta! #'tt/expect-with-temp assoc :deprecated true) +(defonce orig-do-with-temp tt/do-with-temp) + +(defn do-with-temp [& args] + (classloader/require 'metabase.test.initialize) + ((resolve 'metabase.test.initialize/initialize-if-needed!) :db) + ;; so with-temp-defaults are loaded + (classloader/require 'metabase.test.util) + ;; disallow with-temp inside parallel tests. + ;; + ;; TODO -- there's not really a reason that we can't use with-temp in parallel tests -- it depends on the test -- so + ;; once we're a little more comfortable with the current parallel stuff we should remove this restriction. + (classloader/require 'test-runner) + ((resolve 'test-runner/assert-test-is-not-parallel) "with-temp") + (apply orig-do-with-temp args)) + +(alter-var-root #'tt/do-with-temp (constantly do-with-temp)) diff --git a/test/metabase/test/util.clj b/test/metabase/test/util.clj index 407ef0cfc3..cfa258062e 100644 --- a/test/metabase/test/util.clj +++ b/test/metabase/test/util.clj @@ -29,6 +29,7 @@ [metabase.util.files :as u.files] [potemkin :as p] [schema.core :as s] + test-runner [toucan.db :as db] [toucan.models :as t.models] [toucan.util.test :as tt]) @@ -283,6 +284,7 @@ (defn do-with-temp-env-var-value "Impl for `with-temp-env-var-value` macro." [env-var-keyword value thunk] + (test-runner/assert-test-is-not-parallel "with-temp-env-var-value") (let [value (str value)] (testing (colorize/blue (format "\nEnv var %s = %s\n" env-var-keyword (pr-str value))) (try @@ -359,18 +361,19 @@ Prefer the macro `with-temporary-setting-values` over using this function directly." {:style/indent 2} - [setting-k value f] + [setting-k value thunk] + (test-runner/assert-test-is-not-parallel "with-temporary-setting-values") ;; plugins have to be initialized because changing `report-timezone` will call driver methods (initialize/initialize-if-needed! :db :plugins) (let [setting (#'setting/resolve-setting setting-k) env-var-value (#'setting/env-var-value setting) original-db-or-cache-value (#'setting/db-or-cache-value setting)] (if env-var-value - (do-with-temp-env-var-value setting env-var-value f) + (do-with-temp-env-var-value setting env-var-value thunk) (try (setting/set! setting-k value) (testing (colorize/blue (format "\nSetting %s = %s\n" (keyword setting-k) (pr-str value))) - (f)) + (thunk)) (finally (setting/set! setting-k original-db-or-cache-value)))))) @@ -415,6 +418,7 @@ (defn do-with-temp-vals-in-db "Implementation function for `with-temp-vals-in-db` macro. Prefer that to using this directly." [model object-or-id column->temp-value f] + (test-runner/assert-test-is-not-parallel "with-temp-vals-in-db") ;; use low-level `query` and `execute` functions here, because Toucan `select` and `update` functions tend to do ;; things like add columns like `common_name` that don't actually exist, causing subsequent update to fail (let [model (db/resolve-model model) @@ -422,10 +426,10 @@ :from [model] :where [:= :id (u/the-id object-or-id)]})] (assert original-column->value - (format "%s %d not found." (name model) (u/the-id object-or-id))) + (format "%s %d not found." (name model) (u/the-id object-or-id))) (try (db/update! model (u/the-id object-or-id) - column->temp-value) + column->temp-value) (f) (finally (db/execute! @@ -598,8 +602,27 @@ .getZone .getID))))) +(defmulti with-model-cleanup-additional-conditions + "Additional conditions that should be used to restrict which instances automatically get deleted by + `with-model-cleanup`. Conditions should be a HoneySQL `:where` clause." + {:arglists '([model])} + type) + +(defmethod with-model-cleanup-additional-conditions :default + [_] + nil) + +(defmethod with-model-cleanup-additional-conditions (type Collection) + [_] + ;; NEVER delete personal collections for the test users. + [:or + [:= :personal_owner_id nil] + [:not-in :personal_owner_id (set (map (requiring-resolve 'metabase.test.data.users/user->id) + @(requiring-resolve 'metabase.test.data.users/usernames)))]]) + (defn do-with-model-cleanup [models f] {:pre [(sequential? models) (every? t.models/model? models)]} + (test-runner/assert-test-is-not-parallel "with-model-cleanup") (initialize/initialize-if-needed! :db) (let [model->old-max-id (into {} (for [model models] [model (:max-id (db/select-one [model [:%max.id :max-id]]))]))] @@ -609,9 +632,15 @@ (finally (doseq [model models ;; might not have an old max ID if this is the first time the macro is used in this test run. - :let [old-max-id (or (get model->old-max-id model) - 0)]] - (db/simple-delete! model :id [:> old-max-id])))))) + :let [old-max-id (or (get model->old-max-id model) + 0) + max-id-condition [:> :id old-max-id] + additional-conditions (with-model-cleanup-additional-conditions model)]] + (db/execute! + {:delete-from model + :where (if (seq additional-conditions) + [:and max-id-condition additional-conditions] + max-id-condition)})))))) (defmacro with-model-cleanup "Execute `body`, then delete any *new* rows created for each model in `models`. Calls `delete!`, so if the model has @@ -702,6 +731,7 @@ readwrite-path (perms/collection-readwrite-path collection-or-id) groups-with-read-perms (db/select-field :group_id Permissions :object read-path) groups-with-readwrite-perms (db/select-field :group_id Permissions :object readwrite-path)] + (test-runner/assert-test-is-not-parallel "with-discarded-collections-perms-changes") (try (f) (finally @@ -717,6 +747,7 @@ `(do-with-discarded-collections-perms-changes ~collection-or-id (fn [] ~@body))) (defn do-with-non-admin-groups-no-collection-perms [collection f] + (test-runner/assert-test-is-not-parallel "with-non-admin-groups-no-collection-perms") (try (do-with-discarded-collections-perms-changes collection @@ -783,6 +814,7 @@ (defn call-with-locale "Sets the default locale temporarily to `locale-tag`, then invokes `f` and reverts the locale change" [locale-tag f] + (test-runner/assert-test-is-not-parallel "with-locale") (let [current-locale (Locale/getDefault)] (try (Locale/setDefault (Locale/forLanguageTag locale-tag)) diff --git a/test/metabase/test/util/log.clj b/test/metabase/test/util/log.clj index 86cea267d9..be4fb88baa 100644 --- a/test/metabase/test/util/log.clj +++ b/test/metabase/test/util/log.clj @@ -1,11 +1,14 @@ (ns metabase.test.util.log "Utils for controlling the logging that goes on when running tests." - (:require [clojure.tools.logging :as log]) + (:require [clojure.tools.logging :as log] + test-runner) (:import java.io.PrintStream [org.apache.commons.io.output NullOutputStream NullWriter] [org.apache.logging.log4j Level LogManager] [org.apache.logging.log4j.core Logger LoggerContext])) +(comment test-runner/keep-me) + (def ^:private logger->original-level (delay (let [loggers (.getLoggers ^LoggerContext (LogManager/getContext false))] @@ -77,6 +80,7 @@ :trace Level/TRACE}) (defn do-with-log-messages-for-level [x thunk] + (test-runner/assert-test-is-not-parallel "with-log-level") (let [[namespace-symb level] (if (sequential? x) x ['metabase x]) diff --git a/test/metabase/test/util/timezone.clj b/test/metabase/test/util/timezone.clj index 2c0473ac7b..044bf9c61e 100644 --- a/test/metabase/test/util/timezone.clj +++ b/test/metabase/test/util/timezone.clj @@ -1,25 +1,33 @@ (ns metabase.test.util.timezone (:require [clojure.test :as t] [metabase.driver :as driver] - [metabase.test.initialize :as initialize]) + [metabase.test.initialize :as initialize] + test-runner) (:import java.util.TimeZone)) (defn do-with-system-timezone-id [^String timezone-id thunk] - ;; only if the app DB is already set up, we need to make sure plugins are loaded and kill any connection pools that - ;; might exist - (when (initialize/initialized? :db) - (initialize/initialize-if-needed! :plugins) - (#'driver/notify-all-databases-updated)) - (let [original-time-zone (TimeZone/getDefault) - original-system-property (System/getProperty "user.timezone")] - (try - (TimeZone/setDefault (TimeZone/getTimeZone timezone-id)) - (System/setProperty "user.timezone" timezone-id) - (t/testing (format "JVM timezone set to %s" timezone-id) - (thunk)) - (finally - (TimeZone/setDefault original-time-zone) - (System/setProperty "user.timezone" original-system-property))))) + ;; skip all the property changes if the system timezone doesn't need to be changed. + (let [original-timezone (TimeZone/getDefault) + original-system-property (System/getProperty "user.timezone") + new-timezone (TimeZone/getTimeZone timezone-id)] + (if (and (= original-timezone new-timezone) + (= original-system-property timezone-id)) + (thunk) + (do + (test-runner/assert-test-is-not-parallel "with-system-timezone-id") + ;; only if the app DB is already set up, we need to make sure plugins are loaded and kill any connection pools that + ;; might exist + (when (initialize/initialized? :db) + (initialize/initialize-if-needed! :plugins) + (#'driver/notify-all-databases-updated)) + (try + (TimeZone/setDefault new-timezone) + (System/setProperty "user.timezone" timezone-id) + (t/testing (format "JVM timezone set to %s" timezone-id) + (thunk)) + (finally + (TimeZone/setDefault original-timezone) + (System/setProperty "user.timezone" original-system-property))))))) (defmacro with-system-timezone-id "Execute `body` with the system time zone temporarily changed to the time zone named by `timezone-id`. diff --git a/test/metabase/transforms/core_test.clj b/test/metabase/transforms/core_test.clj index 2df21f6a85..add3bea7ff 100644 --- a/test/metabase/transforms/core_test.clj +++ b/test/metabase/transforms/core_test.clj @@ -4,6 +4,7 @@ [metabase.domain-entities.core :as de] [metabase.domain-entities.specs :as de.specs] [metabase.models.card :as card :refer [Card]] + [metabase.models.collection :refer [Collection]] [metabase.models.table :as table :refer [Table]] [metabase.query-processor :as qp] [metabase.test :as mt] @@ -14,12 +15,16 @@ [metabase.util :as u] [toucan.db :as db])) +(use-fixtures :each (fn [thunk] + (mt/with-model-cleanup [Card Collection] + (thunk)))) + (def ^:private test-bindings (delay - (with-test-domain-entity-specs - (let [table (m/find-first (comp #{(mt/id :venues)} u/the-id) (#'t/tableset (mt/id) "PUBLIC"))] - {"Venues" {:dimensions (m/map-vals de/mbql-reference (get-in table [:domain_entity :dimensions])) - :entity table}})))) + (with-test-domain-entity-specs + (let [table (m/find-first (comp #{(mt/id :venues)} u/the-id) (#'t/tableset (mt/id) "PUBLIC"))] + {"Venues" {:dimensions (m/map-vals de/mbql-reference (get-in table [:domain_entity :dimensions])) + :entity table}})))) (deftest add-bindings-test (testing "Can we accure bindings?" diff --git a/test/metabase/util/encryption_test.clj b/test/metabase/util/encryption_test.clj index 675f365393..2d73d10add 100644 --- a/test/metabase/util/encryption_test.clj +++ b/test/metabase/util/encryption_test.clj @@ -47,7 +47,7 @@ (is (not= (encryption/encrypt secret "Hello!") (encryption/encrypt secret "Hello!"))))) -(deftest decrypt-test +(deftest decrypt-test (testing "test that we can decrypt something" (is (= "Hello!" (encryption/decrypt secret (encryption/encrypt secret "Hello!")))))) diff --git a/test/metabase/util/i18n/impl_test.clj b/test/metabase/util/i18n/impl_test.clj index cc3cc60283..0194552d2d 100644 --- a/test/metabase/util/i18n/impl_test.clj +++ b/test/metabase/util/i18n/impl_test.clj @@ -71,8 +71,7 @@ (deftest graceful-fallback-test (testing "If a resource bundle doesn't exist, we should gracefully fall back to English" (is (= "Translate me 100" - (mt/suppress-output - (impl/translate "zz" "Translate me {0}" 100)))))) + (impl/translate "zz" "Translate me {0}" 100))))) (deftest translate-test (mt/with-mock-i18n-bundles {"es" {"Your database has been added!" "¡Tu base de datos ha sido añadida!" diff --git a/test/test_runner.clj b/test/test_runner.clj new file mode 100644 index 0000000000..2c4d6a672a --- /dev/null +++ b/test/test_runner.clj @@ -0,0 +1,148 @@ +(ns test-runner + "Simple wrapper to let us use eftest with the Clojure CLI. Pass `:only` to specify where to look for tests (see dox + for [[find-tests]] for more info.)" + (:require [clojure.java.classpath :as classpath] + [clojure.java.io :as io] + [clojure.pprint :as pprint] + [clojure.string :as str] + [clojure.test :as t] + [clojure.tools.namespace.find :as ns-find] + eftest.report + eftest.report.junit + eftest.report.pretty + eftest.report.progress + eftest.runner + [environ.core :as env] + [metabase.db :as mdb] + [metabase.test.data.env :as tx.env] + metabase.test.redefs)) + +(comment metabase.test.redefs/keep-me) + +(defn- parallel? [test-var] + (let [metta (meta test-var)] + (or (:parallel metta) + (:parallel (-> metta :ns meta))))) + +(def ^:private synchronized? (complement parallel?)) + +(alter-var-root #'eftest.runner/synchronized? (constantly synchronized?)) + +(defonce orig-test-var t/test-var) + +(def ^:dynamic *parallel?* nil) + +(defn test-var [varr] + (binding [*parallel?* (parallel? varr)] + (orig-test-var varr))) + +(alter-var-root #'t/test-var (constantly test-var)) + +(defn assert-test-is-not-parallel + "Throw an exception if we are inside a `^:parallel` test." + [disallowed-message] + (when *parallel?* + (let [e (ex-info (format "%s is not allowed inside parallel tests." disallowed-message) {})] + (t/is (throw e))))) + +;; wrap `with-redefs-fn` (used by `with-redefs`) so it calls `assert-test-is-not-parallel` + +(defonce orig-with-redefs-fn with-redefs-fn) + +(defn new-with-redefs-fn [& args] + (assert-test-is-not-parallel "with-redefs") + (apply orig-with-redefs-fn args)) + +(alter-var-root #'with-redefs-fn (constantly new-with-redefs-fn)) + +(def ^:dynamic *junit-reporter-context* nil) + +(defn- reporter [] + (let [junit-reporter (eftest.report/report-to-file + eftest.report.junit/report + "target/junit/test.xml") + stdout-reporter (if (env/env :ci) + eftest.report.pretty/report + eftest.report.progress/report)] + (fn [m] + (let [start (System/currentTimeMillis)] + (binding [eftest.report/*context* *junit-reporter-context* + t/*report-counters* nil] + (junit-reporter m))) + (stdout-reporter m)))) + +(defmulti find-tests + "Find test vars in `arg`, which can be a string directory name, symbol naming a specific namespace or test, or a + collection of one or more of the above." + {:arglists '([arg])} + type) + +;; collection of one of the things below +(defmethod find-tests clojure.lang.Sequential + [coll] + (mapcat find-tests coll)) + +;; directory name +(defmethod find-tests String + [dir-name] + (find-tests (io/file dir-name))) + +;; directory +(defmethod find-tests java.io.File + [^java.io.File file] + (when (and (.isDirectory file) + (not (str/ends-with? "resources" (.getName file))) + (not (str/ends-with? "classes" (.getName file))) + (if-let [[_ driver] (re-find #"modules/drivers/([^/]+)/" (str file))] + (contains? (tx.env/test-drivers) (keyword driver)) + true)) + (println "Looking for test namespaces in directory" (str file)) + (->> (ns-find/find-namespaces-in-dir file) + (filter #(re-matches #"^metabase.*test$" (name %))) + (mapcat find-tests)))) + +;; a test namespace or individual test +(defmethod find-tests clojure.lang.Symbol + [symb] + (if (namespace symb) + ;; a actual test var e.g. `metabase.whatever-test/my-test` + [symb] + ;; a namespace e.g. `metabase.whatever-test` + (do + (require symb) + (eftest.runner/find-tests symb)))) + +;; default -- look in all dirs on the classpath +(defmethod find-tests nil + [_] + (find-tests (classpath/system-classpath))) + +(defn tests [{:keys [only]}] + (when only + (println "Running tests in" (pr-str only))) + (let [tests (with-redefs [mdb/setup-db! (fn [] + (throw (ex-info "Shouldn't be setting up the app DB as a side effect of loading test namespaces!" {})))] + (find-tests only))] + (println "Running" (count tests) "tests") + tests)) + +(defn run [tests options] + (binding [*junit-reporter-context* (atom {})] + ;; don't randomize test order for now please, thanks anyway + (with-redefs [eftest.runner/deterministic-shuffle (fn [_ tests] tests)] + (eftest.runner/run-tests + tests + (merge + {:capture-output? false + ;; parallel tests disabled for the time being -- some tests randomly fail if the data warehouse connection pool + ;; gets nuked by a different thread. Once we fix that we can re-enable parallel tests. + :multithread? false #_:vars + :report (reporter)} + options))))) + +(defn run-tests [options] + (let [summary (run (tests options) options) + fail? (pos? (+ (:error summary) (:fail summary)))] + (pprint/pprint summary) + (println (if fail? "Tests failed." "All tests passed.")) + (System/exit (if fail? 1 0)))) diff --git a/test_resources/log4j2-test.xml b/test_config/log4j2-test.xml similarity index 100% rename from test_resources/log4j2-test.xml rename to test_config/log4j2-test.xml