From 246361f33d88e5f17bfe4274b62bc6baa164ffc7 Mon Sep 17 00:00:00 2001 From: zmstone Date: Sat, 5 Oct 2024 16:07:02 +0200 Subject: [PATCH] refactor: delete brod_cli code moved to https://github.com/kafka4beam/brod-cli --- .github/workflows/build.yml | 2 - CHANGELOG.md | 3 + Makefile | 7 - README.md | 99 --- elvis.config | 2 - rebar.config | 17 +- scripts/brod | 32 - scripts/brod.escript | 31 - src/brod.erl | 10 +- src/brod_cli.erl | 1320 ---------------------------------- src/brod_cli_pipe.erl | 390 ---------- test/brod_cli_pipe_tests.erl | 162 ----- test/brod_cli_tests.erl | 253 ------- 13 files changed, 5 insertions(+), 2323 deletions(-) delete mode 100755 scripts/brod delete mode 100755 scripts/brod.escript delete mode 100644 src/brod_cli.erl delete mode 100644 src/brod_cli_pipe.erl delete mode 100644 test/brod_cli_pipe_tests.erl delete mode 100644 test/brod_cli_tests.erl diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 11954fbc..b4097e95 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -63,8 +63,6 @@ jobs: rebar3-version: ${{ env.REBAR_VERSION }} - name: Compile run: rebar3 do compile - - name: Make brod_cli script - run: rebar3 as brod_cli escriptize - name: Run tests run: | export KAFKA_VERSION=${{ matrix.kafka }} diff --git a/CHANGELOG.md b/CHANGELOG.md index efe60210..e28a5215 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,8 @@ # Changelog +- 4.3.0 + - Split brod-cli out to a separate project [kafka4beam/brod-cli](https://github.com/kafka4beam/brod-cli) + - 4.2.0 - Optimize consumer fetch latency. Introduced the `share_leader_conn` consumer configuration option (default: `false`). diff --git a/Makefile b/Makefile index 28cfca52..91e02240 100644 --- a/Makefile +++ b/Makefile @@ -35,13 +35,6 @@ hex-publish: clean ## tests that require kafka running at localhost INTEGRATION_CTS = brod_cg_commits brod_client brod_compression brod_consumer brod_producer brod_group_subscriber brod_topic_subscriber brod -## build escript and a release, and copy escript to release bin dir -brod-cli: - @rebar3 as brod_cli do compile,escriptize,release - @cp _build/brod_cli/bin/brod_cli _build/brod_cli/rel/brod/bin/ - @cp scripts/brod _build/brod_cli/rel/brod/bin/ - @cp scripts/brod.escript _build/brod_cli/rel/brod/bin/ - cover: @rebar3 cover -v diff --git a/README.md b/README.md index 8a1f439e..824bc360 100644 --- a/README.md +++ b/README.md @@ -490,102 +490,3 @@ brod:delete_topics(Hosts, [Topic], Timeout). ``` Caution the above delete_topics can fail if you do not have `delete.topic.enable` set to true in your kafka config - -## brod-cli: A command line tool to interact with Kafka - -This will build a self-contained binary with brod application - -```bash -make brod-cli -_build/brod_cli/rel/brod/bin/brod -h -``` - -Disclaimer: This script is NOT designed for use cases where fault-tolerance is a hard requirement. -As it may crash when e.g. kafka cluster is temporarily unreachable, -or (for fetch command) when the partition leader migrates to another broker in the cluster. - -### brod-cli examples (with `alias brod=_build/brod_cli/rel/brod/bin/brod`): - -#### Fetch and print metadata - -```sh -brod meta -b localhost -``` - -#### Produce a Message - -```sh -brod send -b localhost -t test-topic -p 0 -k "key" -v "value" - -``` - -#### Fetch a Message - -```sh -brod fetch -b localhost -t test-topic -p 0 --fmt 'io:format("offset=~p, ts=~p, key=~s, value=~s\n", [Offset, Ts, Key, Value])' -``` - -Bound variables to be used in `--fmt` expression: - -- `Offset`: Message offset -- `Key`: Kafka key -- `Value`: Kafka Value -- `TsType`: Timestamp type either `create` or `append` -- `Ts`: Timestamp, `-1` as no value - -#### Stream Messages to Kafka - -Send `README.md` to kafka one line per kafka message - -```sh -brod pipe -b localhost:9092 -t test-topic -p 0 -s @./README.md -``` - -#### Resolve Offset - -```sh -brod offset -b localhost:9092 -t test-topic -p 0 -``` - -#### List or Describe Groups - -```sh -# List all groups -brod groups -b localhost:9092 - -# Describe groups -brod groups -b localhost:9092 --ids group-1,group-2 -``` - -#### Display Committed Offsets - -```sh -# all topics -brod commits -b localhost:9092 --id the-group-id --describe - -# a specific topic -brod commits -b localhost:9092 --id the-group-id --describe --topic topic-name -``` - -#### Commit Offsets - -NOTE: This feature is designed for force overwriting commits, not for regular use of offset commit. - -```sh -# Commit 'latest' offsets of all partitions with 2 days retention -brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 2d - -# Commit offset=100 for partition 0 and 200 for partition 1 -brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets "0:100,1:200" - -# Use --retention 0 to delete commits (may linger in kafka before cleaner does its job) -brod commits -b localhost:9092 --id the-group-id --topic topic-name --offsets latest --retention 0 - -# Try join an active consumer group using 'range' protocol and steal one partition assignment then commit offset=10000 -brod commits -b localhost:9092 -i the-group-id -t topic-name -o "0:10000" --protocol range -``` - -### TODOs - -- Support scram-sasl in brod-cli -- Transactional produce APIs diff --git a/elvis.config b/elvis.config index c15b9788..03f535dd 100644 --- a/elvis.config +++ b/elvis.config @@ -30,8 +30,6 @@ #{ level => 3, ignore => [ brod_group_coordinator , brod_utils - , brod_cli - , brod_cli_pipe ] }} , {elvis_style, god_modules, diff --git a/rebar.config b/rebar.config index 9b828b7d..fb86e6fb 100644 --- a/rebar.config +++ b/rebar.config @@ -1,26 +1,11 @@ {deps, [{kafka_protocol, "4.1.9"}]}. {project_plugins, [{rebar3_lint, "~> 3.2.5"}]}. -{edoc_opts, [{preprocess, true}, {macros, [{build_brod_cli, true}]}]}. +{edoc_opts, [{preprocess, true}]}. {erl_opts, [warnings_as_errors, warn_unused_vars,warn_shadow_vars,warn_obsolete_guard,debug_info]}. {xref_checks, [undefined_function_calls, undefined_functions, locals_not_used, deprecated_function_calls, deprecated_functions]}. {profiles, [ - {brod_cli, [ - {deps, [ {docopt, {git, "https://github.com/zmstone/docopt-erl.git", {tag, "0.1.3"}}} - , {jsone, "1.7.0"} - , {snappyer, "1.2.9"} - ]}, - {erl_opts, [warnings_as_errors, {d, build_brod_cli}]}, - {escript_name, brod_cli}, - {relx, [{release, {brod, "i"}, % release the interactive shell as brod-i - [brod, jsone, docopt]}, - {include_erts, true}, - {overlay, [{copy, "scripts/brod", "bin/"}, - {copy, "{{lib_dirs}}/crc32cer/priv/crc32cer*.so", "bin/"}, - {copy, "{{lib_dirs}}/snappyer/priv/snappyer.so", "bin/"} - ]} - ]}]}, {test, [ {deps, [ {docopt, {git, "https://github.com/zmstone/docopt-erl.git", {tag, "0.1.3"}}} , {hut, "1.3.0"} diff --git a/scripts/brod b/scripts/brod deleted file mode 100755 index 56361e54..00000000 --- a/scripts/brod +++ /dev/null @@ -1,32 +0,0 @@ -#!/bin/sh - -set -eu - -THIS="$0" -CUR_DIR="$(pwd)" - -# resolve the dir (maybe sym-linked) in which THIS script resides -cd "$(dirname "$THIS")" -TARGET_FILE="$(basename "$THIS")" -# use a while loop because readlink -f does not always work in all systems -while [ -L "$TARGET_FILE" ]; do - TARGET_FILE="$(readlink "$TARGET_FILE")" - cd "$(dirname "$TARGET_FILE")" - TARGET_FILE="$(basename "$TARGET_FILE")" -done -BINDIR="$(pwd -P)" - -# go back to the original PWD -cd "$CUR_DIR" - -if [ -x "$BINDIR/brod-i" ]; then - # this is a release, try to use erts in release - # brod-i find erts bin dir for us - exec "$BINDIR/brod-i" escript bin/brod.escript "$@" -else - # this is not a release, so we assume: - # 1. the user has erts in PATH - # 2. nif binary files are located in the bin dir - export NIF_BIN_DIR=$BINDIR - exec "$BINDIR/brod_cli" "$@" -fi diff --git a/scripts/brod.escript b/scripts/brod.escript deleted file mode 100755 index b5075037..00000000 --- a/scripts/brod.escript +++ /dev/null @@ -1,31 +0,0 @@ -#!/usr/bin/env escript - -%% this script forwards the call to brod_cli module -%% we avoid using the escriptised file due to the need of loading nif binaries - -main(Args) -> - ok = add_libs_dir(), - brod_cli:main(Args). - -add_libs_dir() -> - %% RELEASE_ROOT_DIR is set by bord-i boot script - [_ | _] = RootDir = os:getenv("RELEASE_ROOT_DIR"), - RelFile = filename:join([RootDir, "releases", "RELEASES"]), - case file:consult(RelFile) of - {ok, [Releases]} -> - Release = lists:keyfind("i", 3, Releases), - {release, _Name, _AppVsn, _ErtsVsn, Libs, _State} = Release, - lists:foreach( - fun({Name, Vsn, _}) -> - add_lib_dir(RootDir, Name, Vsn) - end, Libs); - {error, Reason} -> - error({failed_to_read_RELEASES_file, RelFile, Reason}) - end. - -add_lib_dir(RootDir, Name, Vsn) -> - LibDir = filename:join([RootDir, lib, atom_to_list(Name) ++ "-" ++ Vsn, ebin]), - case code:add_patha(LibDir) of - true -> ok; - {error, _} -> error(LibDir) - end. diff --git a/src/brod.erl b/src/brod.erl index d7dbedfb..859a2657 100644 --- a/src/brod.erl +++ b/src/brod.erl @@ -100,7 +100,7 @@ , delete_topics/4 ]). -%% APIs for quick metadata or message inspection and brod_cli +%% APIs for quick metadata or message inspection -export([ get_metadata/1 , get_metadata/2 , get_metadata/3 @@ -128,10 +128,6 @@ , fetch/8 ]). --ifdef(build_brod_cli). --export([main/1]). --endif. - -export_type([ batch_input/0 , bootstrap/0 , call_ref/0 @@ -1365,10 +1361,6 @@ fetch_committed_offsets(BootstrapEndpoints, ConnCfg, GroupId) -> fetch_committed_offsets(Client, GroupId) -> brod_utils:fetch_committed_offsets(Client, GroupId, []). --ifdef(build_brod_cli). -main(X) -> brod_cli:main(X). --endif. - %% @doc Start a new transaction, `TxId' will be the id of the transaction %% @equiv brod_transaction:start_link/3 -spec transaction(client(), transactional_id(), transaction_config()) -> {ok, transaction()}. diff --git a/src/brod_cli.erl b/src/brod_cli.erl deleted file mode 100644 index e3f3cef8..00000000 --- a/src/brod_cli.erl +++ /dev/null @@ -1,1320 +0,0 @@ -%%% -%%% Copyright (c) 2017-2021 Klarna Bank AB (publ) -%%% -%%% Licensed under the Apache License, Version 2.0 (the "License"); -%%% you may not use this file except in compliance with the License. -%%% You may obtain a copy of the License at -%%% -%%% http://www.apache.org/licenses/LICENSE-2.0 -%%% -%%% Unless required by applicable law or agreed to in writing, software -%%% distributed under the License is distributed on an "AS IS" BASIS, -%%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -%%% See the License for the specific language governing permissions and -%%% limitations under the License. -%%% - -%% @private --module(brod_cli). - --ifdef(build_brod_cli). - --export([main/1, main/2]). - --include("brod_int.hrl"). - --define(CLIENT, brod_cli_client). - -%% 'halt' is for escript, stop the vm immediately -%% 'exit' is for testing, we want eunit or ct to be able to capture --define(STOP(How), - begin - try - brod:stop_client(?CLIENT) - catch - exit : {noproc, _} -> - ok - end, - _ = brod:stop(), - case How of - 'halt' -> erlang:halt(?LINE); - 'exit' -> erlang:exit(?LINE) - end - end). - --define(MAIN_DOC, "usage: - brod -h|--help - brod -v|--version - brod [options] [-h|--help] [--verbose|--debug] - -commands: - meta: Inspect topic metadata - offset: Inspect offsets - fetch: Fetch messages - send: Produce messages - pipe: Pipe file or stdin as messages to kafka - groups: List/describe consumer group - commits: List/describe committed offsets - or force overwrite existing commits -"). - -%% NOTE: bad indentation at the first line is intended --define(COMMAND_COMMON_OPTIONS, -" --ssl Use TLS, validate server using trusted CAs - --ssl-versions= Specify SSL versions. Comma separated versions, - e.g. 1.3,1.2 - --cacertfile= Use TLS, validate server using the given certificate - --certfile= Client certificate in case client authentication - is enabled in brokers - --keyfile= Client private key in case client authentication - is enabled in brokers - --sasl-plain= Tell brod to use username/password stored in the - given file, the file should have username and - password in two lines. - --scram256= Like sasl-plain option, but to use scram-sha-256 - --scram512= Like sasl-plain option, but to use scram-sha-512 - --ebin-paths= Comma separated directory names for extra beams, - This is to support user compiled message formatters - --no-api-vsn-query Do not query api version (for kafka 0.9 or earlier) - Or set KAFKA_VERSION environment variable to 0.9 for - the same effect -" -). - --define(META_CMD, "meta"). --define(META_DOC, "usage: - brod meta [options] - -options: - -b,--brokers= Comma separated host:port pairs - [default: localhost:9092] - -t,--topic= Topic name [default: *] - -T,--text Print metadata as aligned texts (default) - -J,--json Print metadata as JSON object - -L,--list List topics, no partition details, - Applicable only for --text option - -U,--under-replicated Display only under-replicated partitions -" -?COMMAND_COMMON_OPTIONS -"Text output schema (out of sync replicas are marked with *): -brokers : - : -topics : - : [[ERROR] []] - : (replicas[*]...) [] -" -). - --define(OFFSET_CMD, "offset"). --define(OFFSET_DOC, "usage: - brod offset [options] - -options: - -b,--brokers= Comma separated host:port pairs - [default: localhost:9092] - -t,--topic= Topic name - -p,--partition= Partition number - [default: all] - -T,--time=