From 7c0169c59214394480e264f74a8e32eee72526bc Mon Sep 17 00:00:00 2001 From: Gabi Davar Date: Sat, 1 Jul 2023 15:12:39 +0300 Subject: [PATCH] numerous updates: * add builah, cgroup-tools, helm, procs, ripgrep, shadow, skopeo and, stress-ng. * use native alpine packages for aws-cli, k9s, kubectl, krew, ctop, httpie and ipython * add bash completions where available. * call neofetch on startup. * README cleanup podman and buildah are broken. --- .bashrc | 4 -- Dockerfile | 135 +++++++++++++++++++++++++++++++++++------------------ README.md | 108 +++++++++++++++++++++++------------------- profile | 1 + 4 files changed, 149 insertions(+), 99 deletions(-) diff --git a/.bashrc b/.bashrc index 5880e57..fd389ae 100644 --- a/.bashrc +++ b/.bashrc @@ -60,10 +60,6 @@ alias where='which -a' [ -f ~/.kubectl_aliases ] && source ~/.kubectl_aliases -source <(kubectl completion bash) -source <(stern --completion bash) -#source <(helm completion bash) - complete -F __start_kubectl k export PATH="${KREW_ROOT:-$HOME/.krew}/bin:$PATH" diff --git a/Dockerfile b/Dockerfile index 5b6108f..59b1e2e 100755 --- a/Dockerfile +++ b/Dockerfile @@ -1,21 +1,4 @@ ARG ALPINE_VERSION=3.18 -FROM python:3.11.3-alpine${ALPINE_VERSION} as builder - -ARG AWS_CLI_VERSION=2.11.23 -RUN apk add --no-cache git unzip groff build-base libffi-dev cmake -RUN git clone --single-branch --depth 1 -b ${AWS_CLI_VERSION} https://github.com/aws/aws-cli.git - -WORKDIR aws-cli -RUN python -m venv venv -RUN . venv/bin/activate -RUN scripts/installers/make-exe -RUN unzip -q dist/awscli-exe.zip -RUN aws/install --bin-dir /aws-cli-bin -RUN /aws-cli-bin/aws --version - -# reduce image size: remove autocomplete and examples -RUN rm -rf /usr/local/aws-cli/v2/current/dist/awscli/examples -RUN find /usr/local/aws-cli/v2/current/dist/awscli/botocore/data -name examples-1.json -delete # build the final image FROM alpine:${ALPINE_VERSION} @@ -25,10 +8,21 @@ RUN set -ex; \ echo "https://dl-cdn.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories; \ echo "https://dl-cdn.alpinelinux.org/alpine/edge/community" >> /etc/apk/repositories; \ apk -U upgrade; \ + apk add -u --no-cache !pyc\ + btrfs-progs \ + e2fsprogs \ + e2fsprogs-extra \ + shadow-uidmap \ + xfsprogs \ + xz \ + pigz; \ apk add -u --no-cache !pyc\ 7zip \ apache2-utils \ atop \ + aws-cli \ + aws-cli-bash-completion \ + aws-cli-doc \ btop \ bash \ bash-completion \ @@ -38,39 +32,55 @@ RUN set -ex; \ bpftrace \ bpftrace-tools \ bridge-utils \ + buildah \ busybox-extras \ ca-certificates \ + cgroup-tools \ conntrack-tools \ coreutils \ curl \ + ctop \ dhcping \ diffutils \ drill \ ethtool \ + ethtool-bash-completion \ fatrace \ file \ findutils \ fio \ fping \ + fuse-overlayfs \ git \ git-bash-completion \ grep \ + helm \ + helm-bash-completion \ hping3 \ htop \ + httpie \ iftop \ inotify-tools \ ioping \ iperf \ iperf3 \ iproute2 \ + iproute2-bash-completion \ ipset \ iptables \ ip6tables \ iptraf-ng \ iputils \ ipvsadm \ + ipython \ jo \ + jo-bash-completion \ jq \ + k9s \ + k9s-bash-completion \ + kubectl \ + kubectl-bash-completion \ + kubectl-krew \ less \ liboping \ ltrace \ @@ -88,19 +98,34 @@ RUN set -ex; \ openssh \ openssl \ perf \ + perf-bash-completion \ podman \ + podman-bash-completion \ + podman-docker \ + podman-tui \ procps \ + procs \ + procs-bash-completion \ pstree \ py3-pip \ + py3-pip-bash-completion \ py3-cryptography \ py3-yaml \ py3-ruamel.yaml \ py3-virtualenv \ python3 \ + ripgrep \ + ripgrep-bash-completion \ scapy \ + shadow \ + skopeo \ + skopeo-bash-completion \ socat \ stern \ + stern-bash-completion \ strace \ + stress-ng \ + stress-ng-bash-completion \ su-exec \ sudo \ swaks \ @@ -119,50 +144,27 @@ RUN set -ex; \ websocat \ wget \ wireguard-tools \ - yq; \ + wireguard-tools-bash-completion \ + yq \ + yq-bash-completion; \ rm -f /var/cache/apk/* SHELL ["/bin/bash", "-e", "-o", "pipefail", "-c"] -COPY --from=builder /usr/local/aws-cli/ /usr/local/aws-cli/ -COPY --from=builder /aws-cli-bin/ /usr/local/bin/ - # let's install httpie, httpx, ipython and jc -RUN pip3 install --upgrade --no-cache-dir httpie httpx[cli,brotli,http2,socks] ipython jc - -# Installing ctop - top-like container monitor -ARG CTOP_VERSION=0.7.7 -RUN \ - wget -q https://github.com/bcicen/ctop/releases/download/v${CTOP_VERSION}/ctop-${CTOP_VERSION}-linux-amd64 -O /usr/local/bin/ctop; \ - chmod +x /usr/local/bin/ctop +RUN pip3 install --upgrade --no-cache-dir httpx[cli,brotli,http2,socks] jc # Installing calicoctl -ARG CALICOCTL_VERSION=v3.26.0 +ARG CALICOCTL_VERSION=v3.26.1 RUN \ wget -q https://github.com/projectcalico/calico/releases/download/${CALICOCTL_VERSION}/calicoctl-linux-amd64 -O /usr/local/bin/calicoctl; \ chmod +x /usr/local/bin/calicoctl -# Installing kubectl -ARG KUBECTL_VERSION=v1.27.2 -RUN \ - wget -q https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl -O /usr/local/bin/kubectl; \ - chmod +x /usr/local/bin/kubectl - -# krew and plugins +# krew plugins RUN \ - curl --fail --silent -L --location https://github.com/kubernetes-sigs/krew/releases/latest/download/krew-linux_amd64.tar.gz | tar xz -C /tmp; \ - /tmp/krew-linux_amd64 install krew; \ PATH="$HOME/.krew/bin:$PATH"; \ kubectl krew install cert-manager ctx explore get-all oidc-login node-shell ns relay resource-capacity sniff topology -# Installing K9s -ARG K9S_VERSION=v0.27.4 -RUN \ - curl --fail --silent -L https://github.com/derailed/k9s/releases/download/${K9S_VERSION}/k9s_Linux_amd64.tar.gz | \ - tar zx k9s; \ - chmod +x k9s; \ - mv k9s /usr/local/bin - # Installing grpcurl ARG GRPCURL_VERSION=1.8.7 RUN \ @@ -183,6 +185,47 @@ RUN \ RUN \ curl --fail --silent --location https://rawgit.com/ahmetb/kubectl-alias/master/.kubectl_aliases -o "$HOME/.kubectl_aliases" +# FIXME: figure out how to run podman in alpine container. +#RUN modprobe tun; \ +# echo tun >>/etc/modules; + + # setting up podman - https://github.com/containers/podman/blob/main/contrib/podmanimage/stable/Containerfile +RUN useradd podman; \ +echo -e "podman:1:999\npodman:1001:64535" > /etc/subuid; \ +echo -e "podman:1:999\npodman:1001:64535" > /etc/subgid; + +ARG _REPO_URL="https://raw.githubusercontent.com/containers/podman/main/contrib/podmanimage/stable" +ADD $_REPO_URL/containers.conf /etc/containers/containers.conf +ADD $_REPO_URL/podman-containers.conf /home/podman/.config/containers/containers.conf + +RUN mkdir -p /home/podman/.local/share/containers && \ + chown podman:podman -R /home/podman && \ + chmod 644 /etc/containers/containers.conf + +# Copy & modify the defaults to provide reference if runtime changes needed. +# Changes here are required for running with fuse-overlay storage inside container. +RUN sed -e 's|^#mount_program|mount_program|g' \ + -e '/additionalimage.*/a "/var/lib/shared",' \ + -e 's|^mountopt[[:space:]]*=.*$|mountopt = "nodev,fsync=0"|g' \ + /usr/share/containers/storage.conf \ + > /etc/containers/storage.conf + +# Note VOLUME options must always happen after the chown call above +# RUN commands can not modify existing volumes +#VOLUME /var/lib/containers +#VOLUME /home/podman/.local/share/containers + +RUN mkdir -p /var/lib/shared/overlay-images \ + /var/lib/shared/overlay-layers \ + /var/lib/shared/vfs-images \ + /var/lib/shared/vfs-layers && \ + touch /var/lib/shared/overlay-images/images.lock && \ + touch /var/lib/shared/overlay-layers/layers.lock && \ + touch /var/lib/shared/vfs-images/images.lock && \ + touch /var/lib/shared/vfs-layers/layers.lock + +ENV _CONTAINERS_USERNS_CONFIGURED="" + # Settings ADD motd /etc/motd ADD .bashrc /root/.bashrc diff --git a/README.md b/README.md index c198aec..c26079e 100755 --- a/README.md +++ b/README.md @@ -42,8 +42,9 @@ namespace like this : * **Host's Network Namespace:** If you think the networking issue is on the host itself, you can launch `netshoot` with that host's network namespace. This is how: - -`$ docker run -it --net host ghcr.io/mindw/netshoot:master` +```shell +$ docker run -it --net host ghcr.io/mindw/netshoot:master +``` * **Network's Network Namespace:** If you want to troubleshoot a Docker network, you can enter the network's namespace using `nsenter`. This is explained in the `nsenter` @@ -51,35 +52,28 @@ can enter the network's namespace using `nsenter`. This is explained in the `nse # Kubernetes * if you want to debug using an [ephemeral container](https://kubernetes.io/docs/tasks/debug/debug-application/debug-running-pod/#ephemeral-container-example) in an existing pod: - - `$ kubectl debug mypod -it --image=nicolaka/netshoot` +```shell +$ kubectl debug mypod -it --image=ghcr.io/mindw/netshoot:master +``` * if you want to spin up a throw away pod for debugging. - -`$ kubectl run tmp-shell --image-pull-policy=Always --rm -it --image ghcr.io/mindw/netshoot:master` +```shell +$ kubectl run tmp-shell --image-pull-policy=Always --rm -it --image ghcr.io/mindw/netshoot:master +``` * And if you want to spin up a container on the host's network namespace. - -`$ kubectl run tmp-shell --image-pull-policy=Always --rm -it --overrides='{"spec": {"hostNetwork": true, "enableServiceLinks": false}}' --image ghcr.io/mindw/netshoot:master` +```shell +$ kubectl run tmp-shell --image-pull-policy=Always --rm -it \ + --overrides='{"spec": {"hostNetwork": true, "enableServiceLinks": false}}' --image ghcr.io/mindw/netshoot:master +``` * Privileged and host networking. - ``` $ kubectl run tmp-shell --image ghcr.io/mindw/netshoot:master --rm -ti\ --overrides='{"spec": {"hostNetwork": true, "enableServiceLinks": false, "containers": [{"name": "tmp-shell", "image": "ghcr.io/mindw/netshoot:master", "imagePullPolicy": "Always", "tty": true, "stdin": true, "securityContext": {"privileged": true} }]}}' ``` -## Installed Kubernetes tools - -- calicoctl: Used to manage Calico policies and configuration, as well as view detailed cluster status -- k9s: A terminal based UI to interact with your Kubernetes clusters -- krew: A plugin manager for kubectl command-line tool -- kubectl: The Kubernetes command-line tool, kubectl, allows you to run commands against Kubernetes clusters -- kubectl-alias: Programmatically generated handy kubectl aliases. -- stern: Multi pod and container log tailing for Kubernetes - # Network Problems - Many network issues could result in application performance degradation. Some of those issues could be related to the underlying networking infrastructure(underlay). Others could be related to misconfiguration at the host or Docker level. Let's take a look at @@ -100,21 +94,13 @@ recommended by this diagram. The following packages are included in `netshoot`. We'll go over some with some sample use-cases. -- apache2-utils: Apache utility programs for webservers - atop: Resource-specific view of processes - btop: Resource monitor - bcc: A toolkit for creating efficient kernel tracing and manipulation programs - bcc-tools: Command line tools for BPF Compiler Collection (BCC) -- bind-tools: The ISC DNS tools -- bird: BIRD Internet Routing Daemon - bpftrace: High-level tracing language for Linux eBPF - bpftrace-tools: High-level tracing language for Linux eBPF (tools) -- bridge-utils: Tools for configuring the Linux kernel 802.1d Ethernet Bridge -- conntrack-tools: Connection tracking userspace tools - ctop: Top-like interface for container metrics -- dhcping: dhcp daemon ping program -- drill: DNS(SEC) query tool ala dig -- ethtool: Utility for controlling network drivers and hardware - fatrace: Report system wide file access events - fio: Flexible I/O Tester - fortio: A load testing library, command line tool, advanced echo server and web UI. @@ -126,50 +112,72 @@ some with some sample use-cases. - inotify-tools: C library and CLI tools providing a simple interface to inotify - iperf: A tool to measure IP bandwidth using UDP or TCP - iperf3: A tool to measure IP bandwidth using UDP or TCP -- iproute2: IP Routing Utilities -- ipset: Manage Linux IP sets -- iptables: Linux kernel firewall, NAT and packet mangling tools -- ip6tables: Linux kernel firewall, NAT and packet mangling tools - iptraf-ng: An IP Network Monitoring tool -- iputils: IP Configuration Utilities (and Ping) -- ipvsadm: The IP Virtual Server administration utility - ltrace: Tracks runtime library calls in dynamically linked programs - lttng-tools: LTTng 2.0 control and utility programs - mtr: Full screen ncurses traceroute tool -- net-snmp-tools: Network management utilities using SNMP +- neofetch: A CLI system information tool written in Bash that supports displaying images - netcat-openbsd: The TCP/IP swiss army knife. OpenBSD variant. -- nftables: Netfilter tables userspace tools - nmap: network exploration tool and security/port scanner - nmap-nping: network exploration tool and security/port scanner (nping tool) -- open-lldp: Open Source implementation of the IEEE standard 802.1AB Link Layer Discovery Protocol (LLDP) - perf: Linux kernel performance auditing tool -- podman: Simple management tool for pods, containers and images +- procs: Modern replacement for ps written in Rust - pstree: A small tool to list processes as a tree - scapy: Powerful interactive packet manipulation - socat: Multipurpose relay for binary protocols +- stress-ng: Stress tests a computer system in various selectable ways - strace: Diagnostic, debugging and instructional userspace tracer -- swaks: Swiss Army Knife for SMTP - sysstat: Performance monitoring tools -- tcpdump: A tool for network monitoring and data acquisition -- tcptraceroute: Display route path using TCP probes - tiptop: Tiptop: Hardware Performance Counters for the Masses -- termshark: terminal UI for tshark, inspired by Wireshark -- tshark: Network protocol analyzer (Console version) -- util-linux: Random collection of Linux utilities -- websocat: Netcat, curl and socat for WebSockets -- wireguard-tools: Next generation secure network tunnel: userspace tools # Included Networking tools -- aws-cli2: Universal Command Line Interface for Amazon Web Services -- curl: URL retrieval utility and library +- apache2-utils: Apache utility programs for webservers +- bind-tools: The ISC DNS tools +- bird: BIRD Internet Routing Daemon +- bridge-utils: Tools for configuring the Linux kernel 802.1d Ethernet Bridge - ca-certificates: Common CA certificates PEM files from Mozilla +- conntrack-tools: Connection tracking userspace tools +- curl: URL retrieval utility and library +- dhcping: dhcp daemon ping program +- drill: DNS(SEC) query tool ala dig +- ethtool: Utility for controlling network drivers and hardware - grpcurl: Like cURL, but for gRPC - Command-line tool for interacting with gRPC servers. - httpie: human-friendly CLI HTTP client for the API era - httpx: a fully featured HTTP client for Python 3, which provides sync and async APIs, and support for both HTTP/1.1 and HTTP/2. +- iptables: Linux kernel firewall, NAT and packet mangling tools +- ip6tables: Linux kernel firewall, NAT and packet mangling tools +- iproute2: IP Routing Utilities +- ipset: Manage Linux IP sets +- iputils: IP Configuration Utilities (and Ping) +- ipvsadm: The IP Virtual Server administration utility +- net-snmp-tools: Network management utilities using SNMP +- ngrep: A grep-like utility that allows you to search for network packets on an interface +- nftables: Netfilter tables userspace tools +- open-lldp: Open Source implementation of the IEEE standard 802.1AB Link Layer Discovery Protocol (LLDP) - openssh: Port of OpenBSD's free SSH release - openssl: Toolkit for Transport Layer Security (TLS) +- swaks: Swiss Army Knife for SMTP +- tcpdump: A tool for network monitoring and data acquisition +- tcptraceroute: Display route path using TCP probes +- termshark: terminal UI for tshark, inspired by Wireshark +- tshark: Network protocol analyzer (Console version) +- websocat: Netcat, curl and socat for WebSockets +- wireguard-tools: Next generation secure network tunnel: userspace tools - wget: Network utility to retrieve files from the Web +# Included Kubernetes, cloud and container tools +- aws-cli2: Universal Command Line Interface for Amazon Web Services +- buildah: tool that facilitates building OCI container images +- calicoctl: Used to manage Calico policies and configuration, as well as view detailed cluster status +- helm: The Kubernetes Package Manager. +- k9s: A terminal based UI to interact with your Kubernetes clusters +- krew: A plugin manager for kubectl command-line tool +- kubectl: The Kubernetes command-line tool, kubectl, allows you to run commands against Kubernetes clusters +- kubectl-alias: Programmatically generated handy kubectl aliases. +- podman: Simple management tool for pods, containers and images +- skopeo: Work with remote images registries - retrieving information, images, signing content +- stern: Multi pod and container log tailing for Kubernetes + # Included text manipulations tools - diffutils: Show differences among files - grep: Searches input files for lines containing a match to a specified pattern @@ -177,7 +185,7 @@ some with some sample use-cases. - jo: JSON output from a shell - jq: A lightweight and flexible command-line JSON processor - less: File pager -- ngrep: A grep-like utility that allows you to search for network packets on an interface +- ripgrep: Combines the usability of The Silver Searcher with the raw speed of grep - yq: Portable command-line YAML processor written in Go # Included mostly well known system tools @@ -185,6 +193,7 @@ some with some sample use-cases. - bash: The GNU Bourne Again shell - bash-completion: bash-completion - busybox-extras: Additional binaries of Busybox +- cgroup-tools: cgroup library (tools) - coreutils: The basic file, shell and text manipulation utilities - file: File type identification utility - findutils: GNU utilities for finding files @@ -192,16 +201,17 @@ some with some sample use-cases. - git-bash-completion: Bash completions for git - ipython: A rich toolkit to help you make the most of using Python interactively - moreutils: A growing collection of the unix tools that nobody thought to write thirty years ago -- neofetch: A CLI system information tool written in Bash that supports displaying images - py3-pip: Tool for installing and managing Python packages - py3-virtualenv: Virtual Python3 Environment builder - python3: A high-level scripting language +- shadow: PAM-using login and passwd utilities (usermod, useradd, ...) - su-exec: switch user and group id, setgroups and exec - sudo: Give certain users the ability to run some commands as root - tar: Utility used to store, backup, and transport files - tini: A tiny but valid init for containers - tree: A recursive directory indented listing of files - unzip: Extract PKZIP-compatible .zip files +- util-linux: Random collection of Linux utilities - vim: Improved vi-style text editor # Sample Use-cases diff --git a/profile b/profile index d3313b2..27da50f 100644 --- a/profile +++ b/profile @@ -40,3 +40,4 @@ function title { prompt +neofetch