From 01e73d32e99fd493ef8b1547be1f6b70471089fb Mon Sep 17 00:00:00 2001 From: duanhongyi Date: Fri, 16 Nov 2018 20:49:53 +0000 Subject: [PATCH] feat(fluentd): use official fluent/fluentd:1.3 docker image --- .gitignore | 2 +- Makefile | 2 +- .../templates/logger-fluentd-clusterrole.yaml | 2 +- rootfs/Dockerfile | 66 ++-- rootfs/entrypoint.sh | 3 + .../fluentd/deis-output/.ruby-version | 0 rootfs/{opt => }/fluentd/deis-output/Gemfile | 0 .../fluentd/deis-output/Gemfile.lock | 0 rootfs/{opt => }/fluentd/deis-output/LICENSE | 0 .../{opt => }/fluentd/deis-output/README.md | 0 rootfs/{opt => }/fluentd/deis-output/Rakefile | 0 .../fluent-plugin-deis_output.gemspec | 0 .../deis-output/lib/fluent/mixin/deis.rb | 0 .../deis-output/lib/fluent/plugin/out_deis.rb | 0 .../fluentd/deis-output/test/test_out_deis.rb | 0 .../fluentd.conf => fluentd/etc/fluent.conf} | 0 rootfs/{opt => }/fluentd/sbin/boot | 19 +- .../fluentd/sbin/filters/custom_filters | 0 rootfs/fluentd/sbin/filters/filters | 3 + .../{opt => }/fluentd/sbin/filters/kubernetes | 0 rootfs/{opt => }/fluentd/sbin/plugins | 0 rootfs/{opt => }/fluentd/sbin/sources | 0 .../fluentd/sbin/stores/custom_stores | 0 rootfs/{opt => }/fluentd/sbin/stores/deis | 0 .../fluentd/sbin/stores/elastic_search | 0 rootfs/{opt => }/fluentd/sbin/stores/gelf | 0 rootfs/fluentd/sbin/stores/stores | 8 + .../{opt => }/fluentd/sbin/stores/sumologic | 0 rootfs/{opt => }/fluentd/sbin/stores/syslog | 0 rootfs/opt/fluentd/sbin/filters/filters | 3 - rootfs/opt/fluentd/sbin/stores/stores | 8 - .../lib/fluent/plugin/out_elasticsearch.rb | 370 ------------------ .../lib/fluent/plugin/out_remote_syslog.rb | 57 --- 33 files changed, 49 insertions(+), 494 deletions(-) create mode 100644 rootfs/entrypoint.sh rename rootfs/{opt => }/fluentd/deis-output/.ruby-version (100%) rename rootfs/{opt => }/fluentd/deis-output/Gemfile (100%) rename rootfs/{opt => }/fluentd/deis-output/Gemfile.lock (100%) rename rootfs/{opt => }/fluentd/deis-output/LICENSE (100%) rename rootfs/{opt => }/fluentd/deis-output/README.md (100%) rename rootfs/{opt => }/fluentd/deis-output/Rakefile (100%) rename rootfs/{opt => }/fluentd/deis-output/fluent-plugin-deis_output.gemspec (100%) rename rootfs/{opt => }/fluentd/deis-output/lib/fluent/mixin/deis.rb (100%) rename rootfs/{opt => }/fluentd/deis-output/lib/fluent/plugin/out_deis.rb (100%) rename rootfs/{opt => }/fluentd/deis-output/test/test_out_deis.rb (100%) rename rootfs/{opt/fluentd/conf/fluentd.conf => fluentd/etc/fluent.conf} (100%) rename rootfs/{opt => }/fluentd/sbin/boot (74%) rename rootfs/{opt => }/fluentd/sbin/filters/custom_filters (100%) create mode 100755 rootfs/fluentd/sbin/filters/filters rename rootfs/{opt => }/fluentd/sbin/filters/kubernetes (100%) rename rootfs/{opt => }/fluentd/sbin/plugins (100%) rename rootfs/{opt => }/fluentd/sbin/sources (100%) rename rootfs/{opt => }/fluentd/sbin/stores/custom_stores (100%) rename rootfs/{opt => }/fluentd/sbin/stores/deis (100%) rename rootfs/{opt => }/fluentd/sbin/stores/elastic_search (100%) rename rootfs/{opt => }/fluentd/sbin/stores/gelf (100%) create mode 100755 rootfs/fluentd/sbin/stores/stores rename rootfs/{opt => }/fluentd/sbin/stores/sumologic (100%) rename rootfs/{opt => }/fluentd/sbin/stores/syslog (100%) delete mode 100755 rootfs/opt/fluentd/sbin/filters/filters delete mode 100755 rootfs/opt/fluentd/sbin/stores/stores delete mode 100644 rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-1.7.0/lib/fluent/plugin/out_elasticsearch.rb delete mode 100644 rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-remote_syslog-0.3.2/lib/fluent/plugin/out_remote_syslog.rb diff --git a/.gitignore b/.gitignore index b37db00..dde229a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ manifests/*.tmp.yaml -rootfs/opt/fluentd/deis-output/pkg +rootfs/fluentd/deis-output/pkg rootfs/*.gem diff --git a/Makefile b/Makefile index 697ce32..15adf94 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ docker-build: docker tag ${IMAGE} ${MUTABLE_IMAGE} test: docker-build - docker run ${IMAGE} /bin/bash -c "cd /opt/fluentd/deis-output && rake test" + docker run ${IMAGE} /bin/bash -c "cd /fluentd/deis-output && rake test" install: helm upgrade fluentd charts/fluentd --install --namespace deis --set org=${IMAGE_PREFIX},docker_tag=${VERSION} diff --git a/charts/fluentd/templates/logger-fluentd-clusterrole.yaml b/charts/fluentd/templates/logger-fluentd-clusterrole.yaml index 56f92e1..65dc5b5 100644 --- a/charts/fluentd/templates/logger-fluentd-clusterrole.yaml +++ b/charts/fluentd/templates/logger-fluentd-clusterrole.yaml @@ -9,7 +9,7 @@ metadata: heritage: deis rules: - apiGroups: [""] - resources: ["pods"] + resources: ["pods", "namespaces"] verbs: ["list", "get", "watch"] {{- end -}} {{- end -}} diff --git a/rootfs/Dockerfile b/rootfs/Dockerfile index a3f9203..9a42478 100644 --- a/rootfs/Dockerfile +++ b/rootfs/Dockerfile @@ -1,47 +1,27 @@ -FROM quay.io/deis/base:v0.3.6 +FROM fluent/fluentd:v1.3 -COPY . / +COPY fluentd /fluentd +COPY entrypoint.sh /bin/ +RUN chmod +x /bin/entrypoint.sh -RUN buildDeps='g++ gcc make ruby-dev'; \ - apt-get update && \ - apt-get install -y \ - $buildDeps \ - ruby && \ - export FLUENTD_VERSION=0.14.13 && \ - gem install --no-document fluentd -v $FLUENTD_VERSION && \ - gem install bundler && \ - bundle install --gemfile=/opt/fluentd/deis-output/Gemfile && \ - rake --rakefile=/opt/fluentd/deis-output/Rakefile build && \ - fluent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 0.25.3 && \ - fluent-gem install --no-document fluent-plugin-elasticsearch -v 2.3.0 && \ - fluent-gem install --no-document fluent-plugin-remote_syslog -v 0.3.2 && \ - fluent-gem install --no-document fluent-plugin-sumologic-mattk42 -v 0.0.4 && \ - fluent-gem install --no-document fluent-plugin-gelf-hs -v 1.0.4 && \ - fluent-gem install --no-document influxdb -v 0.3.2 && \ - fluent-gem install --no-document nsq-ruby -v 1.7.0 && \ - fluent-gem install --local /opt/fluentd/deis-output/pkg/fluent-plugin-deis_output-0.1.0.gem && \ - # cleanup - apt-get purge -y --auto-remove $buildDeps && \ - apt-get autoremove -y && \ - apt-get clean -y && \ - # package up license files if any by appending to existing tar - COPYRIGHT_TAR='/usr/share/copyrights.tar'; \ - gunzip -f $COPYRIGHT_TAR.gz; tar -rf $COPYRIGHT_TAR /usr/share/doc/*/copyright; gzip $COPYRIGHT_TAR && \ - rm -rf \ - /usr/share/doc \ - /usr/share/man \ - /usr/share/info \ - /usr/share/locale \ - /var/lib/apt/lists/* \ - /var/log/* \ - /var/cache/debconf/* \ - /etc/systemd \ - /lib/lsb \ - /lib/udev \ - /usr/lib/x86_64-linux-gnu/gconv/IBM* \ - /usr/lib/x86_64-linux-gnu/gconv/EBC* && \ - bash -c "mkdir -p /usr/share/man/man{1..8}" +RUN apk add bash findutils +RUN apk add --update --virtual .build-deps sudo build-base ruby-dev \ + && gem install bundler \ + && bundle config --global silence_root_warning 1 \ + && bundle install --gemfile=/fluentd/deis-output/Gemfile \ + && rake --rakefile=/fluentd/deis-output/Rakefile build \ + && fluent-gem install --no-document fluent-plugin-kubernetes_metadata_filter -v 2.1.4 \ + && fluent-gem install --no-document fluent-plugin-elasticsearch -v 2.12.0 \ + && fluent-gem install --no-document fluent-plugin-remote_syslog -v 1.0.0 \ + && fluent-gem install --no-document fluent-plugin-sumologic-mattk42 -v 0.0.4 \ + && fluent-gem install --no-document fluent-plugin-gelf-hs -v 1.0.7 \ + && fluent-gem install --no-document influxdb -v 0.6.1 \ + && fluent-gem install --no-document nsq-ruby -v 2.3.1 \ + && fluent-gem install --local /fluentd/deis-output/pkg/fluent-plugin-deis_output-0.1.0.gem \ + && sudo gem sources --clear-all \ + && apk del .build-deps \ + && rm -rf /var/cache/apk/* /root/.gem/ruby/*/cache/*.gem -COPY /var /var +ENTRYPOINT ["/bin/entrypoint.sh"] -CMD ["/opt/fluentd/sbin/boot"] +CMD ["/fluentd/sbin/boot"] diff --git a/rootfs/entrypoint.sh b/rootfs/entrypoint.sh new file mode 100644 index 0000000..5ead6e4 --- /dev/null +++ b/rootfs/entrypoint.sh @@ -0,0 +1,3 @@ +#!/usr/bin/dumb-init /bin/sh + +exec "$@" diff --git a/rootfs/opt/fluentd/deis-output/.ruby-version b/rootfs/fluentd/deis-output/.ruby-version similarity index 100% rename from rootfs/opt/fluentd/deis-output/.ruby-version rename to rootfs/fluentd/deis-output/.ruby-version diff --git a/rootfs/opt/fluentd/deis-output/Gemfile b/rootfs/fluentd/deis-output/Gemfile similarity index 100% rename from rootfs/opt/fluentd/deis-output/Gemfile rename to rootfs/fluentd/deis-output/Gemfile diff --git a/rootfs/opt/fluentd/deis-output/Gemfile.lock b/rootfs/fluentd/deis-output/Gemfile.lock similarity index 100% rename from rootfs/opt/fluentd/deis-output/Gemfile.lock rename to rootfs/fluentd/deis-output/Gemfile.lock diff --git a/rootfs/opt/fluentd/deis-output/LICENSE b/rootfs/fluentd/deis-output/LICENSE similarity index 100% rename from rootfs/opt/fluentd/deis-output/LICENSE rename to rootfs/fluentd/deis-output/LICENSE diff --git a/rootfs/opt/fluentd/deis-output/README.md b/rootfs/fluentd/deis-output/README.md similarity index 100% rename from rootfs/opt/fluentd/deis-output/README.md rename to rootfs/fluentd/deis-output/README.md diff --git a/rootfs/opt/fluentd/deis-output/Rakefile b/rootfs/fluentd/deis-output/Rakefile similarity index 100% rename from rootfs/opt/fluentd/deis-output/Rakefile rename to rootfs/fluentd/deis-output/Rakefile diff --git a/rootfs/opt/fluentd/deis-output/fluent-plugin-deis_output.gemspec b/rootfs/fluentd/deis-output/fluent-plugin-deis_output.gemspec similarity index 100% rename from rootfs/opt/fluentd/deis-output/fluent-plugin-deis_output.gemspec rename to rootfs/fluentd/deis-output/fluent-plugin-deis_output.gemspec diff --git a/rootfs/opt/fluentd/deis-output/lib/fluent/mixin/deis.rb b/rootfs/fluentd/deis-output/lib/fluent/mixin/deis.rb similarity index 100% rename from rootfs/opt/fluentd/deis-output/lib/fluent/mixin/deis.rb rename to rootfs/fluentd/deis-output/lib/fluent/mixin/deis.rb diff --git a/rootfs/opt/fluentd/deis-output/lib/fluent/plugin/out_deis.rb b/rootfs/fluentd/deis-output/lib/fluent/plugin/out_deis.rb similarity index 100% rename from rootfs/opt/fluentd/deis-output/lib/fluent/plugin/out_deis.rb rename to rootfs/fluentd/deis-output/lib/fluent/plugin/out_deis.rb diff --git a/rootfs/opt/fluentd/deis-output/test/test_out_deis.rb b/rootfs/fluentd/deis-output/test/test_out_deis.rb similarity index 100% rename from rootfs/opt/fluentd/deis-output/test/test_out_deis.rb rename to rootfs/fluentd/deis-output/test/test_out_deis.rb diff --git a/rootfs/opt/fluentd/conf/fluentd.conf b/rootfs/fluentd/etc/fluent.conf similarity index 100% rename from rootfs/opt/fluentd/conf/fluentd.conf rename to rootfs/fluentd/etc/fluent.conf diff --git a/rootfs/opt/fluentd/sbin/boot b/rootfs/fluentd/sbin/boot similarity index 74% rename from rootfs/opt/fluentd/sbin/boot rename to rootfs/fluentd/sbin/boot index 4f334c6..e00f316 100755 --- a/rootfs/opt/fluentd/sbin/boot +++ b/rootfs/fluentd/sbin/boot @@ -1,13 +1,12 @@ #!/bin/bash -FLUENTD_CONF=${FLUENTD_CONF:-"/opt/fluentd/conf/fluentd.conf"} +FLUENTD_CONF="/fluentd/etc/${FLUENTD_CONF}" INSTALL_BUILD_TOOLS=${INSTALL_BUILD_TOOLS:-"false"} BUILD_TOOLS='g++ gcc make ruby-dev' if [ "$INSTALL_BUILD_TOOLS" == "true" ] then echo "Installing Build tools!" - apt-get update - apt-get install -y ruby $BUILD_TOOLS + apk add --update --virtual .build-deps sudo build-base ruby-dev fi # is there a broken symlink in /var/log/containers? @@ -24,9 +23,9 @@ if [ -n "$BROKEN_LINK" ]; then echo "linked /var/lib/docker/containers to $TARGET_DIR" fi -source /opt/fluentd/sbin/plugins -source /opt/fluentd/sbin/sources -source /opt/fluentd/sbin/filters/filters +source /fluentd/sbin/plugins +source /fluentd/sbin/sources +source /fluentd/sbin/filters/filters if [ -n "$DROP_FLUENTD_LOGS" ] then @@ -42,7 +41,7 @@ cat << EOF >> $FLUENTD_CONF @type copy EOF -source /opt/fluentd/sbin/stores/stores +source /fluentd/sbin/stores/stores cat << EOF >> $FLUENTD_CONF @@ -51,9 +50,9 @@ EOF if [ "$INSTALL_BUILD_TOOLS" == "true" ] then echo "Removing Build tools!" - apt-get remove -y --auto-remove --purge $BUILD_TOOLS - apt-get clean - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/man /usr/share/doc + sudo gem sources --clear-all + apk del .build-deps + rm -rf /var/cache/apk/* /root/.gem/ruby/*/cache/*.gem fi exec fluentd -c $FLUENTD_CONF diff --git a/rootfs/opt/fluentd/sbin/filters/custom_filters b/rootfs/fluentd/sbin/filters/custom_filters similarity index 100% rename from rootfs/opt/fluentd/sbin/filters/custom_filters rename to rootfs/fluentd/sbin/filters/custom_filters diff --git a/rootfs/fluentd/sbin/filters/filters b/rootfs/fluentd/sbin/filters/filters new file mode 100755 index 0000000..a7f71fc --- /dev/null +++ b/rootfs/fluentd/sbin/filters/filters @@ -0,0 +1,3 @@ +#!/bin/bash +source /fluentd/sbin/filters/kubernetes +source /fluentd/sbin/filters/custom_filters diff --git a/rootfs/opt/fluentd/sbin/filters/kubernetes b/rootfs/fluentd/sbin/filters/kubernetes similarity index 100% rename from rootfs/opt/fluentd/sbin/filters/kubernetes rename to rootfs/fluentd/sbin/filters/kubernetes diff --git a/rootfs/opt/fluentd/sbin/plugins b/rootfs/fluentd/sbin/plugins similarity index 100% rename from rootfs/opt/fluentd/sbin/plugins rename to rootfs/fluentd/sbin/plugins diff --git a/rootfs/opt/fluentd/sbin/sources b/rootfs/fluentd/sbin/sources similarity index 100% rename from rootfs/opt/fluentd/sbin/sources rename to rootfs/fluentd/sbin/sources diff --git a/rootfs/opt/fluentd/sbin/stores/custom_stores b/rootfs/fluentd/sbin/stores/custom_stores similarity index 100% rename from rootfs/opt/fluentd/sbin/stores/custom_stores rename to rootfs/fluentd/sbin/stores/custom_stores diff --git a/rootfs/opt/fluentd/sbin/stores/deis b/rootfs/fluentd/sbin/stores/deis similarity index 100% rename from rootfs/opt/fluentd/sbin/stores/deis rename to rootfs/fluentd/sbin/stores/deis diff --git a/rootfs/opt/fluentd/sbin/stores/elastic_search b/rootfs/fluentd/sbin/stores/elastic_search similarity index 100% rename from rootfs/opt/fluentd/sbin/stores/elastic_search rename to rootfs/fluentd/sbin/stores/elastic_search diff --git a/rootfs/opt/fluentd/sbin/stores/gelf b/rootfs/fluentd/sbin/stores/gelf similarity index 100% rename from rootfs/opt/fluentd/sbin/stores/gelf rename to rootfs/fluentd/sbin/stores/gelf diff --git a/rootfs/fluentd/sbin/stores/stores b/rootfs/fluentd/sbin/stores/stores new file mode 100755 index 0000000..c2ddbd5 --- /dev/null +++ b/rootfs/fluentd/sbin/stores/stores @@ -0,0 +1,8 @@ +#!/bin/bash + +source /fluentd/sbin/stores/deis +source /fluentd/sbin/stores/elastic_search +source /fluentd/sbin/stores/syslog +source /fluentd/sbin/stores/sumologic +source /fluentd/sbin/stores/custom_stores +source /fluentd/sbin/stores/gelf diff --git a/rootfs/opt/fluentd/sbin/stores/sumologic b/rootfs/fluentd/sbin/stores/sumologic similarity index 100% rename from rootfs/opt/fluentd/sbin/stores/sumologic rename to rootfs/fluentd/sbin/stores/sumologic diff --git a/rootfs/opt/fluentd/sbin/stores/syslog b/rootfs/fluentd/sbin/stores/syslog similarity index 100% rename from rootfs/opt/fluentd/sbin/stores/syslog rename to rootfs/fluentd/sbin/stores/syslog diff --git a/rootfs/opt/fluentd/sbin/filters/filters b/rootfs/opt/fluentd/sbin/filters/filters deleted file mode 100755 index 8924a88..0000000 --- a/rootfs/opt/fluentd/sbin/filters/filters +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -source /opt/fluentd/sbin/filters/kubernetes -source /opt/fluentd/sbin/filters/custom_filters diff --git a/rootfs/opt/fluentd/sbin/stores/stores b/rootfs/opt/fluentd/sbin/stores/stores deleted file mode 100755 index d79460c..0000000 --- a/rootfs/opt/fluentd/sbin/stores/stores +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash - -source /opt/fluentd/sbin/stores/deis -source /opt/fluentd/sbin/stores/elastic_search -source /opt/fluentd/sbin/stores/syslog -source /opt/fluentd/sbin/stores/sumologic -source /opt/fluentd/sbin/stores/custom_stores -source /opt/fluentd/sbin/stores/gelf diff --git a/rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-1.7.0/lib/fluent/plugin/out_elasticsearch.rb b/rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-1.7.0/lib/fluent/plugin/out_elasticsearch.rb deleted file mode 100644 index 5596c0b..0000000 --- a/rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-elasticsearch-1.7.0/lib/fluent/plugin/out_elasticsearch.rb +++ /dev/null @@ -1,370 +0,0 @@ -# encoding: UTF-8 -require 'date' -require 'excon' -require 'elasticsearch' -require 'json' -require 'uri' -begin - require 'strptime' -rescue LoadError -end - -require_relative 'elasticsearch_index_template' - -class Fluent::ElasticsearchOutput < Fluent::BufferedOutput - class ConnectionFailure < StandardError; end - - Fluent::Plugin.register_output('elasticsearch', self) - - config_param :host, :string, :default => 'localhost' - config_param :port, :integer, :default => 9200 - config_param :user, :string, :default => nil - config_param :password, :string, :default => nil, :secret => true - config_param :path, :string, :default => nil - config_param :scheme, :string, :default => 'http' - config_param :hosts, :string, :default => nil - config_param :target_index_key, :string, :default => nil - config_param :target_type_key, :string, :default => nil - config_param :time_key_format, :string, :default => nil - config_param :logstash_format, :bool, :default => false - config_param :logstash_prefix, :string, :default => "logstash" - config_param :logstash_dateformat, :string, :default => "%Y.%m.%d" - config_param :utc_index, :bool, :default => true - config_param :type_name, :string, :default => "fluentd" - config_param :index_name, :string, :default => "fluentd" - config_param :id_key, :string, :default => nil - config_param :write_operation, :string, :default => "index" - config_param :parent_key, :string, :default => nil - config_param :routing_key, :string, :default => nil - config_param :request_timeout, :time, :default => 5 - config_param :reload_connections, :bool, :default => true - config_param :reload_on_failure, :bool, :default => false - config_param :resurrect_after, :time, :default => 60 - config_param :time_key, :string, :default => nil - config_param :time_key_exclude_timestamp, :bool, :default => false - config_param :ssl_verify , :bool, :default => true - config_param :client_key, :string, :default => nil - config_param :client_cert, :string, :default => nil - config_param :client_key_pass, :string, :default => nil - config_param :ca_file, :string, :default => nil - config_param :remove_keys, :string, :default => nil - config_param :remove_keys_on_update, :string, :default => "" - config_param :remove_keys_on_update_key, :string, :default => nil - config_param :flatten_hashes, :bool, :default => false - config_param :flatten_hashes_separator, :string, :default => "_" - config_param :template_name, :string, :default => nil - config_param :template_file, :string, :default => nil - - include Fluent::SetTagKeyMixin - include Fluent::ElasticsearchIndexTemplate - config_set_default :include_tag_key, false - - def initialize - super - @time_parser = TimeParser.new(@time_key_format, @router) - end - - def configure(conf) - super - @time_parser = TimeParser.new(@time_key_format, @router) - - if @remove_keys - @remove_keys = @remove_keys.split(/\s*,\s*/) - end - - if @target_index_key && @target_index_key.is_a?(String) - @target_index_key = @target_index_key.split '.' - end - - if @target_type_key && @target_type_key.is_a?(String) - @target_type_key = @target_type_key.split '.' - end - - if @remove_keys_on_update && @remove_keys_on_update.is_a?(String) - @remove_keys_on_update = @remove_keys_on_update.split ',' - end - - if @template_name && @template_file - template_install(@template_name, @template_file) - end - end - - def start - super - end - - # once fluent v0.14 is released we might be able to use - # Fluent::Parser::TimeParser, but it doesn't quite do what we want - if gives - # [sec,nsec] where as we want something we can call `strftime` on... - class TimeParser - def initialize(time_key_format, router) - @time_key_format = time_key_format - @router = router - @parser = if time_key_format - begin - # Strptime doesn't support all formats, but for those it does it's - # blazingly fast. - strptime = Strptime.new(time_key_format) - Proc.new { |value| strptime.exec(value).to_datetime } - rescue - # Can happen if Strptime doesn't recognize the format; or - # if strptime couldn't be required (because it's not installed -- it's - # ruby 2 only) - Proc.new { |value| DateTime.strptime(value, time_key_format) } - end - else - Proc.new { |value| DateTime.parse(value) } - end - end - - def parse(value, event_time) - @parser.call(value) - rescue => e - @router.emit_error_event("Fluent::ElasticsearchOutput::TimeParser.error", Fluent::Engine.now, {'time' => event_time, 'format' => @time_key_format, 'value' => value }, e) - return Time.at(event_time).to_datetime - end - end - - def client - @_es ||= begin - excon_options = { client_key: @client_key, client_cert: @client_cert, client_key_pass: @client_key_pass } - adapter_conf = lambda {|f| f.adapter :excon, excon_options } - transport = Elasticsearch::Transport::Transport::HTTP::Faraday.new(get_connection_options.merge( - options: { - reload_connections: @reload_connections, - reload_on_failure: @reload_on_failure, - resurrect_after: @resurrect_after, - retry_on_failure: 5, - transport_options: { - request: { timeout: @request_timeout }, - ssl: { verify: @ssl_verify, ca_file: @ca_file } - } - }), &adapter_conf) - es = Elasticsearch::Client.new transport: transport - - begin - raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})!" unless es.ping - rescue *es.transport.host_unreachable_exceptions => e - raise ConnectionFailure, "Can not reach Elasticsearch cluster (#{connection_options_description})! #{e.message}" - end - - log.info "Connection opened to Elasticsearch cluster => #{connection_options_description}" - es - end - end - - def get_connection_options - raise "`password` must be present if `user` is present" if @user && !@password - - hosts = if @hosts - @hosts.split(',').map do |host_str| - # Support legacy hosts format host:port,host:port,host:port... - if host_str.match(%r{^[^:]+(\:\d+)?$}) - { - host: host_str.split(':')[0], - port: (host_str.split(':')[1] || @port).to_i, - scheme: @scheme - } - else - # New hosts format expects URLs such as http://logs.foo.com,https://john:pass@logs2.foo.com/elastic - uri = URI(host_str) - %w(user password path).inject(host: uri.host, port: uri.port, scheme: uri.scheme) do |hash, key| - hash[key.to_sym] = uri.public_send(key) unless uri.public_send(key).nil? || uri.public_send(key) == '' - hash - end - end - end.compact - else - [{host: @host, port: @port, scheme: @scheme}] - end.each do |host| - host.merge!(user: @user, password: @password) if !host[:user] && @user - host.merge!(path: @path) if !host[:path] && @path - end - - { - hosts: hosts - } - end - - def connection_options_description - get_connection_options[:hosts].map do |host_info| - attributes = host_info.dup - attributes[:password] = 'obfuscated' if attributes.has_key?(:password) - attributes.inspect - end.join(', ') - end - - def format(tag, time, record) - [tag, time, record].to_msgpack - end - - def shutdown - super - end - - def append_record_to_messages(op, meta, record, msgs) - case op - when "update", "upsert" - if meta.has_key?("_id") - msgs << { "update" => meta } - msgs << update_body(record, op) - end - when "create" - if meta.has_key?("_id") - msgs << { "create" => meta } - msgs << record - end - when "index" - msgs << { "index" => meta } - msgs << record - end - end - - def update_body(record, op) - update = remove_keys(record) - body = { "doc" => update } - if op == "upsert" - if update == record - body["doc_as_upsert"] = true - else - body["upsert"] = record - end - end - body - end - - def remove_keys(record) - keys = record[@remove_keys_on_update_key] || @remove_keys_on_update || [] - record.delete(@remove_keys_on_update_key) - return record unless keys.any? - record = record.dup - keys.each { |key| record.delete(key) } - record - end - - def flatten_record(record, prefix=[]) - ret = {} - if record.is_a? Hash - record.each { |key, value| - ret.merge! flatten_record(value, prefix + [key.to_s]) - } - elsif record.is_a? Array - # Don't mess with arrays, leave them unprocessed - ret.merge!({prefix.join(@flatten_hashes_separator) => record}) - else - return {prefix.join(@flatten_hashes_separator) => record} - end - ret - end - - def write(chunk) - bulk_message = [] - - chunk.msgpack_each do |tag, time, record| - if @flatten_hashes - record = flatten_record(record) - end - next unless record.is_a? Hash - - dt = get_time(record, time) - target_index = get_target_index(record, dt) - - - target_type_parent, target_type_child_key = get_parent_of(record, @target_type_key) - if target_type_parent && target_type_parent[target_type_child_key] - target_type = target_type_parent.delete(target_type_child_key) - else - target_type = @type_name - end - - meta = {"_index" => target_index, "_type" => target_type} - - @meta_config_map ||= { 'id_key' => '_id', 'parent_key' => '_parent', 'routing_key' => '_routing' } - @meta_config_map.each_pair do |config_name, meta_key| - record_key = self.instance_variable_get("@#{config_name}") - meta[meta_key] = record[record_key] if record_key && record[record_key] - end - - if @remove_keys - @remove_keys.each { |key| record.delete(key) } - end - log = record["log"] - record["log"] = log.gsub(/\e\[\d*m{1}/, "").lstrip if log - append_record_to_messages(@write_operation, meta, record, bulk_message) - end - - send(bulk_message) unless bulk_message.empty? - bulk_message.clear - end - - def get_time(record, time) - if record.has_key?("@timestamp") - dt = record["@timestamp"] - dt = @time_parser.parse(record["@timestamp"], time) - elsif record.has_key?(@time_key) - dt = @time_parser.parse(record[@time_key], time) - record['@timestamp'] = record[@time_key] unless time_key_exclude_timestamp - else - if time.class == Time - dt = Time.at(time).to_datetime - else - dt = Time.now.to_datetime - end - record.merge!({"@timestamp" => dt.to_s}) - end - return dt - end - - # Get the target_index - # If we have passed in a target_index_key we should use that value first. - # --- We should also check to see if the user wants this in logstash format meaning we need to append the date to the end - # Next if the user only wants this in logstash format and hasn't provided a target_index_key, then use the logstash prefix - # Last just use the index_name that was passed in to the plugin at configuration. - def get_target_index(record, dt) - target_index_parent, target_index_child_key = get_parent_of(record, @target_index_key) - if target_index_parent && target_index_parent[target_index_child_key] - target_index = target_index_parent.delete(target_index_child_key) - if @logstash_format - target_index = "#{target_index}-#{dt.strftime(@logstash_dateformat)}" - end - elsif @logstash_format && !@target_index_key - dt = dt.new_offset(0) if @utc_index - target_index = "#{@logstash_prefix}-#{dt.strftime(@logstash_dateformat)}" - else - target_index = @index_name - end - - # Change target_index to lower-case since Elasticsearch doesn't - # allow upper-case characters in index names. - target_index = target_index.downcase - if @include_tag_key - record.merge!(@tag_key => tag) - end - return target_index - end - - # returns [parent, child_key] of child described by path array in record's tree - # returns [nil, child_key] if path doesnt exist in record - def get_parent_of(record, path) - return [nil, nil] unless path - - parent_object = path[0..-2].reduce(record) { |a, e| a.is_a?(Hash) ? a[e] : nil } - [parent_object, path[-1]] - end - - def send(data) - retries = 0 - begin - client.bulk body: data - rescue *client.transport.host_unreachable_exceptions => e - if retries < 2 - retries += 1 - @_es = nil - log.warn "Could not push logs to Elasticsearch, resetting connection and trying again. #{e.message}" - sleep 2**retries - retry - end - raise ConnectionFailure, "Could not push logs to Elasticsearch after #{retries} retries. #{e.message}" - end - end -end diff --git a/rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-remote_syslog-0.3.2/lib/fluent/plugin/out_remote_syslog.rb b/rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-remote_syslog-0.3.2/lib/fluent/plugin/out_remote_syslog.rb deleted file mode 100644 index b7317c2..0000000 --- a/rootfs/var/lib/gems/2.3.0/gems/fluent-plugin-remote_syslog-0.3.2/lib/fluent/plugin/out_remote_syslog.rb +++ /dev/null @@ -1,57 +0,0 @@ -require "fluent/mixin/config_placeholders" -require "fluent/mixin/plaintextformatter" -require 'fluent/mixin/rewrite_tag_name' - -module Fluent - class RemoteSyslogOutput < Fluent::Output - Fluent::Plugin.register_output("remote_syslog", self) - - config_param :hostname, :string, :default => "" - - include Fluent::Mixin::PlainTextFormatter - include Fluent::Mixin::ConfigPlaceholders - include Fluent::HandleTagNameMixin - include Fluent::Mixin::RewriteTagName - - config_param :host, :string - config_param :port, :integer, :default => 25 - - config_param :facility, :string, :default => "user" - config_param :severity, :string, :default => "notice" - config_param :tag, :string, :default => "fluentd" - - def initialize - super - require "remote_syslog_logger" - end - - def start - super - end - - def shutdown - super - end - - def emit(tag, es, chain) - chain.next - es.each do |time, record| - record.each_pair do |k, v| - if v.is_a?(String) - v.force_encoding("utf-8") - end - end - - tag = rewrite_tag!(tag.dup) - sender = RemoteSyslogLogger::UdpSender.new(@host, @port, {facility: @facility, severity: @severity, program: tag, local_hostname: @hostname}) - begin - sender.transmit format(tag, time, record) - rescue Exception => e - puts "Error:#{e.message}" - ensure - sender.close if sender - end - end - end - end -end