ふり返る暇なんて無いね

日々のメモ書きをつらつらと。メインブログに書くほどでもないことを流してます

fluent-plugin-forestを使いつつ、tag_partsが複数あるときにpathにtag_partsを含めたときに、same buffer_pathを使っていると怒られる。

tech

要件としては、

ファイル名としては、remove_suffix後、最後のtag_partsを無視して欲しい
ログにはtagを残して欲しい

どうやら${tag_parts[1]}が異なっていることが原因でtemplateから別の設定が作られるがbuffer_pathが同一の箇所を示しているため、怒られる模様。

あとでちゃんとソースを見る。

環境

Ubuntu 12.04 64bit
td-agent: 1.1.18-1
fluent-plugin-forest: 0.2.4
fluent-plugin-file-alternative: 0.1.5

# uname -a
Linux mlab 3.8.0-33-generic #48~precise1-Ubuntu SMP Thu Oct 24 16:28:06 UTC 2013 x86_64 x86_64 x86_64 GNU/Linux

設定

<source>
  type        tail
  tag         app.foo.bar
  path        /tmp/app.foo.bar.log
  pos_file    /tmp/app.foo.bar.pos
  format      none
</source>
 
<source>
  type        tail
  tag         app.foo.baz
  path        /tmp/app.foo.baz.log
  pos_file    /tmp/app.foo.baz.pos
  format      none
</source>
 
 
<match app.**>
  type              forest
  subtype           file_alternative
  remove_prefix     app
  <template>
    path                /tmp/td-agent/app/${tag_parts[0]}/*.log
    symlink_path        /tmp/td-agent/app/${tag_parts[0]}.log
    output_data_type    attr:message
    output_include_time false
    compress            gz
  </template>
</match>

ログ

td-agent.log

2014-03-26 11:15:39 +0900 [info]: shutting down fluentd
2014-03-26 11:15:39 +0900 [info]: process finished code=0
2014-03-26 11:15:40 +0900 [info]: starting fluentd-0.10.41
2014-03-26 11:15:40 +0900 [info]: reading config file path="/etc/td-agent/td-agent.conf"
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-mixin-config-placeholders' version '0.2.3'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-mixin-plaintextformatter' version '0.2.4'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-amplifier-filter' version '0.1.5'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-datacounter' version '0.4.3'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-elasticsearch' version '0.2.0'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-file-alternative' version '0.1.5'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-flowcounter' version '0.2.0'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-flume' version '0.1.1'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-forest' version '0.2.4'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-growthforecast' version '0.2.7'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-mongo' version '0.7.1'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-numeric-monitor' version '0.1.7'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-parser' version '0.3.2'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '1.4.1'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-rewrite-tag-filter' version '1.3.1'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-s3' version '0.3.5'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-scribe' version '0.10.10'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-td' version '0.10.17'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-td-monitoring' version '0.1.0'
2014-03-26 11:15:40 +0900 [info]: gem 'fluent-plugin-webhdfs' version '0.2.1'
2014-03-26 11:15:40 +0900 [info]: gem 'fluentd' version '0.10.41'
2014-03-26 11:15:40 +0900 [info]: using configuration file: <ROOT>
  <source>
    type tail
    tag app.foo.bar
    path /tmp/app.foo.bar.log
    pos_file /tmp/app.foo.bar.pos
    format none
  </source>
  <source>
    type tail
    tag app.foo.baz
    path /tmp/app.foo.baz.log
    pos_file /tmp/app.foo.baz.pos
    format none
  </source>
  <match app.**>
    type forest
    subtype file_alternative
    remove_prefix app
    <template>
      path /tmp/td-agent/app/${tag_parts[0]}/*.log
      symlink_path /tmp/td-agent/app/${tag_parts[0]}.log
      output_data_type attr:message
      output_include_time false
      compress gz
    </template>
  </match>
</ROOT>
2014-03-26 11:15:40 +0900 [info]: adding source type="tail"
2014-03-26 11:15:40 +0900 [info]: adding source type="tail"
2014-03-26 11:15:40 +0900 [info]: adding match pattern="app.**" type="forest"
2014-03-26 11:15:40 +0900 [info]: following tail of /tmp/app.foo.bar.log
2014-03-26 11:15:40 +0900 [info]: following tail of /tmp/app.foo.baz.log
2014-03-26 11:15:47 +0900 [info]: out_forest plants new output: file_alternative for tag 'foo.baz'
2014-03-26 11:15:50 +0900 [error]: failed to configure sub output file_alternative: Other '' plugin already use same buffer_path: type = , buffer_path = /tmp/td-agent/app/foo/*.log
2014-03-26 11:15:50 +0900 [error]: /usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/buf_file.rb:94:in `configure'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/output.rb:185:in `configure'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/output.rb:465:in `configure'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-mixin-plaintextformatter-0.2.4/lib/fluent/mixin/plaintextformatter.rb:20:in `configure'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-file-alternative-0.1.5/lib/fluent/plugin/out_file_alternative.rb:65:in `configure'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-forest-0.2.4/lib/fluent/plugin/out_forest.rb:132:in `block in plant'
<internal:prelude>:10:in `synchronize'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-forest-0.2.4/lib/fluent/plugin/out_forest.rb:128:in `plant'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluent-plugin-forest-0.2.4/lib/fluent/plugin/out_forest.rb:168:in `emit'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/match.rb:36:in `emit'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/engine.rb:151:in `emit_stream'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:106:in `receive_lines'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:324:in `call'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:324:in `on_notify'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:155:in `on_notify'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:260:in `call'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:260:in `on_change'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/cool.io-1.1.1/lib/cool.io/loop.rb:96:in `run_once'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/cool.io-1.1.1/lib/cool.io/loop.rb:96:in `run'
/usr/lib/fluent/ruby/lib/ruby/gems/1.9.1/gems/fluentd-0.10.41/lib/fluent/plugin/in_tail.rb:83:in `run'
2014-03-26 11:15:50 +0900 [error]: Cannot output messages with tag 'foo.bar'