filebeat.prospectors:
- input_type: log
  paths:
    - /alidata/log/tomcat-*/catalina.out
  multiline.pattern: '^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} \['
  multiline.negate: true
  multiline.match: after

output.redis:
  hosts: "192.168.1.61:6379"
  password: "redispwd"
  key: "tomcat-catalina-log"
  db: 0
  timeout: 5

{
  "@timestamp": "2017-06-21T03:40:42.312Z",
  "beat": {
    "hostname": "front-end",
    "name": "front-end",
    "version": "5.4.2"
  },
  "input_type": "log",
  "message": "2017-06-21 11:40:33,572 [ERROR] xxx.xxxxxxx.Logger:87 - org.springframework.dao.DataIntegrityViolationException: SqlMapClient operation; SQL []; Column 'FID' cannot be null; nested exception is java.sql.BatchUpdateException: Column 'FID' cannot be null\\n\\tat org.springframework.jdbc.support.SQLStateSQLExceptionTranslator.doTranslate(SQLStateSQLExceptionTranslator.java:102)\\n\\tat org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:73)\\n\\tat org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)\\n\\tat org.springframework.jdbc.support.AbstractFallbackSQLExceptionTranslator.translate(AbstractFallbackSQLExceptionTranslator.java:81)\\n\\tat org.springframework.orm.ibatis.SqlMapClientTemplate.execute(SqlMapClientTemplate.java:204)\\n\\tat java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)\\n\\tat java.util.concurrent.FutureTask.runAndReset(FutureTask.java:308)\\n\\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.access$301(ScheduledThreadPoolExecutor.java:180)\\n\\tat java.util.concurrent.ScheduledThreadPoolExecutor$ScheduledFutureTask.run(ScheduledThreadPoolExecutor.java:294)\\n\\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\\n\\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\\n\\tat java.lang.Thread.run(Thread.java:745)\\n",
  "offset": 29117034,
  "source": "/alidata/log/tomcat-xxx/catalina.out",
  "type": "log"
}

input {
	redis {
		data_type => "list"
		host => "192.168.1.61"
		port => 6379
		password => "redispwd"
		db => 0
		key => "tomcat-catalina-log"
	}
}

filter {
	grok {
		patterns_dir => ["/root/elk/patterns"]
		match => {
		    "message" => "(?<log_time>(^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3})) \[(?<log_level>([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?))\s?\] (?<log_class>(?:[a-zA-Z$_][a-zA-Z$_0-9]*\.*)*[a-zA-Z$_][a-zA-Z$_0-9]*):[0-9]* - (?<log_msg>.*)"
		}
		match => {
			"source" => "/alidata/log/(?<log_from>.*)/catalina\.out"
		}
		break_on_match => false
		remove_field => ["beat"]
	}
	date {
		match => ["log_time", "yyyy-MM-dd HH:mm:ss,SSS"]
		timezone => "Asia/Shanghai"
	}

	if "_grokparsefailure" in [tags] {
		grok {
			match => {
				"message" => "(?<log_class>(?:[a-zA-Z$_][a-zA-Z$_0-9]*\.*)*[a-zA-Z$_][a-zA-Z$_0-9]*)"
			}
			add_field => {
				"log_level" => "ERROR"
				"log_time" => "%{@timestamp}"
				"log_msg" => "%{message}"
			}
		}
	}
}

output {
	elasticsearch {
		hosts => ["127.0.0.1:9200"]
		index => "logstash-%{type}-%{+YYYY.MM.dd}"
	}
}

path.data: /root/elk/data/es/data
path.logs: /root/elk/data/es/logs
network.host: 192.168.1.61
http.port: 9200