A Flume Configuration for Avro Ingestion

Home directory for Flume user. The File Channel uses paths for checkpoint and data directories that are within the user home: /var/lib/flume_ng

Plugin Directories: var/lib/flume_ng/plugins.d

Below is an agent configuration for ingesting avro message from avro-http-source to hdfs-sink.

# Name the components on this agent
sample_agent.sources =  avro_http_source
sample_agent.sinks = hdfs_sink
sample_agent.channels = file_channel

# Configure the source
sample_agent.sources.avro_http_source.type = avro
sample_agent.sources.avro_http_source.bind = 127.0.0.1
sample_agent.sources.avro_http_source.port = 8080
#sample_agent.sources.avro_http_source.threads = 10

# Configure the sinks
sample_agent.sinks.hdfs_sink.type = hdfs
sample_agent.sinks.hdfs_sink.hdfs.path = hdfs://localhost/user/flume/avro_ingest/%y/%m/%d
sample_agent.sinks.hdfs_sink.hdfs.useLocalTimeStamp = true
sample_agent.sinks.hdfs_sink.hdfs.fileType = DataStream
sample_agent.sinks.hdfs_sink.hdfs.fileSuffix = .avro
sample_agent.sinks.hdfs_sink.serializer = org.apache.flume.sink.hdfs.AvroEventSerializer$Builder
sample_agent.sinks.hdfs_sink.hdfs.rollInterval = 3600
sample_agent.sinks.hdfs_sink.hdfs.rollSize = 125000000
sample_agent.sinks.hdfs_sink.hdfs.rollCount = 0
sample_agent.sinks.hdfs_sink.hdfs.batchsize = 10000

# Bind the source and sink to the channel
sample_agent.sources.avro_http_source.channels = file_channel
sample_agent.sinks.hdfs_sink.channel = file_channel

# Configure the channels
sample_agent.channels.file_channel.type = file
sample_agent.channels.file_channel.checkpointDir = ~/sample_agent/checkpoint
sample_agent.channels.file_channel.dataDirs = ~/sample_agent/data
sample_agent.channels.file_channel.capacity = 1000000
sample_agent.channels.file_channel.transactionCapacity = 50000
sample_agent.channels.file_channel.keep_alive = 3
sample_agent.channels.file_channel.write_timeout = 3