Streaming weather Data into HDFS using Flume

Hi All

i am trying to stream weather data into Hdfs i have written Java class as follows

public class OpenWeatherParser extends AbstractSource implements EventDrivenSource {

private static String [] citys = {“Amsterdam”, “Ankara”, “Andorra la Vella”, “Athen”, “Belgrad”, “Berlin”, “Bern”,
“Bratislava”, “Brüssel”, “Budapest”, “ChiÈ™inău”,
“Dublin, irl”, “Helsinki”, “Kiew”, “Kopenhagen “, “Lissabon”, “Ljubljana”, “London,uk”,
“Madrid”, “Minsk”, “Monaco “, “Moskau”, “Oslo”,”Paris,fr”,
“Podgorica”, “Reykjavík”, “Riga “, “Rom”, “Skopje”,
“Sofia”, “Stockholm”, “Talinn”, “Tirana”, “Vaduz”, “Valletta”, “Vilnius”,
“Warschau”, “Wien”, “Zagreb”};

private static String weatherJsonObject;

@Override
public void start() {
// TODO Auto-generated method stub

//wird stündlich ausgeführt
while(true){

final ChannelProcessor channel = getChannelProcessor();

//Einheitlicher Zeitpunkt an dem die Daten abgerufen wurden
Date aDate = new Date( );
SimpleDateFormat sdt = new SimpleDateFormat (“yyyy.MM.dd_HH:mm:ss”);
String dt = sdt.format(aDate);

//ruft die Daten aller Städte ab
for (String city: citys){

String url = “http://api.openweathermap.org/data/2.5/weather?q=” + city +”&mode=json&units=metric&lang=de”;

final Map<String, String> headers = new HashMap<String, String>();
Event event = null;
Weather aWeather = new Weather();

//JSON holen und auswerten
try {

InputStream input = new URL(url).openStream();
weatherJsonObject = IOUtils.toString(input);
JSONObject obj1 = (JSONObject) JSONValue.parseWithException(weatherJsonObject);

if (!obj1.isEmpty()){

JSONObject obj2 = (JSONObject) obj1.get(“coord”);
JSONObject obj3 = (JSONObject) obj1.get(“sys”);
JSONObject obj4 = (JSONObject) obj1.get(“main”);
JSONObject obj5 = (JSONObject) obj1.get(“wind”);

aWeather.setId(obj1.get(“id”).toString());
aWeather.setDt(obj1.get(“dt”).toString());
aWeather.setName(obj1.get(“name”).toString());

//coord
if (!obj2.isEmpty()){

aWeather.setLon(obj2.get(“lon”).toString());
aWeather.setLat(obj2.get(“lat”).toString());
}

//sys
if(!obj3.isEmpty()){
aWeather.setCountry(obj3.get(“country”).toString());
aWeather.setSunrise(obj3.get(“sunrise”).toString());
aWeather.setSunset(obj3.get(“sunset”).toString());
}

//main
if (!obj4.isEmpty()){

aWeather.setTemp(obj4.get(“temp”).toString());
aWeather.setHumidity(obj4.get(“humidity”).toString());
aWeather.setPressure(obj4.get(“pressure”).toString());
aWeather.setTemp_min(obj4.get(“temp_min”).toString());
aWeather.setTemp_max(obj4.get(“temp_max”).toString());
}

//wind
if (!obj5.isEmpty()){

aWeather.setSpeed(obj5.get(“speed”).toString());
aWeather.setDeg(obj5.get(“deg”).toString());
}
}

} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
} catch (ParseException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}

//body zusammenbauen: Zeit,Stadt,Land,Sonnaufgang, Sonnenuntergang, aktuelleTemperatur,
// maxTemperatur, mainTemperatur, Luftfeuchtigkeit, Luftdruck, Windgeschwindigkeit
String body = dt + “,” + aWeather.getName() + “,”+ aWeather.getCountry() + “,”
+ aWeather.getSunrise() + “,” + aWeather.getSunset() + “,”
+ aWeather.getTemp() + “,” + aWeather.getTemp_max() + “,” + aWeather.getTemp_min() + “,”
+ aWeather.getHumidity() + “,” + aWeather.getPressure() + “,” + aWeather.getSpeed() + “;”;

//header zusammenbauen
long now = System.currentTimeMillis();
headers.put(“timestamp”, Long.toString(now));

//event generieren und an den Channel übergeben
event = EventBuilder.withBody(body.getBytes(), headers);
channel.processEvent(event);

}

try {
Thread.sleep(3600000); // 1 Stunde schlafen:
} catch (InterruptedException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
}
}

And I have written Flume Agent as follows

agent1.sources = Weather
agent1.channels = MemChannel
agent1.sinks = HDFS

agent1.sources.Weather.type = com.main.OpenWeatherParser
agent1.sources.Weather.channels = MemChannel

agent1.sinks.HDFS.channel = MemChannel
agent1.sinks.HDFS.type = hdfs
agent1.sinks.HDFS.hdfs.path = hdfs://hadoop1:8020/user/flume/input/
agent1.sinks.HDFS.hdfs.fileType = DataStream
agent1.sinks.HDFS.hdfs.writeFormat = Text
agent1.sinks.HDFS.hdfs.batchSize = 1000
agent1.sinks.HDFS.hdfs.rollSize = 0
agent1.sinks.HDFS.hdfs.rollCount = 10000

agent1.channels.MemChannel.type = memory
agent1.channels.MemChannel.capacity = 10000
agent1.channels.MemChannel.transactionCapacity = 100

And My Flume-env.sh is as follows

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# “License”); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an “AS IS” BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# If this file is placed at FLUME_CONF_DIR/flume-env.sh, it will be sourced
# during Flume startup.

# Enviroment variables can be set here.

JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-1.7.0.71.x86_64

# Give Flume more memory and pre-allocate, enable remote monitoring via JMX
#JAVA_OPTS=”-Xms100m -Xmx200m -Dcom.sun.management.jmxremote”

# Note that the Flume conf directory is always included in the classpath.
FLUME_CLASSPATH=”/apache-flume-1.5.2-bin/lib/OpenWeatherParser.jar”

I am getting error as follows

15/03/24 11:32:44 ERROR node.PollingPropertiesFileConfigurationProvider: Failed
to load configuration data. Exception follows.
org.apache.flume.FlumeException: Unable to load source type: com.main.OpenWeathe
rParser, class: com.main.OpenWeatherParser
at org.apache.flume.source.DefaultSourceFactory.getClass(DefaultSourceFa
ctory.java:67)
at org.apache.flume.source.DefaultSourceFactory.create(DefaultSourceFact
ory.java:40)
at org.apache.flume.node.AbstractConfigurationProvider.loadSources(Abstr
actConfigurationProvider.java:327)
at org.apache.flume.node.AbstractConfigurationProvider.getConfiguration(
AbstractConfigurationProvider.java:102)
at org.apache.flume.node.PollingPropertiesFileConfigurationProvider$File

I have created the jar file of above java code and placed it in flume lib .
I am still not getting result.
please help me on this i am new to flume.