๐2022-07-20 ๐
ํ๋กํ์ผ ์์คํ ๋ฃจํธ ๋ฐ์ tmp.. ls ํ๋ฉด no such file์ด ๋์ค๋ ์ด์ ๋ ls ํ๋ฉด ์๋ ์ด์ ๋ ์ฌ์ฉ์๊ณ์ ์ด ์์ด์ ๋ฃจํธ๋ก ์ ๊ทผํ๋๊น,, ls๊ฐ ์ฌ์ฉ์ ๊ณ์ ์ผ๋ก ์ ๊ทผํ๊ฒ ๋์ด์ ๊ณ์ ์ ์ง์ ๋ง๋ค์ด์ค์ผํ๋ค.
๊ธฐ๋ณธ์ ์ผ๋ก ํ๋ก์ด๋ linux๋ช ๋ น์ด๊ฐ ๋น์ท.. ์ ์ฌํ๋ค
ubuntu22 ๋ก๊ทธ์ธ
mapred historyserver start& ์๋ฒ ์คํ
2๋ฒ์งธ ์ฐฝ์ผ๋ก ๋ค์ด๊ฐ๊ธฐ
jps์ ์ธํ 7๊ฐ ๋จ๋ ๊ฑฐ ํ์ธ
linux-> firefox -> localhost:50070
jps ๊ฐ์ํ์ธ
start-yarn.sh ์์
vi . ~/.bashrc ์์
hdfs dfs -ls / -> ์๋์ค๋๊น
hdfs dfs -mkdir /user ์ ์ ๋ฅผ ๋ฐ๋ก ๋ง๋ค์ด์ ์ค์ ํด์ค๋ค.
hdfs dfs -mkdir /user /gyuwon ์ด๋ ๊ฒ!
hdfs dfs -ls / ๋ค์ ์น๋ฉด
hdfs dfs -ls /user ์ ์ ๊ฐ ๋์จ๋ค.
rm test๊ฐ ์ ์์ง์์ง๊น,,,? dir... ํ์ผ์ด๋ผ์ ใ
getmerge ํ์ผ ์ ์ฅํ๋ค.
yarn jar ~/hadoop-3.2.3/share/hadoop/mapreduce-examples-3.2.3.jar wordcount sample-input sample-output
jar -tf WordCount-0.1.jar
hdfs dfs -cat sample-output/part-t-00000
์์ ์ค์ผ์ค๋ง ์์คํ ์์ ์ฌ์ฉ๋์ด ๋ชจ๋ ๋ฐ์ดํฐ๊ฐ ์ถ๋ ฅ๋ ๋
์ด ๋๋ ํ ๋ฆฌ์ ๋ด์ฉ์ ๋ํ ํ์ ์ฒ๋ฆฌ๊ฐ ์์๋ ์ ์์์ ๋ํ๋ธ๋ค. reducer 00000
cat ๋ณด์ฌ์ฃผ๋ ๊ฒ
text - cat์ ๊ธฐ๋ฅ + zip์ผ๋ก ๋ฌถ์ด์๋๋ผ๋ ์์ถ์ ์น๊ณ ๋ค์ด๊ฐ์ ๋ณด์ฌ์ค๋ค.
jar ํ์ผ ํ์ธ
https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-mapreduce-client-core/3.2.3
์ฌ๊ธฐ ๋ค์ด๊ฐ์ maven ๋ด์ฉ ๋์ด์ค๊ธฐ
Wordcount/pom.xml
์ ์ฌ...
alt + F5 update
์ ๋ฐ์ดํธ Alt + F5 ๋๋ฅธ๋ค
WordCount
WordCount.java
WordCountReducer.java
maven ํ์ผ jar ํ์ผ๋ก ๋ฐ๊พธ๊ธฐ
clean ์ด๋ build
install ํ๊ณ success ํ์ธํ๊ธฐ
scp....... ๊ธฐ์ตํด๋์........
jar -tf WordCount-0.1.jar
jar -tf WordCount-0.1.jar
yarn jar ./WordCount-0.1.jar com.gyuone.driver.WordCount sample-input sample-output
hdfs dfs -ls sample-output
๐ jar ํ์ผ ๋ง๋๋ ๋ฐฉ๋ฒ ๐
ใด ๊ธฐ์ตํ๊ธฐ......
jar ํ์ผ ์คํ
hdfs dfs -cat sample-output/part-r-00000
AirPerformance
pom.xml
clean -> update
Alt+F5 Update.. ์ ๋ฐ์ดํธ ํ๊ณ ์ ์ฉํ๊ธฐ
AirlinePerformanceParser.java
package com.gyuone.common;
import org.apache.hadoop.io.Text;
public class AirlinePerformanceParser {
private int year;
private int month;
private int day;
private int arriveDelayTime = 0;
private int departureDelayTime = 0;
private int distance = 0;
private boolean arriveDelayAvailable = true;
private boolean departureDelayAvailable = true;
private boolean distanceAvailable = true;
private String uniqueCarrier;
// "YEAR","MONTH","DAY_OF_MONTH","DAY_OF_WEEK","FL_DATE",
// "UNIQUE_CARRIER","TAIL_NUM","FL_NUM","ORIGIN_AIRPORT_ID","ORIGIN",
// "ORIGIN_STATE_ABR","DEST_AIRPORT_ID","DEST","DEST_STATE_ABR","CRS_DEP_TIME",
// "DEP_TIME","DEP_DELAY","DEP_DELAY_NEW","DEP_DEL15","DEP_DELAY_GROUP",
// "TAXI_OUT","WHEELS_OFF","WHEELS_ON","TAXI_IN","CRS_ARR_TIME",
// "ARR_TIME","ARR_DELAY","ARR_DELAY_NEW","ARR_DEL15","ARR_DELAY_GROUP",
// "CANCELLED","CANCELLATION_CODE","DIVERTED","CRS_ELAPSED_TIME","ACTUAL_ELAPSED_TIME",
// "AIR_TIME","FLIGHTS","DISTANCE","DISTANCE_GROUP","CARRIER_DELAY",
// "WEATHER_DELAY","NAS_DELAY","SECURITY_DELAY","LATE_AIRCRAFT_DELAY",
public AirlinePerformanceParser(Text text) {
try {
String[] columns = text.toString().split(",");
year = Integer.parseInt(columns[0]);
month = Integer.parseInt(columns[1]);
month = Integer.parseInt(columns[1]);
uniqueCarrier = columns[5]; // string ์ด๋ผ ํ์ฑ ์ํจ
if (!columns[16].equals("")) { // DEP_DELAY ์ index๊ฐ 16์ด๋ค.
departureDelayTime = (int)Float.parseFloat(columns[16]); // ๊ฐ์ ํ๋ณํ
} else {
departureDelayAvailable = false;
}
if (!columns[26].equals("")) { // ARR_DELAY์ index๊ฐ 26์ด๋ค.
arriveDelayTime = (int)Float.parseFloat(columns[26]); // ๊ฐ์ ํ๋ณํ
} else {
arriveDelayAvailable = false;
}
if (!columns[37].equals("")) { // DISTANCE์ index๊ฐ 37์ด๋ค.
distance = (int)Float.parseFloat(columns[37]); // ๊ฐ์ ํ๋ณํ
} else {
distanceAvailable = false;
}
} catch (Exception e) {
// TODO: handle exception
}
}
public int getYear() {
return year;
}
public int getMonth() {
return month;
}
public int getDay() {
return day;
}
public int getArriveDelayTime() {
return arriveDelayTime;
}
public int getDepartureDelayTime() {
return departureDelayTime;
}
public int getDistance() {
return distance;
}
public boolean isArriveDelayAvailable() {
return arriveDelayAvailable;
}
public boolean isDistanceAvailable() {
return distanceAvailable;
}
public String getUniqueCarrier() {
return uniqueCarrier;
}
}
DepartureDelayCount.java
getInstance conf ๊ฐ์ฒด์ ์ธ์คํด์ค๊ฐ ์ค์ง 1๊ฐ๋ง ์์ฑ๋๋ ํจํด
DelayCountReducer.java
DepartureDelayCountMapper.java
jar ๋ค์ด๊ฐ ์๋ ๊ฑฐ ํ์ธํ๊ธฐ..
jar ์คํ์ํค๊ธฐ
yarn jar ./AirPerformance-0.1.jar com.gyuone.driver.DepartureDelayCount air-input dep-delay-count
hdfs dfs -ls
hdfs dfs -ls dep-delay-count
hdfs dfs -cat dep-delay-count/part-r-00000
์ด๋ฆ ๋ฐ๊พธ๊ธฐ -cp ์ด์ฉํด์ ... delay-count -> dep-delay-count
hdfs dfs -cp delay-count dep-delay-count
hdfs dfs -ls dep-delay-count
โ ์ข ๋ฃํ๋ ๋ฐฉ๋ฒ โ
jps์์ JobHistoryServer์ ๋ค์ด๊ฐ๋ ์ซ์ = 4767
kill -9 4767
stop-yarn.sh
stop-dfs.sh
jps
ls -l
sudo shutdown -h
๋๊ธ