forked from InMobi/hraven
-
Notifications
You must be signed in to change notification settings - Fork 5
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
*ability to output to sinks other than just hbase (for job conf and history) *added graphite sink and refactored hbase to work as a sink *generic object model and abstraction for output records of JobFileProcessor's mapper instead of emitting Hbase puts *changes no hraven behaviour for hbase output note: this commit does makes hadoop2 support broken. work in progress for JobHistoryFileParserHadoop2
- Loading branch information
Angad Singh
authored and
Angad Singh
committed
May 20, 2014
1 parent
fb60917
commit 0df1646
Showing
50 changed files
with
1,391 additions
and
401 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
66 changes: 66 additions & 0 deletions
66
hraven-core/src/main/java/com/twitter/hraven/HravenRecord.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,66 @@ | ||
package com.twitter.hraven; | ||
|
||
import com.google.protobuf.DescriptorProtos.FieldDescriptorProto.Type; | ||
|
||
/** | ||
* | ||
* @author angad.singh | ||
* | ||
* {@link JobFileTableMapper outputs this as value. It corresponds to the | ||
* Put record which was earlier emitted | ||
* | ||
* @param <K> key type | ||
* @param <V> type of dataValue object to be stored | ||
*/ | ||
|
||
public abstract class HravenRecord<K,V> { | ||
private K key; | ||
private RecordCategory dataCategory; | ||
private RecordDataKey dataKey; | ||
private V dataValue; | ||
private long submitTime; | ||
|
||
public HravenRecord() { | ||
|
||
} | ||
|
||
public K getKey() { | ||
return key; | ||
} | ||
|
||
public void setKey(K key) { | ||
this.key = key; | ||
} | ||
|
||
public RecordCategory getDataCategory() { | ||
return dataCategory; | ||
} | ||
|
||
public void setDataCategory(RecordCategory dataCategory) { | ||
this.dataCategory = dataCategory; | ||
} | ||
|
||
public RecordDataKey getDataKey() { | ||
return dataKey; | ||
} | ||
|
||
public void setDataKey(RecordDataKey dataKey) { | ||
this.dataKey = dataKey; | ||
} | ||
|
||
public V getDataValue() { | ||
return dataValue; | ||
} | ||
|
||
public void setDataValue(V dataValue) { | ||
this.dataValue = dataValue; | ||
} | ||
|
||
public long getSubmitTime() { | ||
return submitTime; | ||
} | ||
|
||
public void setSubmitTime(long submitTime) { | ||
this.submitTime = submitTime; | ||
} | ||
} |
32 changes: 32 additions & 0 deletions
32
hraven-core/src/main/java/com/twitter/hraven/HravenService.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
package com.twitter.hraven; | ||
|
||
/** | ||
* | ||
* @author angad.singh | ||
* | ||
* {@link JobFileTableMapper outputs this as key. It corresponds to the | ||
* Hbase table which was earlier emitted | ||
*/ | ||
|
||
public enum HravenService { | ||
JOB_HISTORY_RAW { | ||
@Override | ||
public HravenRecord getNewRecord() { | ||
return new JobHistoryRawRecord(); | ||
} | ||
}, | ||
JOB_HISTORY { | ||
@Override | ||
public HravenRecord getNewRecord() { | ||
return new JobHistoryRecord(); | ||
} | ||
}, | ||
JOB_HISTORY_TASK { | ||
@Override | ||
public HravenRecord getNewRecord() { | ||
return new JobHistoryRecord(); | ||
} | ||
}; | ||
|
||
public abstract HravenRecord getNewRecord(); | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
106 changes: 106 additions & 0 deletions
106
hraven-core/src/main/java/com/twitter/hraven/JobHistoryMultiRecord.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,106 @@ | ||
package com.twitter.hraven; | ||
|
||
import java.util.HashMap; | ||
import java.util.Iterator; | ||
import java.util.Map; | ||
import java.util.Map.Entry; | ||
|
||
/** | ||
* | ||
* @author angad.singh | ||
* | ||
* Store multiple {@link JobHistoryRecord}s in a 2 level HashMap | ||
* | ||
* Supports iteration to get individual {@link JobHistoryRecord}s | ||
*/ | ||
|
||
public class JobHistoryMultiRecord extends HravenRecord<JobKey, Object> implements | ||
Iterable<JobHistoryRecord> { | ||
|
||
private Map<RecordCategory, Map<RecordDataKey, Object>> valueMap; | ||
|
||
public JobHistoryMultiRecord() { | ||
valueMap = new HashMap<RecordCategory, Map<RecordDataKey, Object>>(); | ||
} | ||
|
||
public JobHistoryMultiRecord(JobKey jobKey) { | ||
this.setKey(jobKey); | ||
valueMap = new HashMap<RecordCategory, Map<RecordDataKey, Object>>(); | ||
} | ||
|
||
public void add(RecordCategory category, RecordDataKey key, Object value) { | ||
if (valueMap.containsKey(category)) { | ||
valueMap.get(category).put(key, value); | ||
} else { | ||
HashMap<RecordDataKey, Object> categoryMap = new HashMap<RecordDataKey, Object>(); | ||
valueMap.put(category, categoryMap); | ||
categoryMap.put(key, value); | ||
} | ||
} | ||
|
||
public void add(HravenRecord record) { | ||
add(record.getDataCategory(), record.getDataKey(), record.getDataValue()); | ||
} | ||
|
||
public Map<RecordCategory, Map<RecordDataKey, Object>> getValueMap() { | ||
return valueMap; | ||
} | ||
|
||
public Object getValue(RecordCategory category, RecordDataKey key) { | ||
return valueMap.containsKey(category) ? valueMap.get(category).get(key) : null; | ||
} | ||
|
||
public int size() { | ||
int size = 0; | ||
for (Entry<RecordCategory, Map<RecordDataKey, Object>> catMap : valueMap.entrySet()) { | ||
size += catMap.getValue().size(); | ||
} | ||
|
||
return size; | ||
} | ||
|
||
/** | ||
* Be able to iterate easily to get individual {@link JobHistoryRecord}s | ||
*/ | ||
|
||
@Override | ||
public Iterator<JobHistoryRecord> iterator() { | ||
|
||
return new Iterator<JobHistoryRecord>() { | ||
|
||
private Iterator<Entry<RecordCategory, Map<RecordDataKey, Object>>> catIterator; | ||
private Iterator<Entry<RecordDataKey, Object>> dataIterator; | ||
Entry<RecordCategory, Map<RecordDataKey, Object>> nextCat; | ||
Entry<RecordDataKey, Object> nextData; | ||
|
||
{ | ||
catIterator = valueMap.entrySet().iterator(); | ||
nextCat = catIterator.next(); | ||
dataIterator = nextCat.getValue().entrySet().iterator(); | ||
} | ||
|
||
@Override | ||
public boolean hasNext() { | ||
return dataIterator.hasNext() || catIterator.hasNext(); | ||
} | ||
|
||
@Override | ||
public JobHistoryRecord next() { | ||
if (!dataIterator.hasNext()) { | ||
nextCat = catIterator.next(); | ||
dataIterator = nextCat.getValue().entrySet().iterator(); | ||
} | ||
|
||
nextData = dataIterator.next(); | ||
|
||
return new JobHistoryRecord(nextCat.getKey(), getKey(), nextData.getKey(), | ||
nextData.getValue(), getSubmitTime()); | ||
} | ||
|
||
@Override | ||
public void remove() { | ||
} | ||
|
||
}; | ||
} | ||
} |
26 changes: 26 additions & 0 deletions
26
hraven-core/src/main/java/com/twitter/hraven/JobHistoryRawRecord.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,26 @@ | ||
package com.twitter.hraven; | ||
|
||
public class JobHistoryRawRecord extends HravenRecord<String, Object> { | ||
|
||
public JobHistoryRawRecord(RecordCategory dataCategory, String key, RecordDataKey dataKey, | ||
Object dataValue) { | ||
this.setKey(key); | ||
this.setDataCategory(dataCategory); | ||
this.setDataKey(dataKey); | ||
this.setDataValue(dataValue); | ||
} | ||
|
||
public JobHistoryRawRecord() { | ||
|
||
} | ||
|
||
public JobHistoryRawRecord(String taskKey) { | ||
this.setKey(taskKey); | ||
} | ||
|
||
public void set(RecordCategory category, RecordDataKey key, String value) { | ||
this.setDataCategory(category); | ||
this.setDataKey(key); | ||
this.setDataValue(value); | ||
} | ||
} |
40 changes: 40 additions & 0 deletions
40
hraven-core/src/main/java/com/twitter/hraven/JobHistoryRecord.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,40 @@ | ||
package com.twitter.hraven; | ||
|
||
/** | ||
* | ||
* @author angad.singh | ||
* | ||
* Abstraction of a record to be stored in the {@link HravenService#JOB_HISTORY} service. | ||
* Was earlier directly written as an Hbase put | ||
*/ | ||
|
||
public class JobHistoryRecord extends HravenRecord<JobKey, Object> { | ||
|
||
public JobHistoryRecord(RecordCategory dataCategory, JobKey key, RecordDataKey dataKey, | ||
Object dataValue) { | ||
this.setKey(key); | ||
this.setDataCategory(dataCategory); | ||
this.setDataKey(dataKey); | ||
this.setDataValue(dataValue); | ||
} | ||
|
||
public JobHistoryRecord(RecordCategory dataCategory, JobKey key, RecordDataKey dataKey, | ||
Object dataValue, long submitTime) { | ||
this(dataCategory, key, dataKey, dataValue); | ||
setSubmitTime(submitTime); | ||
} | ||
|
||
public JobHistoryRecord() { | ||
|
||
} | ||
|
||
public JobHistoryRecord(JobKey jobKey) { | ||
this.setKey(jobKey); | ||
} | ||
|
||
public void set(RecordCategory category, RecordDataKey key, String value) { | ||
this.setDataCategory(category); | ||
this.setDataKey(key); | ||
this.setDataValue(value); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
6 changes: 6 additions & 0 deletions
6
hraven-core/src/main/java/com/twitter/hraven/RecordCategory.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,6 @@ | ||
package com.twitter.hraven; | ||
|
||
public enum RecordCategory { | ||
HISTORY_COUNTER, HISTORY_META, HISTORY_TASK_COUNTER, HISTORY_TASK_META, CONF, CONF_META, META, | ||
INFERRED | ||
} |
37 changes: 37 additions & 0 deletions
37
hraven-core/src/main/java/com/twitter/hraven/RecordDataKey.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,37 @@ | ||
package com.twitter.hraven; | ||
|
||
import java.util.ArrayList; | ||
import java.util.Arrays; | ||
import java.util.List; | ||
|
||
import org.apache.hadoop.util.StringUtils; | ||
|
||
public class RecordDataKey { | ||
private List<String> components; | ||
|
||
public RecordDataKey(String[] components) { | ||
this.components = Arrays.asList(components); | ||
} | ||
|
||
public RecordDataKey(String firstComponent) { | ||
this.components = new ArrayList<String>(); | ||
this.components.add(firstComponent); | ||
} | ||
|
||
public void add(String component) { | ||
this.components.add(component); | ||
} | ||
|
||
public String get(int index) { | ||
return components.get(index); | ||
} | ||
|
||
public List<String> getComponents() { | ||
return components; | ||
} | ||
|
||
@Override | ||
public String toString() { | ||
return StringUtils.join(Constants.SEP_CHAR, components); | ||
} | ||
} |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.