Skip to content

Commit

Permalink
Fixed resolving classes
Browse files Browse the repository at this point in the history
  • Loading branch information
Michael Spector committed Mar 29, 2015
1 parent d646fdc commit 2258d6b
Show file tree
Hide file tree
Showing 8 changed files with 311 additions and 452 deletions.
Binary file added examples/addressbook/addressbook.gpb
View file
Binary file not shown.
580 changes: 186 additions & 394 deletions examples/addressbook/decode_addressbook.ktr
View file

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions examples/addressbook/java/.classpath
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="src/main/java" including="**/*.java"/>
<classpathentry kind="output" path="target/classes"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry kind="var" path="M2_REPO/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0.jar" sourcepath="M2_REPO/com/google/protobuf/protobuf-java/2.5.0/protobuf-java-2.5.0-sources.jar"/>
</classpath>
14 changes: 14 additions & 0 deletions examples/addressbook/java/.project
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>protobuf-addressbook-example</name>
<comment>NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse.</comment>
<projects/>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
57 changes: 30 additions & 27 deletions src/main/java/com/ruckuswireless/pentaho/protobuf/decode/ProtobufDecodeDialog.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,8 +293,7 @@ public void widgetSelected(SelectionEvent e) {
FileDialog dialog = new FileDialog(shell, SWT.OPEN);
dialog.setFilterExtensions(new String[] { "*.jar", "*" });
if (wClasspath.getText() != null) {
String fname = transMeta.environmentSubstitute(wClasspath
.getText());
String fname = wClasspath.getText(); // transMeta.environmentSubstitute(wClasspath.getText());
dialog.setFileName(fname);
}

Expand Down Expand Up @@ -382,10 +381,9 @@ private void cancel() {
* Copy information from the dialog fields to the meta-data input
*/
private void setData(ProtobufDecodeMeta consumerMeta) {
consumerMeta.setInputField(transMeta.environmentSubstitute(wInputField
.getText()));
consumerMeta.setClasspath(transMeta.environmentSubstitute(
wClasspath.getText().trim()).split(File.pathSeparator));
consumerMeta.setInputField(wInputField.getText());
consumerMeta.setClasspath(wClasspath.getText().trim()
.split(File.pathSeparator));
consumerMeta.setRootClass(wRootClass.getText());

int nrNonEmptyFields = wFields.nrNonEmpty();
Expand Down Expand Up @@ -417,28 +415,33 @@ private void ok() {

private void detectFields() {
try {
ProtobufDecoder protobufDecoder = new ProtobufDecoder(transMeta
.environmentSubstitute(wClasspath.getText().trim()).split(
File.pathSeparator), wRootClass.getText(), null);
Map<String, Class<?>> fields = protobufDecoder.guessFields();
RowMeta rowMeta = new RowMeta();
for (Entry<String, Class<?>> e : fields.entrySet()) {
String fieldPath = e.getKey();
int i = fieldPath.lastIndexOf('.');
String fieldName = i != -1 ? fieldPath.substring(i + 1)
: fieldPath;
rowMeta.addValueMeta(new FieldMeta(fieldName, fieldPath,
KettleTypesConverter.javaToKettleType(e.getValue())));
ProtobufDecoder protobufDecoder = new ProtobufDecoder(
transMeta.environmentSubstitute(wClasspath.getText().trim()
.split(File.pathSeparator)), wRootClass.getText(),
null);
try {
Map<String, Class<?>> fields = protobufDecoder.guessFields();
RowMeta rowMeta = new RowMeta();
for (Entry<String, Class<?>> e : fields.entrySet()) {
String fieldPath = e.getKey();
int i = fieldPath.lastIndexOf('.');
String fieldName = i != -1 ? fieldPath.substring(i + 1)
: fieldPath;
rowMeta.addValueMeta(new FieldMeta(fieldName, fieldPath,
KettleTypesConverter.javaToKettleType(e.getValue())));
}
BaseStepDialog.getFieldsFromPrevious(rowMeta, wFields, 1,
new int[] { 1 }, new int[] { 3 }, -1, -1,
new TableItemInsertListener() {
public boolean tableItemInserted(
TableItem tableItem, ValueMetaInterface v) {
tableItem.setText(2, ((FieldMeta) v).path);
return true;
}
});
} finally {
protobufDecoder.dispose();
}
BaseStepDialog.getFieldsFromPrevious(rowMeta, wFields, 1,
new int[] { 1 }, new int[] { 3 }, -1, -1,
new TableItemInsertListener() {
public boolean tableItemInserted(TableItem tableItem,
ValueMetaInterface v) {
tableItem.setText(2, ((FieldMeta) v).path);
return true;
}
});
} catch (ProtobufDecoderException e) {
new ErrorDialog(
shell,
Expand Down
71 changes: 49 additions & 22 deletions src/main/java/com/ruckuswireless/pentaho/protobuf/decode/ProtobufDecodeStep.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@
import org.pentaho.di.core.Const;
import org.pentaho.di.core.exception.KettleException;
import org.pentaho.di.core.row.RowDataUtil;
import org.pentaho.di.core.row.RowMetaInterface;
import org.pentaho.di.trans.Trans;
import org.pentaho.di.trans.TransMeta;
import org.pentaho.di.trans.step.BaseStep;
Expand All @@ -23,8 +22,9 @@
*/
public class ProtobufDecodeStep extends BaseStep implements StepInterface {

public ProtobufDecodeStep(StepMeta stepMeta, StepDataInterface stepDataInterface, int copyNr, TransMeta transMeta,
Trans trans) {
public ProtobufDecodeStep(StepMeta stepMeta,
StepDataInterface stepDataInterface, int copyNr,
TransMeta transMeta, Trans trans) {
super(stepMeta, stepDataInterface, copyNr, transMeta, trans);
}

Expand All @@ -34,15 +34,34 @@ public boolean init(StepMetaInterface smi, StepDataInterface sdi) {
ProtobufDecodeMeta meta = (ProtobufDecodeMeta) smi;
ProtobufDecodeData data = (ProtobufDecodeData) sdi;
try {
data.decoder = new ProtobufDecoder(meta.getClasspath(), meta.getRootClass(), meta.getFields());
data.decoder = new ProtobufDecoder(
environmentSubstitute(meta.getClasspath()),
meta.getRootClass(), meta.getFields());
} catch (ProtobufDecoderException e) {
logError(Messages.getString("ProtobufDecodeStep.Init.Error", getStepname()), e);
logError(Messages.getString("ProtobufDecodeStep.Dispose.Error",
getStepname()), e);
return false;
}
return true;
}

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
public void dispose(StepMetaInterface smi, StepDataInterface sdi) {

ProtobufDecodeData data = (ProtobufDecodeData) sdi;
if (data.decoder != null) {
try {
data.decoder.dispose();
} catch (ProtobufDecoderException e) {
logError(Messages.getString("ProtobufDecodeStep.Init.Error",
getStepname()), e);
}
data.decoder = null;
}
super.dispose(smi, sdi);
}

public boolean processRow(StepMetaInterface smi, StepDataInterface sdi)
throws KettleException {
Object[] r = getRow();
if (r == null) {
setOutputDone();
Expand All @@ -52,66 +71,74 @@ public boolean processRow(StepMetaInterface smi, StepDataInterface sdi) throws K
ProtobufDecodeMeta meta = (ProtobufDecodeMeta) smi;
ProtobufDecodeData data = (ProtobufDecodeData) sdi;

RowMetaInterface inputRowMeta = getInputRowMeta();

if (first) {
first = false;
data.outputRowMeta = inputRowMeta.clone();
data.outputRowMeta = getInputRowMeta().clone();
meta.getFields(data.outputRowMeta, getStepname(), null, null, this);

String inputField = environmentSubstitute(meta.getInputField());

int numErrors = 0;
if (Const.isEmpty(inputField)) {
logError(Messages.getString("ProtobufDecodeStep.Log.FieldNameIsNull")); //$NON-NLS-1$
logError(Messages
.getString("ProtobufDecodeStep.Log.FieldNameIsNull")); //$NON-NLS-1$
numErrors++;
}
data.inputFieldNr = inputRowMeta.indexOfValue(inputField);
data.inputFieldNr = getInputRowMeta().indexOfValue(inputField);
if (data.inputFieldNr < 0) {
logError(Messages.getString("ProtobufDecodeStep.Log.CouldntFindField", inputField)); //$NON-NLS-1$
logError(Messages.getString(
"ProtobufDecodeStep.Log.CouldntFindField", inputField)); //$NON-NLS-1$
numErrors++;
}
if (!inputRowMeta.getValueMeta(data.inputFieldNr).isBinary()) {
logError(Messages.getString("ProtobufDecodeStep.Log.FieldNotValid", inputField)); //$NON-NLS-1$
if (!getInputRowMeta().getValueMeta(data.inputFieldNr).isBinary()) {
logError(Messages.getString(
"ProtobufDecodeStep.Log.FieldNotValid", inputField)); //$NON-NLS-1$
numErrors++;
}
if (numErrors > 0) {
setErrors(numErrors);
stopAll();
return false;
}
data.inputFieldMeta = inputRowMeta.getValueMeta(data.inputFieldNr);
data.inputFieldMeta = getInputRowMeta().getValueMeta(
data.inputFieldNr);
}

try {
byte[] message = data.inputFieldMeta.getBinary(r[data.inputFieldNr]);
byte[] message = data.inputFieldMeta
.getBinary(r[data.inputFieldNr]);
try {
List<Object[]> decodedData = data.decoder.decode(message);
for (Object[] d : decodedData) {
r = RowDataUtil.addRowData(r, inputRowMeta.size(), d);
r = RowDataUtil.addRowData(r, getInputRowMeta().size(), d);
putRow(data.outputRowMeta, r);
if (isRowLevel()) {
logRowlevel(Messages.getString("ProtobufDecodeStep.Log.OutputRow",
Long.toString(getLinesWritten()), data.outputRowMeta.getString(r)));
logRowlevel(Messages.getString(
"ProtobufDecodeStep.Log.OutputRow",
Long.toString(getLinesWritten()),
data.outputRowMeta.getString(r)));
}
}
} catch (ProtobufDecoderException e) {
throw new KettleException(e);
}
} catch (KettleException e) {
if (!getStepMeta().isDoingErrorHandling()) {
logError(Messages.getString("ProtobufDecodeStep.ErrorInStepRunning", e.getMessage()));
logError(Messages
.getString("ProtobufDecodeStep.ErrorInStepRunning",
e.getMessage()));
setErrors(1);
stopAll();
setOutputDone();
return false;
}
putError(inputRowMeta, r, 1, e.toString(), null, getStepname());
putError(getInputRowMeta(), r, 1, e.toString(), null, getStepname());
}
return true;
}

public void stopRunning(StepMetaInterface smi, StepDataInterface sdi) throws KettleException {
public void stopRunning(StepMetaInterface smi, StepDataInterface sdi)
throws KettleException {

ProtobufDecodeData data = (ProtobufDecodeData) sdi;
data.canceled = true;
Expand Down
33 changes: 24 additions & 9 deletions src/main/java/com/ruckuswireless/pentaho/protobuf/decode/ProtobufDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
public class ProtobufDecoder {

private final Object EMPTY = new Object();
private URLClassLoader classLoader;
private Class<?> rootClass;
private Method rootParseFromMethod;
private LinkedHashMap<String, Integer> paths;
Expand All @@ -36,13 +37,17 @@ public class ProtobufDecoder {
public ProtobufDecoder(String[] classpath, String rootClass,
FieldDefinition[] fields) throws ProtobufDecoderException {

URLClassLoader classLoader;
try {
URL[] url = new URL[classpath.length];
for (int i = 0; i < classpath.length; ++i) {
url[i] = new File(classpath[i]).toURI().toURL();
String file = classpath[i];
if (file.startsWith("file://")) {
file = file.substring(7);
}
url[i] = new File(file).toURI().toURL();
}
classLoader = new URLClassLoader(url, getClass().getClassLoader());
this.classLoader = new URLClassLoader(url, getClass()
.getClassLoader());
} catch (MalformedURLException e) {
throw new ProtobufDecoderException(e);
}
Expand All @@ -51,12 +56,6 @@ public ProtobufDecoder(String[] classpath, String rootClass,
this.rootClass = classLoader.loadClass(rootClass);
} catch (ClassNotFoundException e) {
throw new ProtobufDecoderException("Can't find root class", e);
} finally {
try {
classLoader.close();
} catch (IOException e) {
throw new ProtobufDecoderException(e);
}
}

try {
Expand All @@ -78,6 +77,22 @@ public ProtobufDecoder(String[] classpath, String rootClass,
}
}

/**
* Disposes the decoder
*
* @throws ProtobufDecoderException
*/
public void dispose() throws ProtobufDecoderException {
if (classLoader != null) {
try {
classLoader.close();
} catch (IOException e) {
throw new ProtobufDecoderException(e);
}
classLoader = null;
}
}

/**
* Decodes message, and returns denormalized rows for the object fields
* exactly in the order they appear in <code>fields</code> parameter.
Expand Down
1 change: 1 addition & 0 deletions ...n/resources/com/ruckuswireless/pentaho/protobuf/decode/messages/messages_en_US.properties
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
ProtobufDecodeStep.Init.Error=Error initializing step\: {0}
ProtobufDecodeStep.Dispose.Error=Error disposing step\: {0}
ProtobufDecodeStep.Log.OutputRow=Outputting row {0} \: {1}
ProtobufDecodeStep.Log.FieldNameIsNull=Input field name not specified\!
ProtobufDecodeStep.Log.CouldntFindField=Couldn''t find field ''{0}'' in input stream\!
Expand Down

0 comments on commit 2258d6b

Please sign in to comment.