diff --git a/.gitignore b/.gitignore index 4451fb7..40974c9 100644 --- a/.gitignore +++ b/.gitignore @@ -1,10 +1,17 @@ -/target -/bin -/.classpath -/.settings -/.project +.eunit +deps +*.o +*.beam +*.plt +erl_crash.dump +ebin +rel/example_project +.concrete/DEV_MODE +.rebar *~ - -*.iml -.idea -/conf/** \ No newline at end of file +src/cf_scan.erl +src/cf_parse.erl +src/cf_prescan.erl +doc +_build +rebar.lock diff --git a/.travis.yml b/.travis.yml index f1b285a..925969f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ -language: java -jdk: - - oraclejdk8 - - oraclejdk7 - - openjdk7 - +language: erlang +otp_release: +# - 18.3 + - 18.2.1 + - 18.2 + - 18.1 + - 18.0 diff --git a/CITATION b/CITATION new file mode 100644 index 0000000..c7a9c3d --- /dev/null +++ b/CITATION @@ -0,0 +1,15 @@ +@InProceedings{Brandt2015, + Title = {Cuneiform: A Functional Language for Large Scale Scientific Data Analysis}, + Author = {Brandt, J{\"o}rgen and Bux, Marc and Leser, Ulf}, + Booktitle = {Proceedings of the Workshops of the EDBT/ICDT}, + Year = {2015}, + + Address = {Brussels, Belgium}, + Month = {March}, + Pages = {17--26}, + Volume = {1330}, + + Abstract = {The need to analyze massive scientific data sets on the one hand and the availability of distributed compute resources with an increasing number of CPU cores on the other hand have promoted the development of a variety of languages and systems for parallel, distributed data analysis. Among them are data-parallel query languages such as Pig Latin or Spark as well as scientific workflow languages such as Swift or Pegasus DAX. While data-parallel query languages focus on the exploitation of data parallelism, scientific workflow languages focus on the integration of external tools and libraries. However, a language that combines easy integration of arbitrary tools, treated as black boxes, with the ability to fully exploit data parallelism does not exist yet. Here, we present Cuneiform, a novel language for large-scale scientific data analysis. We highlight its functionality with respect to a set of desirable features for such languages, introduce its syntax and semantics by example, and show its flexibility and conciseness with use cases, including a complex real-life workflow from the area of genome research. Cuneiform scripts are executed dynamically on the workflow execution platform Hi-WAY which is based on Hadoop YARN. The language Cuneiform, including tool support for programming, workflow visualization, debugging, logging, and provenance-tracing, and the parallel execution engine Hi-WAY are fully implemented.}, + Doi = {10.13140/RG.2.1.3547.6561}, + Url = {http://ceur-ws.org/Vol-1330/paper-03.pdf} +} diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..c8b2e97 --- /dev/null +++ b/Makefile @@ -0,0 +1,19 @@ +PWD=$(shell pwd) + +all: compile + +install: compile + ln -sf $(PWD)/_build/default/bin/cuneiform /usr/local/bin/cuneiform + +compile: + rebar3 escriptize + +dev: + rebar3 do escriptize, eunit, dialyzer, cover, edoc + +clean: + rm -rf .rebar + rm -rf _build + rm -rf doc + rm -f rebar.lock + diff --git a/cuneiform-addons/.gitignore b/cuneiform-addons/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-addons/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-addons/cuneiform-cfide/.gitignore b/cuneiform-addons/cuneiform-cfide/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-addons/cuneiform-cfide/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-addons/cuneiform-cfide/pom.xml b/cuneiform-addons/cuneiform-cfide/pom.xml deleted file mode 100644 index a5fb9e7..0000000 --- a/cuneiform-addons/cuneiform-cfide/pom.xml +++ /dev/null @@ -1,55 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform-addons - 2.0.4-RELEASE - - cuneiform-cfide - cuneiform-cfide - - - - de.hu-berlin.wbi.cuneiform - cuneiform-core - ${project.version} - - - de.hu-berlin.wbi.cuneiform - cuneiform-starlinger - ${project.version} - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - - \ No newline at end of file diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/DarkStyleConf.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/DarkStyleConf.java deleted file mode 100644 index 519a807..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/DarkStyleConf.java +++ /dev/null @@ -1,24 +0,0 @@ -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.Color; - -public class DarkStyleConf extends StyleConf { - - public DarkStyleConf() { - setCommentStyle( SimpleStyle.createPlainStyle( new Color( 100, 200, 255 ) ) ); - setCallStyle( SimpleStyle.createItStyle() ); - setApplyStyle( SimpleStyle.createUlStyle() ); - setVarnameStyle( SimpleStyle.createPlainStyle( new Color( 100, 180, 140 ) ) ); - setStatStyle( SimpleStyle.createPlainStyle( Color.WHITE ) ); - setKeywordStyle( SimpleStyle.createBfStyle() ); - setDataStyle( SimpleStyle.createPlainStyle( new Color( 255, 255, 150 ) ) ); - setTypeStyle( SimpleStyle.createPlainStyle( new Color( 200, 255, 200 ) ) ); - setForeignStyle( SimpleStyle.createPlainStyle( Color.LIGHT_GRAY ) ); - } - - @Override - public Color getBackgroundColor() { - return new Color( 0, 10, 30 ); - } - -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/EditPanel.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/EditPanel.java deleted file mode 100644 index 83cfcd7..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/EditPanel.java +++ /dev/null @@ -1,216 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.BorderLayout; -import java.awt.Component; -import java.awt.event.KeyEvent; -import java.awt.event.KeyListener; -import java.io.File; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import javax.swing.BorderFactory; -import javax.swing.Box; -import javax.swing.BoxLayout; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JSplitPane; -import javax.swing.JTable; -import javax.swing.event.CaretEvent; -import javax.swing.event.CaretListener; -import javax.swing.text.Element; - -public class EditPanel extends SyntaxPanel implements KeyListener, CaretListener { - - private static final long serialVersionUID = -7538677478512722934L; - private static final String TITLE_UNSAVED = "Unsaved*"; - - private JLabel posLabel; - private ErrorTableModel errorTableModel; - private File file; - private String originalContent; - private MainPanel mainPanel; - - public EditPanel( MainPanel mainPanel ) { - this( mainPanel, null, "" ); - } - - public EditPanel( MainPanel mainPanel, File file, String content ) { - - JTable errorTable; - JPanel posPanel; - StaticPanel staticPanel; - ExecutorService es; - JScrollPane errorPane; - JSplitPane editErrorSplitPane, editStaticPane; - Component original; - - setMainPanel( mainPanel ); - setFile( file, content ); - setText( content ); - - original = getComponent( 0 ); - remove( original ); - - addKeyListener( this ); - addCaretListener( this ); - setEditable( true ); - - // error table - errorTableModel = new ErrorTableModel(); - errorTable = new JTable( errorTableModel ); - errorTable.getColumnModel().getColumn( 0 ).setMaxWidth( 60 ); - errorTable.getColumnModel().getColumn( 1 ).setMaxWidth( 60 ); - errorTable.getColumnModel().getColumn( 3 ).setMaxWidth( 60 ); - - // caret position label - posLabel = new JLabel( "1 : 1" ); - - posPanel = new JPanel(); - posPanel.setLayout( new BoxLayout( posPanel, BoxLayout.X_AXIS ) ); - posPanel.add( Box.createHorizontalGlue() ); - posPanel.add( posLabel ); - - add( posPanel, BorderLayout.SOUTH ); - - staticPanel = new StaticPanel( this, errorTableModel ); - es = Executors.newSingleThreadExecutor(); - es.submit( staticPanel ); - es.shutdown(); - - editStaticPane = new JSplitPane( - JSplitPane.HORIZONTAL_SPLIT, staticPanel, original ); - editStaticPane.setDividerLocation( 450 ); - editStaticPane.setResizeWeight( 0 ); - editStaticPane.setOneTouchExpandable( true ); - - errorPane = new JScrollPane( errorTable ); - errorPane.setBorder( BorderFactory.createTitledBorder( "Errors" ) ); - editErrorSplitPane = new JSplitPane( - JSplitPane.VERTICAL_SPLIT, - errorPane, - editStaticPane ); - editErrorSplitPane.setDividerLocation( 100 ); - editErrorSplitPane.setResizeWeight( 0 ); - editErrorSplitPane.setOneTouchExpandable( true ); - - add( editErrorSplitPane, BorderLayout.CENTER ); - } - - @Override - public void caretUpdate( CaretEvent arg0 ) { - - int line, pos, col; - Element root; - - if( posLabel == null ) - return; - - pos = getCaretPosition(); - root = getDocument().getDefaultRootElement(); - - for( line = 0; line < root.getElementCount(); line++ ) - if( root.getElement( line ).getStartOffset() > pos ) - break; - - col = pos-root.getElement( line-1 ).getStartOffset()+1; - - posLabel.setText( line+" : "+col ); - } - - public ErrorTableModel getErrorTableModel() { - return errorTableModel; - } - - public File getFile() { - return file; - } - - public String getTitle() { - - String title; - - if( file == null ) - return TITLE_UNSAVED; - - title = file.getName(); - - if( hasChanged() ) - title += "*"; - - return title; - } - - public boolean hasChanged() { - return !originalContent.equals( getText() ); - } - - public boolean hasFile() { - return file != null; - } - - @Override - public void keyPressed( KeyEvent arg0 ) {} - - @Override - public void keyReleased( KeyEvent arg0 ) { - SyntaxListener.process( getEditPane(), sc ); - mainPanel.updateSelectedEditPanelTitle(); - } - - @Override - public void keyTyped( KeyEvent arg0 ) {} - - public void revert() { - setText( originalContent ); - } - - public void setFile( File file, String content ) { - - if( content == null ) - throw new NullPointerException( "Original content must not be null." ); - - this.file = file; - originalContent = content; - } - - public void setMainPanel( MainPanel mainPanel ) { - - if( mainPanel == null ) - throw new NullPointerException( "Main panel must not be null." ); - - this.mainPanel = mainPanel; - } -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/ErrorTableModel.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/ErrorTableModel.java deleted file mode 100644 index 126cd42..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/ErrorTableModel.java +++ /dev/null @@ -1,105 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.util.LinkedList; -import java.util.List; - -import javax.swing.table.AbstractTableModel; - -import de.huberlin.wbi.cuneiform.core.preprocess.ParseException; - - -public class ErrorTableModel extends AbstractTableModel { - - private static final long serialVersionUID = -5752707318286728959L; - private List errorList; - - public ErrorTableModel() { - errorList = new LinkedList<>(); - } - - public void clear() { - errorList.clear(); - } - - public void add( ParseException entry ) { - - if( entry == null ) - throw new NullPointerException( "Error entry must not be null." ); - - errorList.add( entry ); - } - - @Override - public int getColumnCount() { - return 5; - } - - @Override - public Object getValueAt( int rowIndex, int columnIndex ) { - - ParseException entry; - - entry = errorList.get( rowIndex ); - - switch( columnIndex ) { - case 0 : return entry.hasLine() ? entry.getLine() : ""; - case 1 : return entry.hasCharPositionInLine() ? entry.getCharPositionInLine() : ""; - case 2 : return entry.hasNear() ? entry.getNear() : ""; - case 3 : return "Error"; - case 4 : return entry.getMessage(); - default : - throw new UnsupportedOperationException( - "Invalid column." ); - } - } - - @Override - public int getRowCount() { - return errorList.size(); - } - - @Override - public String getColumnName( int columnIndex ) { - - switch( columnIndex ) { - case 0 : return "Line"; - case 1 : return "Column"; - case 2 : return "Near"; - case 3 : return "Level"; - case 4 : return "Message"; - default : throw new UnsupportedOperationException( "Invalid column." ); - } - } -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/LightStyleConf.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/LightStyleConf.java deleted file mode 100644 index 36b3b5a..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/LightStyleConf.java +++ /dev/null @@ -1,23 +0,0 @@ -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.Color; - -public class LightStyleConf extends StyleConf { - - public LightStyleConf() { - setCommentStyle( SimpleStyle.createPlainStyle( new Color( 0, 100, 155 ) ) ); - setCallStyle( SimpleStyle.createItStyle() ); - setApplyStyle( SimpleStyle.createUlStyle() ); - setVarnameStyle( SimpleStyle.createPlainStyle( new Color( 0, 80, 40 ) ) ); - setStatStyle( SimpleStyle.createPlainStyle( Color.BLACK ) ); - setKeywordStyle( SimpleStyle.createBfStyle() ); - setDataStyle( SimpleStyle.createPlainStyle( new Color( 100, 100, 0 ) ) ); - setTypeStyle( SimpleStyle.createPlainStyle( new Color( 70, 125, 70 ) ) ); - setForeignStyle( SimpleStyle.createPlainStyle( Color.DARK_GRAY ) ); - } - - @Override - public Color getBackgroundColor() { - return Color.WHITE; - } -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/MainPanel.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/MainPanel.java deleted file mode 100644 index bc577fc..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/MainPanel.java +++ /dev/null @@ -1,466 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.BorderLayout; -import java.awt.Component; -import java.awt.Event; -import java.awt.event.ActionEvent; -import java.awt.event.ActionListener; -import java.awt.event.KeyEvent; -import java.awt.event.WindowEvent; -import java.awt.event.WindowListener; -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.File; -import java.io.FileReader; -import java.io.FileWriter; -import java.io.IOException; - -import javax.swing.JFileChooser; -import javax.swing.JFrame; -import javax.swing.JMenu; -import javax.swing.JMenuBar; -import javax.swing.JMenuItem; -import javax.swing.JOptionPane; -import javax.swing.JPanel; -import javax.swing.JTabbedPane; -import javax.swing.KeyStroke; - -import de.huberlin.wbi.cuneiform.core.repl.BaseRepl; - -public class MainPanel extends JPanel implements ActionListener, WindowListener { - - private static final long serialVersionUID = -8425834761771677470L; - - private static final String LABEL_NEW = "New"; - private static final String LABEL_OPEN = "Open ..."; - private static final String LABEL_CLOSE = "Close"; - private static final String LABEL_SAVE = "Save"; - private static final String LABEL_SAVEAS = "Save as ..."; - private static final String LABEL_QUIT = "Quit"; - private static final String LABEL_ABOUT = "About ..."; - private static final String LABEL_REVERT = "Revert"; - - private JTabbedPane editTabbedPane; - private JFileChooser fc; - private JFrame parent; - private JMenuItem saveItem, closeItem, saveAsItem, revertItem; - - public MainPanel( JFrame parentFrame ) { - - setParent( parentFrame ); - setLayout( new BorderLayout() ); - - fc = new JFileChooser(); - editTabbedPane = new JTabbedPane(); - - - add( editTabbedPane ); - } - - @Override - public void actionPerformed( ActionEvent arg0 ) { - - int returnVal; - File f; - String ac; - EditPanel editPanel; - - - ac = arg0.getActionCommand(); - - if( ac.equals( LABEL_REVERT ) ) { - - editPanel = ( EditPanel )editTabbedPane.getSelectedComponent(); - - if( !editPanel.hasChanged() ) - return; - - returnVal = JOptionPane.showConfirmDialog( parent, "Unsaved changes will be lost.\nAre you sure?", "Revert "+editPanel.getTitle(), JOptionPane.YES_NO_OPTION ); - - if( returnVal == JOptionPane.NO_OPTION ) - return; - - editPanel.revert(); - updateSelectedEditPanelTitle(); - - return; - } - - if( ac.equals( LABEL_ABOUT ) ) { - - JOptionPane.showMessageDialog( parent, "Cuneiform Editor\nversion "+BaseRepl.LABEL_VERSION+" build "+BaseRepl.LABEL_BUILD+"\nJörgen Brandt", "About Cuneiform Editor", JOptionPane.INFORMATION_MESSAGE ); - - return; - } - - if( ac.equals( LABEL_NEW ) ) { - - editPanel = new EditPanel( this ); - editTabbedPane.add( editPanel.getTitle(), editPanel ); - editTabbedPane.setSelectedComponent( editPanel ); - updateMenu(); - - return; - } - - if( ac.equals( LABEL_OPEN ) ) { - - returnVal = fc.showOpenDialog( parent ); - - if( returnVal == JFileChooser.APPROVE_OPTION ) { - - f = fc.getSelectedFile(); - open( f ); - } - - return; - } - - if( ac.equals( LABEL_CLOSE ) ) { - - editPanel = ( EditPanel )editTabbedPane.getSelectedComponent(); - - if( editPanel.hasChanged() ) { - - - if( editPanel.hasFile() ) { - - returnVal = JOptionPane.showConfirmDialog( parent, - "Save changes in "+editPanel.getFile().getName()+"?", - "Save changes", - JOptionPane.YES_NO_CANCEL_OPTION ); - - if( returnVal == JOptionPane.OK_OPTION ) - saveAction( editPanel.getFile() ); - } - else { - - returnVal = JOptionPane.showConfirmDialog( parent, - "Save changes in unsaved file?", - "Save changes", - JOptionPane.YES_NO_CANCEL_OPTION ); - - if( returnVal == JOptionPane.OK_OPTION ) { - - returnVal = saveAsAction(); - - if( returnVal == JFileChooser.CANCEL_OPTION ) - return; - } - else - if( returnVal == JOptionPane.CANCEL_OPTION ) - return; - - - } - - if( returnVal == JOptionPane.CANCEL_OPTION ) - return; - } - - - editTabbedPane.remove( editTabbedPane.getSelectedComponent() ); - updateMenu(); - - return; - } - - if( ac.equals( LABEL_SAVE ) ) { - - editPanel = ( EditPanel )editTabbedPane.getSelectedComponent(); - - if( !editPanel.hasChanged() ) - return; - - if( editPanel.hasFile() ) { - - saveAction( editPanel.getFile() ); - return; - } - - saveAsAction(); - - return; - } - - if( ac.equals( LABEL_SAVEAS ) ) { - - saveAsAction(); - return; - } - - if( ac.equals( LABEL_QUIT ) ) { - quitAction(); - return; - } - - throw new RuntimeException( "Action not recognized: '"+arg0.getActionCommand()+"'." ); - } - - public JMenuBar getMenuBar() { - - JMenuBar menuBar; - JMenu menu; - JMenuItem menuItem; - - menuBar = new JMenuBar(); - - menu = new JMenu( "File" ); - menuBar.add( menu ); - - menuItem = new JMenuItem( LABEL_NEW ); - menuItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_N, Event.CTRL_MASK ) ); - menuItem.addActionListener( this ); - menu.add( menuItem ); - - menuItem = new JMenuItem( LABEL_OPEN ); - menuItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_O, Event.CTRL_MASK ) ); - menuItem.addActionListener( this ); - menu.add( menuItem ); - - menu.addSeparator(); - - saveItem = new JMenuItem( LABEL_SAVE ); - saveItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_S, Event.CTRL_MASK ) ); - saveItem.setEnabled( false ); - saveItem.addActionListener( this ); - menu.add( saveItem ); - - saveAsItem = new JMenuItem( LABEL_SAVEAS ); - saveAsItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_S, Event.CTRL_MASK | Event.SHIFT_MASK ) ); - saveAsItem.setEnabled( false ); - saveAsItem.addActionListener( this ); - menu.add( saveAsItem ); - - revertItem = new JMenuItem( LABEL_REVERT ); - revertItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_R, Event.CTRL_MASK ) ); - revertItem.setEnabled( false ); - revertItem.addActionListener( this ); - menu.add( revertItem ); - - menu.addSeparator(); - - closeItem = new JMenuItem( LABEL_CLOSE ); - closeItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_W, Event.CTRL_MASK ) ); - closeItem.setEnabled( false ); - closeItem.addActionListener( this ); - menu.add( closeItem ); - - - menuItem = new JMenuItem( LABEL_QUIT ); - menuItem.setAccelerator( KeyStroke.getKeyStroke( KeyEvent.VK_Q, Event.CTRL_MASK ) ); - menuItem.addActionListener( this ); - menu.add( menuItem ); - - menu = new JMenu( "Help" ); - menuBar.add( menu ); - - menuItem = new JMenuItem( LABEL_ABOUT ); - menuItem.addActionListener( this ); - menu.add( menuItem ); - - return menuBar; - - } - - public void setParent( JFrame parent ) { - - if( parent == null ) - throw new NullPointerException( "Parent frame must not be null." ); - - this.parent = parent; - } - - public void updateSelectedEditPanelTitle() { - - int i; - EditPanel editPanel; - - i = editTabbedPane.getSelectedIndex(); - editPanel = ( EditPanel )editTabbedPane.getComponent( i ); - editTabbedPane.setTitleAt( i, editPanel.getTitle() ); - } - - public void updateMenu() { - - boolean enabled; - - - enabled = editTabbedPane.getComponentCount() > 0; - - saveItem.setEnabled( enabled ); - saveAsItem.setEnabled( enabled ); - closeItem.setEnabled( enabled ); - revertItem.setEnabled( enabled ); - - } - - - public void open( File f ) { - - EditPanel editPanel; - StringBuffer buf; - String line; - - - try( BufferedReader reader = new BufferedReader( new FileReader( f ) ) ) { - - buf = new StringBuffer(); - - while( ( line = reader.readLine() ) != null ) - buf.append( line ).append( '\n' ); - - editPanel = new EditPanel( this, f, buf.toString() ); - editTabbedPane.add( f.getName(), editPanel ); - editTabbedPane.setSelectedComponent( editPanel ); - updateMenu(); - } - catch( IOException e ) { - JOptionPane.showMessageDialog( parent, - e.getMessage(), - "IOException", - JOptionPane.ERROR_MESSAGE ); - } - } - - @Override - public void windowOpened( WindowEvent e ) {} - - @Override - public void windowClosing( WindowEvent e ) { - quitAction(); - } - - @Override - public void windowClosed( WindowEvent e ) {} - - @Override - public void windowIconified( WindowEvent e ) {} - - @Override - public void windowDeiconified( WindowEvent e ) {} - - @Override - public void windowActivated( WindowEvent e ) {} - - @Override - public void windowDeactivated( WindowEvent e ) {} - - private int saveAsAction() { - - int returnVal; - - returnVal = fc.showSaveDialog( parent ); - - if( returnVal == JFileChooser.APPROVE_OPTION ) { - saveAction( fc.getSelectedFile() ); - } - - return returnVal; - } - - private void saveAction( File f ) { - - EditPanel editPanel; - - editPanel = ( EditPanel )editTabbedPane.getSelectedComponent(); - - try( BufferedWriter writer = new BufferedWriter( new FileWriter( f ) ) ) { - - writer.write( editPanel.getText() ); - editPanel.setFile( f, editPanel.getText() ); - updateSelectedEditPanelTitle(); - - } - catch( IOException e ) { - JOptionPane.showMessageDialog( parent, - e.getMessage(), - "IOException", - JOptionPane.ERROR_MESSAGE ); - } - - } - - private void quitAction() { - - EditPanel editPanel; - int option; - int dialogOption; - - for( Component c : editTabbedPane.getComponents() ) { - - editPanel = ( EditPanel )c; - if( editPanel.hasChanged() ) { - - editTabbedPane.setSelectedComponent( editPanel ); - - if( editPanel.hasFile() ) { - - option = JOptionPane.showConfirmDialog( parent, - "Save changes in "+editPanel.getFile().getName()+"?", - "Save changes", - JOptionPane.YES_NO_CANCEL_OPTION ); - - if( option == JOptionPane.OK_OPTION ) - saveAction( editPanel.getFile() ); - } - else { - - option = JOptionPane.showConfirmDialog( parent, - "Save changes in unsaved file?", - "Save changes", - JOptionPane.YES_NO_CANCEL_OPTION ); - - dialogOption = JFileChooser.APPROVE_OPTION; - - if( option == JOptionPane.OK_OPTION ) - dialogOption = saveAsAction(); - - if( dialogOption == JFileChooser.CANCEL_OPTION ) - return; - - - } - - if( option == JOptionPane.CANCEL_OPTION ) - return; - } - } - - System.exit( 0 ); - } - - -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SimpleStyle.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SimpleStyle.java deleted file mode 100644 index 015f443..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SimpleStyle.java +++ /dev/null @@ -1,115 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.Color; - -import javax.swing.text.Style; -import javax.swing.text.StyleConstants; -import javax.swing.text.StyledDocument; - - -public class SimpleStyle { - - private Boolean it; - private Boolean ul; - private Boolean bf; - private Color col; - - public SimpleStyle( Color col, Boolean it, Boolean bf, Boolean ul ) { - - setIt( it ); - setUl( ul ); - setBf( bf ); - setColor( col ); - } - - public void setIt( Boolean it ) { - this.it = it; - } - - public void setUl( Boolean ul ) { - this.ul = ul; - } - - public void setBf( Boolean bf ) { - this.bf = bf; - } - - public void setColor( Color col ) { - this.col = col; - } - - public void setColor( int r, int g, int b ) { - col = new Color( r, g, b ); - } - - public void addStyleToDoc( StyledDocument doc, String key ) { - - Style style; - - style = doc.addStyle( key, null ); - - if( ul != null ) - StyleConstants.setUnderline( style, ul ); - - if( bf != null ) - StyleConstants.setBold( style, bf ); - - if( it != null ) - StyleConstants.setItalic( style, it ); - - if( col != null ) - StyleConstants.setForeground( style, col ); - } - - public static SimpleStyle createItStyle( Color c ) { - return new SimpleStyle( c, true, null, null ); - } - - public static SimpleStyle createItStyle() { - return new SimpleStyle( null, true, null, null ); - } - - public static SimpleStyle createBfStyle() { - return new SimpleStyle( null, null, true, null ); - } - - public static SimpleStyle createUlStyle() { - return new SimpleStyle( null, null, null, true ); - } - - public static SimpleStyle createPlainStyle( Color c ) { - return new SimpleStyle( c, null, null, null ); - } -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/StaticPanel.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/StaticPanel.java deleted file mode 100644 index d5aa13b..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/StaticPanel.java +++ /dev/null @@ -1,312 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.BorderLayout; -import java.awt.image.BufferedImage; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import javax.imageio.ImageIO; -import javax.swing.BorderFactory; -import javax.swing.ImageIcon; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JTabbedPane; -import javax.swing.JTextPane; - -import de.huberlin.wbi.cuneiform.core.preprocess.ChannelListener; -import de.huberlin.wbi.cuneiform.core.preprocess.ParseException; -import de.huberlin.wbi.cuneiform.core.preprocess.PreListener; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; -import de.huberlin.wbi.cuneiform.core.staticreduction.DotNodeVisitor; -import de.huberlin.wbi.cuneiform.core.staticreduction.StaticNodeVisitor; -import de.huberlin.wbi.cuneiform.starlinger.StarlingerNodeVisitor; - - -public class StaticPanel extends JPanel implements Runnable { - - private static final long serialVersionUID = -5302001074704951699L; - - private SyntaxPanel channelPanel, reductionPanel, prePanel, modelPanel; - - private JLabel dagLabel; - private JTextPane dotPane, starlingerPane; - private SyntaxPanel srcPane; - private ErrorTableModel errorTableModel; - private int hash; - - public StaticPanel( SyntaxPanel srcPane, ErrorTableModel errorTableModel ) { - - JTabbedPane tabbedPane; - JPanel panel; - - setSrcPane( srcPane ); - setErrorTableModel( errorTableModel ); - - setLayout( new BorderLayout() ); - - - tabbedPane = new JTabbedPane(); - tabbedPane.setBorder( BorderFactory.createTitledBorder( "Static info" ) ); - add( tabbedPane, BorderLayout.CENTER ); - - prePanel = new SyntaxPanel(); - tabbedPane.addTab( "(i) Pre", prePanel ); - - channelPanel = new SyntaxPanel(); - tabbedPane.addTab( "(ii) Channel", channelPanel ); - - modelPanel = new SyntaxPanel(); - tabbedPane.addTab( "(iii) Model", modelPanel ); - - reductionPanel = new SyntaxPanel(); - tabbedPane.addTab( "(iv) Static red.", reductionPanel ); - - starlingerPane = new JTextPane(); - tabbedPane.addTab( "Starlinger", new JScrollPane( starlingerPane ) ); - - dotPane = new JTextPane(); - tabbedPane.addTab( "Dot", new JScrollPane( dotPane ) ); - - dagLabel = new JLabel(); - - panel = new JPanel(); - panel.setLayout( new BorderLayout() ); - panel.add( new JScrollPane( dagLabel ), BorderLayout.CENTER ); - - - tabbedPane.add( "DAG", panel ); - - - } - - public void setSrcPane( SyntaxPanel srcPane ) { - - if( srcPane == null ) - throw new NullPointerException( "Source pane must not be null." ); - - this.srcPane = srcPane; - } - - public void setErrorTableModel( ErrorTableModel errorTableModel ) { - - if( errorTableModel == null ) - throw new NullPointerException( "Error table model must not be null." ); - - this.errorTableModel = errorTableModel; - } - - @Override - public void run() { - - String afterPre, afterChannel; - TopLevelContext tlc; - String dot; - ImageIcon img; - StaticNodeVisitor staticVisitor; - DotNodeVisitor dotVisitor; - CompoundExpr ce1; - StarlingerNodeVisitor starlingerVisitor; - String starlinger; - String reduction; - - - while( true ) { - - if( srcPane.getText().hashCode() != hash ) - - try { - - hash = srcPane.getText().hashCode(); - - errorTableModel.clear(); - - afterPre = PreListener.process( srcPane.getText() ); - prePanel.setText( afterPre ); - - afterChannel = ChannelListener.process( afterPre ); - channelPanel.setText( afterChannel ); - - tlc = CfSemanticModelVisitor.process( afterChannel ); - modelPanel.setText( tlc.toString() ); - - if( tlc.isTargetListEmpty() ) { - starlinger = "{ 'nodes'=>{},'edges'=>{}}"; - dot = "digraph G {}"; - reduction = ""; - } - else { - - staticVisitor = new StaticNodeVisitor( tlc ); - - ce1 = tlc.visit( staticVisitor ); - reduction = ce1.toString()+";"; - - starlingerVisitor = new StarlingerNodeVisitor(); - - ce1.visit( starlingerVisitor ); - starlinger = starlingerVisitor.toString(); - - dotVisitor = new DotNodeVisitor(); - - ce1.visit( dotVisitor ); - dot = dotVisitor.toString(); - - } - - reductionPanel.setText( reduction ); - starlingerPane.setText( starlinger ); - dotPane.setText( dot ); - - img = runGraphViz( dot ); - dagLabel.setIcon( img ); - - } - catch( ParseException e ) { - errorTableModel.add( e ); - } - catch( RuntimeException e ) { - e.printStackTrace(); - } - catch( HasFailedException e ) { - e.printStackTrace(); - } catch (NotBoundException e) { - e.printStackTrace(); - } - finally { - errorTableModel.fireTableDataChanged(); - } - - try { - Thread.sleep( 2000 ); - } - catch( InterruptedException e ) { - e.printStackTrace(); - } - } - - - } - - private static ImageIcon runGraphViz( String dot ) { - - ImageIcon img; - - - - ProcessBuilder builder; - Process p; - ExecutorService executorService; - BufferedImage rawImage; - - builder = new ProcessBuilder( "/bin/bash", "-c", "dot -T png" ); - - try { - p = builder.start(); - - executorService = Executors.newSingleThreadExecutor(); - executorService.submit( new ProducerThread( p, dot ) ); - executorService.shutdown(); - - try( InputStream inStream = p.getInputStream() ) { - - - rawImage = ImageIO.read( inStream ); - if( rawImage == null ) { - System.err.println( "Cannot produce image. Is graphviz installed properly?" ); - return null; - } - - rawImage.flush(); - - img = new ImageIcon( rawImage ); - - - return img; - } - } - catch( IOException e ) { - e.printStackTrace(); - } - - return null; - - - } - - private static class ProducerThread implements Runnable { - - private Process p; - private String dot; - - public ProducerThread( Process p, String dot ) { - - if( p == null ) - throw new NullPointerException( "Process must not be null." ); - - if( dot == null ) - throw new NullPointerException( "Dot string must not be null." ); - - this.p = p; - this.dot = dot; - } - - @Override - public void run() { - - - - - try( OutputStream outStream = p.getOutputStream() ) { - - outStream.write( dot.getBytes() ); - outStream.close(); - } catch( IOException e ) { - e.printStackTrace(); - } - - } - - - } - -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/StyleConf.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/StyleConf.java deleted file mode 100644 index fe1bbc3..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/StyleConf.java +++ /dev/null @@ -1,129 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.Color; -import java.util.HashMap; -import java.util.Map; - -import javax.swing.text.StyledDocument; - -public abstract class StyleConf { - - - public static final String KEY_KEYWORD = "keyword"; - public static final String KEY_VARNAME = "varname"; - public static final String KEY_COMMENT = "comment"; - public static final String KEY_CALL = "call"; - public static final String KEY_APPLY = "apply"; - public static final String KEY_STAT = "stat"; - public static final String KEY_DATA = "data"; - public static final String KEY_TYPE = "type"; - public static final String KEY_FOREIGN = "foreign"; - - - private Map styleMap; - - public StyleConf() { - styleMap = new HashMap<>(); - } - - public abstract Color getBackgroundColor(); - - public void setDoc( StyledDocument doc ) { - - SimpleStyle s; - - if( doc == null ) - throw new NullPointerException( "Document must not be null." ); - - for( String key : styleMap.keySet() ) { - - s = styleMap.get( key ); - if( s == null ) - throw new NullPointerException( "Style for '"+key+"' not registered." ); - - s.addStyleToDoc( doc, key ); - } - } - - private void setStyle( String key, SimpleStyle s ) { - - if( key == null ) - throw new NullPointerException( "Key must not be null." ); - - if( key.isEmpty() ) - throw new RuntimeException( "Key must not be empty." ); - - if( s == null ) - throw new NullPointerException( "Style object must not be null." ); - - styleMap.put( key, s ); - } - - public void setApplyStyle( SimpleStyle s ) { - setStyle( KEY_APPLY, s ); - } - - public void setCallStyle( SimpleStyle s ) { - setStyle( KEY_CALL, s ); - } - - public void setKeywordStyle( SimpleStyle s ) { - setStyle( KEY_KEYWORD, s ); - } - - public void setVarnameStyle( SimpleStyle s ) { - setStyle( KEY_VARNAME, s ); - } - - public void setCommentStyle( SimpleStyle s ) { - setStyle( KEY_COMMENT, s ); - } - - public void setStatStyle( SimpleStyle s ) { - setStyle( KEY_STAT, s ); - } - - public void setDataStyle( SimpleStyle s ) { - setStyle( KEY_DATA, s ); - } - - public void setTypeStyle( SimpleStyle s ) { - setStyle( KEY_TYPE, s ); - } - - public void setForeignStyle( SimpleStyle s ) { - setStyle( KEY_FOREIGN, s ); - } -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SyntaxListener.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SyntaxListener.java deleted file mode 100644 index a71dac0..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SyntaxListener.java +++ /dev/null @@ -1,385 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - - -import javax.swing.JTextPane; -import javax.swing.text.Style; -import javax.swing.text.StyledDocument; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.ParserRuleContext; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.misc.NotNull; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; -import org.antlr.v4.runtime.tree.TerminalNode; - -import de.huberlin.wbi.cuneiform.core.parser.CuneiformBaseListener; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformLexer; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser; - -public class SyntaxListener extends CuneiformBaseListener { - - private JTextPane textPane; - - public SyntaxListener( JTextPane textPane, StyleConf conf ) { - - if( textPane == null ) - throw new NullPointerException( "Text pane must not be null." ); - - if( conf == null ) - throw new NullPointerException( "Style configuration object never set." ); - - this.textPane = textPane; - conf.setDoc( getDoc() ); - - } - - public StyledDocument getDoc() { - return textPane.getStyledDocument(); - } - - public static void process( JTextPane textPane, StyleConf conf ) { - - ANTLRInputStream instream; - CuneiformLexer lexer; - TokenStream tokenStream; - CuneiformParser parser; - ParseTree tree; - SyntaxListener syntaxListener; - ParseTreeWalker walker; - - walker = new ParseTreeWalker(); - - - // parse content - instream = new ANTLRInputStream( textPane.getText() ); - - lexer = new CuneiformLexer( instream ); - lexer.removeErrorListeners(); - - tokenStream = new CommonTokenStream( lexer ); - - parser = new CuneiformParser( tokenStream ); - parser.removeErrorListeners(); - - syntaxListener = new SyntaxListener( textPane, conf ); - - tree = parser.script(); - walker.walk( syntaxListener, tree ); - } - - @Override - public void enterNilExpr( @NotNull CuneiformParser.NilExprContext ctx ) { - mark( ctx, StyleConf.KEY_KEYWORD ); - mark( ctx, StyleConf.KEY_DATA ); - } - - @Override - public void enterStat( @NotNull CuneiformParser.StatContext ctx ) { - mark( ctx, StyleConf.KEY_STAT, true ); - } - - @Override - public void enterScript( @NotNull CuneiformParser.ScriptContext ctx ) { - - StyledDocument doc; - - doc = getDoc(); - - doc.setCharacterAttributes( - 0, - doc.getLength(), - doc.getStyle( StyleConf.KEY_COMMENT ), - true ); - } - - @Override - public void enterTarget( @NotNull CuneiformParser.TargetContext ctx ) { - mark( ctx, StyleConf.KEY_APPLY ); - } - - @Override - public void enterNativeDefTask( @NotNull CuneiformParser.NativeDefTaskContext ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterForeignDefTask( @NotNull CuneiformParser.ForeignDefTaskContext ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterDefTaskErr1( @NotNull CuneiformParser.DefTaskErr1Context ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - } - - @Override - public void enterDefTaskErr2( @NotNull CuneiformParser.DefTaskErr2Context ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterDefTaskErr3( @NotNull CuneiformParser.DefTaskErr3Context ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterFnPrototypeErr1( @NotNull CuneiformParser.FnPrototypeErr1Context ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterFnPrototypeErr2( @NotNull CuneiformParser.FnPrototypeErr2Context ctx ) { - mark( ctx.DEFTASK(), StyleConf.KEY_KEYWORD ); - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterForeignBody( @NotNull CuneiformParser.ForeignBodyContext ctx ) { - mark( ctx.INLANG(), StyleConf.KEY_KEYWORD ); - mark( ctx.BODY(), StyleConf.KEY_FOREIGN ); - } - - @Override - public void enterForeignFnBodyErr2( @NotNull CuneiformParser.ForeignFnBodyErr2Context ctx ) { - mark( ctx.INLANG(), StyleConf.KEY_KEYWORD ); - } - - @Override - public void enterNameInferredType( @NotNull CuneiformParser.NameInferredTypeContext ctx ) { - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterNameDataType( @NotNull CuneiformParser.NameDataTypeContext ctx ) { - mark( ctx, StyleConf.KEY_TYPE ); - mark( ctx.ID( 0 ), StyleConf.KEY_VARNAME ); - } - - @Override public void enterNameDeepFnType( @NotNull CuneiformParser.NameDeepFnTypeContext ctx ) { - mark( ctx, StyleConf.KEY_TYPE ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterNamePlainFnType( @NotNull CuneiformParser.NamePlainFnTypeContext ctx ) { - mark( ctx, StyleConf.KEY_TYPE ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterIntExpr( @NotNull CuneiformParser.IntExprContext ctx ) { - mark( ctx, StyleConf.KEY_DATA ); - } - - @Override - public void enterStringExpr( @NotNull CuneiformParser.StringExprContext ctx ) { - mark( ctx, StyleConf.KEY_DATA ); - } - - @Override - public void enterApplyExpr( @NotNull CuneiformParser.ApplyExprContext ctx ) { - mark( ctx.APPLY(), StyleConf.KEY_KEYWORD ); - mark( ctx.APPLY(), StyleConf.KEY_CALL ); - } - - @Override - public void enterCallExpr( @NotNull CuneiformParser.CallExprContext ctx ) { - mark( ctx.ID(), StyleConf.KEY_CALL ); - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override public void enterCurryExpr(@NotNull CuneiformParser.CurryExprContext ctx) { - mark( ctx.CURRY(), StyleConf.KEY_KEYWORD ); - mark( ctx.CURRY(), StyleConf.KEY_CALL ); - } - - @Override - public void enterNativeLambdaExpr( @NotNull CuneiformParser.NativeLambdaExprContext ctx ) { - mark( ctx.LAMBDA(), StyleConf.KEY_KEYWORD ); - } - - @Override - public void enterForeignLambdaExpr( @NotNull CuneiformParser.ForeignLambdaExprContext ctx ) { - mark( ctx.LAMBDA(), StyleConf.KEY_KEYWORD ); - } - - @Override - public void enterParamBind( @NotNull CuneiformParser.ParamBindContext ctx ) { - mark( ctx.ID(), StyleConf.KEY_VARNAME ); - } - - @Override - public void enterDanglingExpr( @NotNull CuneiformParser.DanglingExprContext ctx ) { - mark( ctx.TOSTACK(), StyleConf.KEY_COMMENT ); - } - - @Override - public void enterFromStackExpr( @NotNull CuneiformParser.FromStackExprContext ctx ) { - mark( ctx.FROMSTACK(), StyleConf.KEY_COMMENT ); - } - - private void mark( TerminalNode tn, String styleName ) { - - Token sym; - - if( tn == null ) - return; - - sym = tn.getSymbol(); - if( sym == null ) - return; - - mark( sym, styleName ); - } - - private void mark( Token t, String styleName ) { - - Style style; - int start, len; - StyledDocument doc; - - if( t == null ) - return; - - if( styleName == null ) - throw new NullPointerException( "Style name must not be null." ); - - if( styleName.isEmpty() ) - throw new RuntimeException( "Style name must not be empty." ); - - doc = getDoc(); - - style = doc.getStyle( styleName ); - if( style == null ) - throw new NullPointerException( "A style with name '"+styleName+"' has never been defined." ); - - start = t.getStartIndex(); - if( start < 0 ) - return; - - len = t.getStopIndex()-start+1; - if( len <= 0 ) - return; - - doc.setCharacterAttributes( - start, - len, - style, - false ); - } - - private void mark( ParserRuleContext ctx, String styleName ) { - mark( ctx, styleName, false ); - } - - private void mark( ParserRuleContext ctx, String styleName, boolean supersede ) { - - Style style; - int start, len; - StyledDocument doc; - - if( ctx == null ) - throw new NullPointerException( "Context must not be null." ); - - if( styleName == null ) - throw new NullPointerException( "Style name must not be null." ); - - if( styleName.isEmpty() ) - throw new RuntimeException( "Style name must not be empty." ); - - if( ctx.stop == null ) - return; - - if( ctx.start == null ) - return; - - doc = getDoc(); - - style = doc.getStyle( styleName ); - if( style == null ) - throw new NullPointerException( "A style with name '"+styleName+"' has never been defined." ); - - start = getStart( ctx ); - if( start < 0 ) - return; - - len = getLen( ctx ); - if( len <= 0 ) - return; - - doc.setCharacterAttributes( - start, - len, - style, - supersede ); - } - - private static int getStart( ParserRuleContext ctx ) { - - if( ctx == null ) - throw new NullPointerException( "Context must not be null." ); - - if( ctx.start == null ) - throw new NullPointerException( "Start token must not be null." ); - - return ctx.start.getStartIndex(); - } - - private static int getLen( ParserRuleContext ctx ) { - - if( ctx == null ) - throw new NullPointerException( "Context must not be null." ); - - if( ctx.stop == null ) - throw new NullPointerException( "Stop token must not be null." ); - - return ctx.stop.getStopIndex()-getStart( ctx )+1; - } - -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SyntaxPanel.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SyntaxPanel.java deleted file mode 100644 index 221e90b..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/editor/SyntaxPanel.java +++ /dev/null @@ -1,121 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.editor; - -import java.awt.BorderLayout; -import java.awt.Color; -import java.awt.Cursor; -import java.awt.Font; -import java.awt.event.KeyListener; - -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JTextPane; -import javax.swing.event.CaretListener; -import javax.swing.text.Document; - -public class SyntaxPanel extends JPanel { - - private static final long serialVersionUID = -385807381349136708L; - - private JTextPane editPane; - protected StyleConf sc; - - public SyntaxPanel() { - - JPanel editPanel, innerPanel; - JScrollPane scrollPane; - - - setLayout( new BorderLayout() ); - sc = new DarkStyleConf(); - - // edit pane - editPane = new JTextPane(); - editPane.setBackground( sc.getBackgroundColor() ); - editPane.setFont( new Font( "Courier", Font.PLAIN, 14 ) ); - editPane.setForeground( Color.WHITE ); - editPane.setCaretColor( Color.WHITE ); - editPane.setCursor( new Cursor( Cursor.TEXT_CURSOR ) ); - editPane.setEditable( false ); - - innerPanel = new JPanel(); - innerPanel.setLayout( new BorderLayout() ); - innerPanel.add( editPane ); - - scrollPane = new JScrollPane( innerPanel ); - scrollPane.getVerticalScrollBar().setUnitIncrement( 32 ); - scrollPane.getHorizontalScrollBar().setUnitIncrement( 32 ); - - editPanel = new JPanel(); - editPanel.setLayout( new BorderLayout() ); - editPanel.add( scrollPane, BorderLayout.CENTER ); - add( editPanel, BorderLayout.CENTER ); - - - } - - public int getCaretPosition() { - return editPane.getCaretPosition(); - } - - public Document getDocument() { - return editPane.getDocument(); - } - - public JTextPane getEditPane() { - return editPane; - } - - public String getText() { - return editPane.getText(); - } - - public void setText( String text ) { - editPane.setText( text ); - SyntaxListener.process( editPane, sc ); - } - - public void setEditable( boolean editable ) { - editPane.setEditable( editable ); - } - - @Override - public synchronized void addKeyListener( KeyListener listener ) { - editPane.addKeyListener( listener ); - } - - public void addCaretListener( CaretListener listener ) { - editPane.addCaretListener( listener ); - } -} diff --git a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/main/Main.java b/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/main/Main.java deleted file mode 100644 index ff65983..0000000 --- a/cuneiform-addons/cuneiform-cfide/src/main/java/de/huberlin/wbi/cuneiform/cfide/main/Main.java +++ /dev/null @@ -1,108 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cfide.main; - -import java.io.File; - -import javax.swing.JFrame; -import javax.swing.JMenuBar; -import javax.swing.SwingUtilities; -import javax.swing.WindowConstants; - -import de.huberlin.wbi.cuneiform.cfide.editor.MainPanel; - -public class Main implements Runnable { - - private String[] loadList; - - public static void main( String[] args ) { - - Main m; - - m = new Main( args ); - - SwingUtilities.invokeLater( m ); - } - - public Main( String[] args ) { - - if( args == null ) - throw new IllegalArgumentException( "Argument vector must not be null." ); - - loadList = args; - } - - - @Override - public void run() { - - JFrame frame; - JMenuBar menuBar; - MainPanel editRunPanel; - - - // prepare frame - frame = new JFrame( "Cuneiform Editor" ); - frame.setSize( 1000, 750 ); - frame.setLocation( 0, 50 ); - frame.setDefaultCloseOperation( WindowConstants.DO_NOTHING_ON_CLOSE ); - - // add main panel - editRunPanel = new MainPanel( frame ); - frame.addWindowListener( editRunPanel ); - frame.add( editRunPanel ); - - // add menu bar - menuBar = editRunPanel.getMenuBar(); - frame.setJMenuBar( menuBar ); - - // load designated files - for( String filename : loadList ) - editRunPanel.open( new File( filename ) ); - - - - - - - - - - - - - - - frame.setVisible( true ); - } -} diff --git a/cuneiform-addons/cuneiform-dax/.gitignore b/cuneiform-addons/cuneiform-dax/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-addons/cuneiform-dax/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-addons/cuneiform-dax/pom.xml b/cuneiform-addons/cuneiform-dax/pom.xml deleted file mode 100644 index 67f719c..0000000 --- a/cuneiform-addons/cuneiform-dax/pom.xml +++ /dev/null @@ -1,67 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform-addons - 2.0.4-RELEASE - - cuneiform-dax - - - - de.hu-berlin.wbi.cuneiform - cuneiform-core - ${project.version} - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - org.antlr - antlr4-maven-plugin - 4.5 - - ${basedir}/src/main/antlr - true - true - - - - - antlr4 - - - - - - - - \ No newline at end of file diff --git a/cuneiform-addons/cuneiform-dax/src/main/antlr/de/huberlin/cuneiform/libdax/parser/Dax.g4 b/cuneiform-addons/cuneiform-dax/src/main/antlr/de/huberlin/cuneiform/libdax/parser/Dax.g4 deleted file mode 100644 index 9450b9b..0000000 --- a/cuneiform-addons/cuneiform-dax/src/main/antlr/de/huberlin/cuneiform/libdax/parser/Dax.g4 +++ /dev/null @@ -1,138 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -grammar Dax; - -adag : LTAG ADAG adagProp* RTAG adagEl* LTAG SLASH ADAG RTAG ; - -adagProp : XMLNS EQ STRING # AdagPropXmlns - | XSI EQ STRING # AdagPropXsi - | SCHEMALOCATION EQ STRING # AdagPropSchemaLocation - | VERSION EQ STRING # AdagPropVersion - | COUNT EQ STRING # AdagPropCount - | INDEX EQ STRING # AdagPropIndex - | NAME EQ STRING # AdagPropName - ; - -adagEl : filename - | job - | child - ; - -filename : LTAG( FILE | FILENAME )filenameProp* SLASH RTAG ; - -filenameProp : ( FILE | NAME )EQ STRING # FilenamePropFile - | LINK EQ INPUT # FilenamePropLinkInput - | LINK EQ OUTPUT # FilenamePropLinkOutput - | LINK EQ INOUT # FilenamePropLinkInout - ; - -job : LTAG JOB jobProp* RTAG jobEl* LTAG SLASH JOB RTAG ; - -jobProp : ID EQ STRING # JobPropId - | NAME EQ STRING # JobPropName - | VERSION EQ STRING # JobPropVersion - | LEVEL EQ STRING # JobPropLevel - | DVNAME EQ STRING # JobPropDvName - | DVVERSION EQ STRING # JobPropDvVersion - | NAMESPACE EQ STRING # JobPropNamespace - ; - -jobEl : LARGUMENT argumentEl* RARGUMENT # JobElArgument - | LTAG USES jobUsesProp* SLASH RTAG # JobElUses - ; - -argumentEl : ARG # ArgumentElPlain - | filename # ArgumentElFilename - ; - -jobUsesProp : ( FILE | NAME )EQ STRING # JobUsesPropFile - | LINK EQ INPUT # JobUsesPropLinkInput - | LINK EQ OUTPUT # JobUsesPropLinkOutput - | REGISTER EQ( TRUE | FALSE ) # JobUsesPropRegister - | TRANSFER EQ( TRUE | FALSE ) # JobUsesPropTransfer - | TYPE EQ EXECUTABLE # JobUsesPropExecutable - | OPTIONAL EQ TRUE # JobUsesPropOptionalTrue - | OPTIONAL EQ FALSE # JobUsesPropOptionalFalse - ; - -child : LTAG CHILD REF EQ STRING RTAG parent+ LTAG SLASH CHILD RTAG ; - -parent : LTAG PARENT REF EQ STRING SLASH RTAG ; - -ADAG : 'adag' ; -ARGUMENT : 'argument' ; -CHILD : 'child' ; -COUNT : 'count' ; -DVNAME : 'dv-name' ; -DVVERSION : 'dv-version' ; -EQ : '=' ; -EXECUTABLE : '"executable"' ; -FALSE : '"false"' ; -FILE : 'file' ; -FILENAME : 'filename' ; -ID : 'id' ; -INDEX : 'index' ; -INOUT : '"inout"' ; -INPUT : '"input"' ; -JOB : 'job' ; -LARGUMENT : '' ; -LEVEL : 'level' ; -LINK : 'link' ; -LTAG : '<' ; -NAME : 'name' ; -NAMESPACE : 'namespace' ; -OPTIONAL : 'optional' ; -OUTPUT : '"output"' ; -PARENT : 'parent' ; -RARGUMENT : '' ; -REF : 'ref' ; -REGISTER : 'register' ; -RTAG : '>' ; -SCHEMALOCATION : 'xsi:schemaLocation' ; -SLASH : '/' ; -TRANSFER : 'transfer' ; -TRUE : '"true"' ; -TYPE : 'type' ; -USES : 'uses' ; -VERSION : 'version' ; -XMLNS : 'xmlns' ; -XSI : 'xmlns:xsi' ; - -STRING : '"' .*? '"' ; -ARG : [a-zA-Z0-9\-\.]+ ; - -METAINFO : '' -> skip ; -COMMENT : '' -> skip ; -WS : [ \t\r\n] -> skip ; - - diff --git a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/repl/DaxRepl.java b/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/repl/DaxRepl.java deleted file mode 100644 index 183a68f..0000000 --- a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/repl/DaxRepl.java +++ /dev/null @@ -1,116 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.cuneiform.dax.repl; - -import java.util.UUID; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; -import org.apache.commons.logging.Log; - -import de.huberlin.cuneiform.dax.semanticmodel.DaxSemanticModelListener; -import de.huberlin.cuneiform.libdax.parser.DaxLexer; -import de.huberlin.cuneiform.libdax.parser.DaxParser; -import de.huberlin.wbi.cuneiform.core.repl.BaseRepl; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor; - -public class DaxRepl extends BaseRepl { - - public DaxRepl( TicketSrcActor ticketSrc, Log statLog ) { - super( ticketSrc, statLog ); - } - - @Override - public synchronized int interpret( String input ) { - - DaxSemanticModelListener adag; - TopLevelContext tlc; - - adag = process( input ); - tlc = adag.toTopLevelContext(); - - return interpret( tlc ); - - } - - @Override - public void queryFailedPost( UUID queryId, Long ticketId, Exception e, - String script, String stdOut, String stdErr ) {} - - @Override - public void queryFinishedPost( UUID queryId, CompoundExpr result ) {} - - @Override - public void queryStartedPost( UUID runId ) {} - - public static DaxSemanticModelListener process( String input ) { - - ANTLRInputStream instream; - DaxLexer lexer; - CommonTokenStream tokenStream; - DaxParser parser; - ParseTree tree; - ParseTreeWalker walker; - DaxSemanticModelListener adag; - - walker = new ParseTreeWalker(); - - // parse original content - instream = new ANTLRInputStream( input ); - - lexer = new DaxLexer( instream ); - lexer.removeErrorListeners(); - - tokenStream = new CommonTokenStream( lexer ); - - parser = new DaxParser( tokenStream ); - parser.removeErrorListeners(); - - adag = new DaxSemanticModelListener(); - lexer.addErrorListener( adag ); - parser.addErrorListener( adag ); - - - tree = parser.adag(); - - walker.walk( adag, tree ); - - return adag; - } - - -} diff --git a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxFilename.java b/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxFilename.java deleted file mode 100644 index fbb7ba1..0000000 --- a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxFilename.java +++ /dev/null @@ -1,120 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.cuneiform.dax.semanticmodel; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; - -public class DaxFilename { - - public static final int LINK_INPUT = 1; - public static final int LINK_OUTPUT = 2; - public static final int LINK_INOUT = 3; - private static final String PREFIX_LINK = "link"; - - private String file; - private Integer link; - - @Override - public boolean equals( Object obj ) { - - DaxFilename other; - - if( !( obj instanceof DaxFilename ) ) - return false; - - other = ( DaxFilename )obj; - if( file == null ) - throw new NullPointerException( - "Trying to compare filenames when file member is not set." ); - - return file.equals( other.file ); - } - - public String getFile() { - return file; - } - - @Override - public int hashCode() { - return file.hashCode(); - } - - public boolean isLinkInput() { - - if( link == null ) - throw new NullPointerException( "Link direction not set." ); - - return link == LINK_INPUT; - } - - public boolean isLinkOutput() { - - if( link == null ) - throw new NullPointerException( "Link direction not set." ); - - return link == LINK_OUTPUT; - } - - public void setFile( String file ) { - - if( file == null ) - throw new NullPointerException( "File string must not be null." ); - - if( file.isEmpty() ) - throw new RuntimeException( "File string must not be empty." ); - - this.file = file; - } - - public void setLinkInput() { - link = LINK_INPUT; - } - - public void setLinkOutput() { - link = LINK_OUTPUT; - } - - public void setLinkInout() { - link = LINK_INOUT; - } - - public NameExpr getNameExpr() { - return new NameExpr( PREFIX_LINK+file.hashCode() ); - } - - @Override - public String toString() { - return file; - } - -} diff --git a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxJob.java b/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxJob.java deleted file mode 100644 index 9d29508..0000000 --- a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxJob.java +++ /dev/null @@ -1,383 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.cuneiform.dax.semanticmodel; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.DataType; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ReduceVar; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Type; - -public class DaxJob { - - private static final String PREFIX_OUT = "out"; - private static final String PREFIX_IN = "in"; - - private String name; - private final List parentSet; - private final List childSet; - private final List argList; - private final List jobUsesList; - - public DaxJob() { - parentSet = new ArrayList<>(); - childSet = new ArrayList<>(); - argList = new ArrayList<>(); - jobUsesList = new ArrayList<>(); - } - - public void addChild( DaxJob child ) { - - if( child == null ) - throw new NullPointerException( "Child DAX job must not be null." ); - - childSet.add( child ); - } - - public void addFilenameArg( DaxFilename filename ) { - - if( filename == null ) - throw new NullPointerException( "Filename must not be null." ); - - argList.add( filename ); - } - - public void addJobUses( DaxJobUses jobUses ) { - - if( jobUses == null ) - throw new NullPointerException( "Job-Uses element object must not be null." ); - - jobUsesList.add( jobUses ); - } - - public void addParent( DaxJob parent ) { - - if( parent == null ) - throw new NullPointerException( "Parent DAX job must not be null." ); - - parentSet.add( parent ); - } - - public void addPlainArg( String arg ) { - - if( arg == null ) - throw new NullPointerException( "Argument string must not be null." ); - - if( arg.isEmpty() ) - throw new RuntimeException( "Argument string must not be empty." ); - - argList.add( arg ); - } - - public String getName() { - return name; - } - - public boolean hasName() { - return name != null; - } - - public boolean isLeaf() { - return childSet.isEmpty(); - } - - public boolean isRoot() { - return parentSet.isEmpty(); - } - - public void setName( String name ) { - - if( name == null ) { - this.name = null; - return; - } - - if( name.isEmpty() ) - throw new RuntimeException( "Name string must not be empty." ); - - this.name = name; - } - - public Set getInputJobUsesSet() { - - HashSet set; - - set = new HashSet<>(); - for( DaxJobUses jobUses : jobUsesList ) - if( jobUses.isLinkInput() ) - set.add( jobUses ); - - return set; - } - - public SetgetOutputJobUsesSet() { - - HashSet set; - - set = new HashSet<>(); - for( DaxJobUses jobUses : jobUsesList ) { - - if( isBidirectional( jobUses ) ) - continue; - - if( jobUses.isLinkOutput() ) - set.add( jobUses ); - } - - return set; - } - - public Prototype getPrototype() { - - Prototype prototype; - Type type; - int n, m; - - prototype = new Prototype(); - prototype.addParam( new NameExpr( CfSemanticModelVisitor.LABEL_TASK ) ); - - n = 1; - m = 1; - type = new DataType( DataType.LABEL_FILE ); - - for( DaxJobUses jobUses : jobUsesList ) { - - if( jobUses.isExecutable() ) - continue; - - if( jobUses.isLinkOutput() ) { - - if( isBidirectional( jobUses ) ) - continue; - - if( jobUses.isOptional() ) - prototype.addOutput( new ReduceVar( PREFIX_OUT+( n++ ), type ) ); - else - prototype.addOutput( new NameExpr( PREFIX_OUT+( n++ ), type ) ); - } - else - prototype.addParam( new NameExpr( PREFIX_IN+( m++ ), type ) ); - - } - - return prototype; - } - - public boolean isBidirectional( DaxJobUses filename ) { - - int i; - - i = 0; - for( DaxJobUses jobUses : jobUsesList ) { - - if( !jobUses.equals( filename ) ) - continue; - - i++; - } - - return i > 1; - } - - public String getReference( DaxFilename filename ) { - - int i; - - i = 1; - for( DaxJobUses jobUses : jobUsesList ) { - - if( !jobUses.isLinkInput() ) - continue; - - if( jobUses.isExecutable() ) - continue; - - if( jobUses.equals( filename ) ) - return PREFIX_IN+i; - - i++; - } - - i = 1; - for( DaxJobUses jobUses : jobUsesList ) { - - if( !jobUses.isLinkOutput() ) - continue; - - if( jobUses.equals( filename ) ) - return PREFIX_OUT+i; - - i++; - } - - throw new RuntimeException( "DAX filename '"+filename.getFile()+"' not registered in any direction." ); - } - - public ForeignLambdaExpr getLambda() { - - ForeignLambdaExpr lambda; - StringBuffer buf; - - buf = new StringBuffer(); - - for( DaxJobUses jobUses : jobUsesList ) { - - if( jobUses.isLinkInput() ) - continue; - - if( isBidirectional( jobUses ) ) - continue; - - buf.append( getReference( jobUses ) ).append( "=\"" ); - buf.append( jobUses.getFile() ).append( "\"\n" ); - } - - buf.append( name ); - for( Object arg : argList ) { - - buf.append( ' ' ); - - if( arg instanceof String ) { - buf.append( arg ); - continue; - } - - if( arg instanceof DaxFilename ) { - - buf.append( '$' ); - buf.append( getReference( ( DaxFilename )arg ) ); - continue; - } - - throw new RuntimeException( "Argument type not recognized." ); - } - - for( DaxJobUses jobUses : jobUsesList ) - if( jobUses.isLinkOutput() ) - if( jobUses.isOptional() ) - buf.append( "\nif [[ ! -f "+jobUses.getFile()+" ]]\nthen\n"+getReference( jobUses )+"=\nfi" ); - - lambda = new ForeignLambdaExpr( - getPrototype(), - ForeignLambdaExpr.LANGID_PEGASUS, - buf.toString() ); - - return lambda; - } - - public int getChannel( DaxFilename filename ) { - - int channel; - - if( filename == null ) - throw new NullPointerException( "DAX filename must not be null." ); - - channel = 1; - for( DaxJobUses jobUses : jobUsesList ) { - - if( !jobUses.isLinkOutput() ) - continue; - - if( jobUses.equals( filename ) ) - return channel; - - channel++; - } - - throw new RuntimeException( "DAX filename not registered." ); - } - - public ApplyExpr getApplyExpr( DaxFilename filename, List fileList ) { - - ApplyExpr applyExpr; - int channel, i; - Prototype prototype; - - // find out channel - channel = getChannel( filename ); - - // create new apply expression - applyExpr = new ApplyExpr( channel, false ); - - // set task expression - applyExpr.setTaskExpr( new CompoundExpr( getLambda() ) ); - - // bind parameters - i = 1; - for( DaxJobUses jobUses : jobUsesList ) { - - if( jobUses.isLinkOutput() ) - continue; - - if( jobUses.isExecutable() ) - continue; - - if( fileList.contains( jobUses ) ) - applyExpr.putAssign( - new NameExpr( PREFIX_IN+( i++ ) ), - new CompoundExpr( jobUses.getNameExpr() ) ); - else { - - try { - - prototype = applyExpr.getPrototype(); - prototype.removeParam( PREFIX_IN+i ); - prototype.addParam( new NameExpr( PREFIX_IN+i ) ); - - applyExpr.putAssign( - new NameExpr( PREFIX_IN+( i++ ) ), - new CompoundExpr( new StringExpr( jobUses.getFile() ) ) ); - } - catch( NotDerivableException e ) { - throw new RuntimeException( e ); - } - - } - } - - return applyExpr; - } - - -} diff --git a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxJobUses.java b/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxJobUses.java deleted file mode 100644 index c336591..0000000 --- a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxJobUses.java +++ /dev/null @@ -1,62 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.cuneiform.dax.semanticmodel; - - -public class DaxJobUses extends DaxFilename { - - private boolean executable; - private boolean optional; - - public DaxJobUses() { - executable = false; - optional = false; - } - - public boolean isExecutable() { - return executable; - } - - public boolean isOptional() { - return optional; - } - - public void setExecutable( boolean e ) { - executable = e; - } - - public void setOptional( boolean o ) { - optional = o; - } - -} diff --git a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxSemanticModelListener.java b/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxSemanticModelListener.java deleted file mode 100644 index 2801847..0000000 --- a/cuneiform-addons/cuneiform-dax/src/main/java/de/huberlin/cuneiform/dax/semanticmodel/DaxSemanticModelListener.java +++ /dev/null @@ -1,321 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.cuneiform.dax.semanticmodel; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.ArrayList; -import java.util.BitSet; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.antlr.v4.runtime.ANTLRErrorListener; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.atn.ATNConfigSet; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.NotNull; -import org.antlr.v4.runtime.tree.TerminalNode; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import de.huberlin.cuneiform.libdax.parser.DaxBaseListener; -import de.huberlin.cuneiform.libdax.parser.DaxParser; -import de.huberlin.wbi.cuneiform.core.preprocess.ParseException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; - -public class DaxSemanticModelListener extends DaxBaseListener implements ANTLRErrorListener { - - private final Map idJobMap; - private final Map fileJobMap; - private final List fileList; - private final List outputList; - private DaxFilename filename; - private DaxJobUses jobUses; - private DaxJob job; - private final Log log; - - public DaxSemanticModelListener() { - idJobMap = new HashMap<>(); - fileJobMap = new HashMap<>(); - fileList = new ArrayList<>(); - outputList = new ArrayList<>(); - log = LogFactory.getLog( DaxSemanticModelListener.class ); - } - - @Override - public void enterFilename( @NotNull DaxParser.FilenameContext ctx ) { - filename = new DaxFilename(); - } - - @Override - public void exitFilename( @NotNull DaxParser.FilenameContext ctx ) { - - if( job == null ) { - - fileList.add( filename ); - if( filename.isLinkOutput() ) - outputList.add( filename ); - } - else - job.addFilenameArg( filename ); - - filename = null; - } - - @Override - public void enterFilenamePropFile( @NotNull DaxParser.FilenamePropFileContext ctx ) { - filename.setFile( getString( ctx.STRING() ) ); - } - - @Override - public void enterFilenamePropLinkInput( @NotNull DaxParser.FilenamePropLinkInputContext ctx ) { - filename.setLinkInput(); - } - - @Override - public void enterFilenamePropLinkOutput( @NotNull DaxParser.FilenamePropLinkOutputContext ctx ) { - filename.setLinkOutput(); - } - - @Override - public void enterFilenamePropLinkInout( @NotNull DaxParser.FilenamePropLinkInoutContext ctx ) { - filename.setLinkInout(); - } - - @Override - public void enterJob( @NotNull DaxParser.JobContext ctx ) { - job = new DaxJob(); - } - - @Override - public void exitJob( @NotNull DaxParser.JobContext ctx ) { - job = null; - } - - @Override - public void exitJobPropId( @NotNull DaxParser.JobPropIdContext ctx ) { - - String id; - - id = getString( ctx.STRING() ); - - idJobMap.put( id, job ); - } - - @Override - public void enterJobPropName( @NotNull DaxParser.JobPropNameContext ctx ) { - job.setName( getString( ctx.STRING() ) ); - } - - @Override - public void enterArgumentElPlain( @NotNull DaxParser.ArgumentElPlainContext ctx ) { - job.addPlainArg( ctx.ARG().getText() ); - } - - @Override - public void enterJobElUses( @NotNull DaxParser.JobElUsesContext ctx ) { - jobUses = new DaxJobUses(); - job.addJobUses( jobUses ); - } - - @Override - public void exitJobElUses( @NotNull DaxParser.JobElUsesContext ctx ) { - - if( jobUses.isLinkOutput() ) - fileJobMap.put( jobUses.getFile(), job ); - - jobUses = null; - } - - @Override - public void enterJobUsesPropFile( @NotNull DaxParser.JobUsesPropFileContext ctx ) { - jobUses.setFile( getString( ctx.STRING() ) ); - } - - @Override - public void enterJobUsesPropLinkInput( @NotNull DaxParser.JobUsesPropLinkInputContext ctx ) { - jobUses.setLinkInput(); - } - - @Override - public void enterJobUsesPropLinkOutput( @NotNull DaxParser.JobUsesPropLinkOutputContext ctx ) { - jobUses.setLinkOutput(); - } - - @Override - public void enterJobUsesPropExecutable( @NotNull DaxParser.JobUsesPropExecutableContext ctx ) { - jobUses.setExecutable( true ); - } - - @Override - public void enterJobUsesPropOptionalTrue( @NotNull DaxParser.JobUsesPropOptionalTrueContext ctx ) { - jobUses.setOptional( true ); - } - - - @Override - public void enterChild( @NotNull DaxParser.ChildContext ctx ) { - - String id; - - id = getString( ctx.STRING() ); - - job = idJobMap.get( id ); - if( job == null ) - throw new NullPointerException( "Could not retrieve referenced child job." ); - } - - @Override - public void exitChild( @NotNull DaxParser.ChildContext ctx ) { - job = null; - } - - @Override - public void enterParent( @NotNull DaxParser.ParentContext ctx ) { - - DaxJob parent; - String id; - - id = getString( ctx.STRING() ); - - parent = idJobMap.get( id ); - if( job == null ) - throw new NullPointerException( "Could not retrieve referenced parent job." ); - - parent.addChild( job ); - job.addParent( parent ); - } - - public TopLevelContext toTopLevelContext() { - - TopLevelContext tlc; - CompoundExpr ce; - ApplyExpr applyExpr; - - tlc = new TopLevelContext(); - ce = new CompoundExpr(); - tlc.addTarget( ce ); - - - // gather input contract - for( DaxFilename input : fileList ) - tlc.putAssign( input.getNameExpr(), new CompoundExpr( new StringExpr( input.getFile() ) ) ); - - // gather jobs - for( DaxJob j : idJobMap.values() ) { - - for( DaxFilename f : j.getOutputJobUsesSet() ) { - - applyExpr = j.getApplyExpr( f, fileList ); - tlc.putAssign( f.getNameExpr(), new CompoundExpr( applyExpr ) ); - } - } - - // gather output contract - for( DaxFilename output : outputList ) - ce.addSingleExpr( output.getNameExpr() ); - - return tlc; - } - - public static int getInt( TerminalNode node ) { - return Integer.valueOf( getString( node ) ); - } - - public static String getString( TerminalNode node ) { - - String s; - - s = node.getText(); - s = s.substring( 1, s.length()-1 ); - - return s; - } - - public static URL getUrl( TerminalNode node ) { - - try { - return new URL( getString( node ) ); - } - catch( MalformedURLException e ) { - throw new RuntimeException( e ); - } - } - - @Override - public void reportAmbiguity( Parser arg0, DFA arg1, int arg2, int arg3, - boolean arg4, BitSet arg5, ATNConfigSet arg6 ) { - - if( log.isDebugEnabled() ) - log.debug( "Ambiguity detected." ); - - } - - @Override - public void reportAttemptingFullContext( Parser arg0, DFA arg1, int arg2, - int arg3, BitSet arg4, ATNConfigSet arg5 ) { - - if( log.isDebugEnabled() ) - log.debug( "Attempting full context." ); - - } - - @Override - public void reportContextSensitivity( Parser arg0, DFA arg1, int arg2, - int arg3, int arg4, ATNConfigSet arg5) { - - if( log.isDebugEnabled() ) - log.debug( "Context sensitivity detected." ); - } - - @Override - public void syntaxError( Recognizer arg0, Object offendingSymbol, int line, - int charPositionInLine, String msg, RecognitionException arg5 ) { - - String near; - - near = null; - if( offendingSymbol != null ) - near = ( ( Token )offendingSymbol ).getText(); - - throw new ParseException( line, charPositionInLine, near, msg ); - } - -} diff --git a/cuneiform-addons/cuneiform-htcondorcre/.gitignore b/cuneiform-addons/cuneiform-htcondorcre/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-addons/cuneiform-htcondorcre/InstallHTCondor.md b/cuneiform-addons/cuneiform-htcondorcre/InstallHTCondor.md deleted file mode 100644 index 85912a9..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/InstallHTCondor.md +++ /dev/null @@ -1,21 +0,0 @@ -# Setup HTCondor Master Node -This project uses the HTCondor distributed server software. If you have no access to a HTCondor pool, you need to setup your own server, starting with a master node. -Download the HTCondor binaries or source code from [the official website](http://research.cs.wisc.edu/htcondor/downloads/). -This project has been developed with version 8.2.4-281588 (Nov 12, 2014) for Unix. - -## Preparation -The [install guide](http://research.cs.wisc.edu/htcondor/manual/v8.2/3_2Installation_Start.html) suggests a few decisions before the installation. Here are the some decisions made for this project: -1. What machines should be allowed to submit jobs? - Use the default settings for testing. The default setting will allow no machine to connect and submit jobs. If you need this to be changed, referr to the install guide to see how. -2. Will HTCondor run as root or not? - Yes. -3. Will you have a Unix user named condor, and will its home directory be shared? - Yes. - - - - -## Configuration -1. Edit the global config file condor_config to include the command "ALLOW_WRITE = */*" -2. Add the command "UID_DOMAIN = $(FULL_HOSTNAME)" to ensure that the jobs are not run as user "nobody", wich would result in permission problems - -## Start HTCondor - -Run the condor_master command with root access and check with "ps -ef | egrep condor_" if it worked (check part 3.2.4 "Starting HTCondor Under Unix After Installation" in the installation guide to see what result to expect). Your server should be ready to work with cuneiform now. diff --git a/cuneiform-addons/cuneiform-htcondorcre/README.md b/cuneiform-addons/cuneiform-htcondorcre/README.md deleted file mode 100644 index 5a2dde5..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/README.md +++ /dev/null @@ -1,11 +0,0 @@ -htcondorcre -=========== - -The HTCondorCre extends the [Cuneiform](https://github.com/joergen7/cuneiform) functional language to run on HTCondor Servers (http://research.cs.wisc.edu/htcondor/). Cuneiform is a workflow specification language which makes it easy to integrate heterogeneous tools and libraries and exploit data parallelism. This extension creates another execution enviroment for the language, beside [Hi-WAY](https://github.com/marcbux/Hi-WAY), wich is used to run Cuneiform on Hadoop YARN. - - -#Attention - -The code is under active developement and not executable yet. It still lacks the ability to check the status of the job after it has been submitted to HTCondor and will not recieve the output files. - - diff --git a/cuneiform-addons/cuneiform-htcondorcre/htcondor_settings/condor_config b/cuneiform-addons/cuneiform-htcondorcre/htcondor_settings/condor_config deleted file mode 100644 index 90b78b9..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/htcondor_settings/condor_config +++ /dev/null @@ -1,100 +0,0 @@ -###################################################################### -## -## condor_config -## -## This is the global configuration file for condor. This is where -## you define where the local config file is. Any settings -## made here may potentially be overridden in the local configuration -## file. KEEP THAT IN MIND! To double-check that a variable is -## getting set from the configuration file that you expect, use -## condor_config_val -v -## -## condor_config.annotated is a more detailed sample config file -## -## Unless otherwise specified, settings that are commented out show -## the defaults that are used if you don't define a value. Settings -## that are defined here MUST BE DEFINED since they have no default -## value. -## -###################################################################### - -## Where have you installed the bin, sbin and lib condor directories? -RELEASE_DIR = /usr - -## Where is the local condor directory for each host? This is where the local config file(s), logs and -## spool/execute directories are located. this is the default for Linux and Unix systems. -LOCAL_DIR = /var - -## Where is the machine-specific local config file for each host? -LOCAL_CONFIG_FILE = /etc/condor/condor_config.local -## If your configuration is on a shared file system, then this might be a better default -#LOCAL_CONFIG_FILE = $(RELEASE_DIR)/etc/$(HOSTNAME).local -## If the local config file is not present, is it an error? (WARNING: This is a potential security issue.) -REQUIRE_LOCAL_CONFIG_FILE = false - -## The normal way to do configuration with RPMs is to read all of the -## files in a given directory that don't match a regex as configuration files. -## Config files are read in lexicographic order. -LOCAL_CONFIG_DIR = /etc/condor/config.d -#LOCAL_CONFIG_DIR_EXCLUDE_REGEXP = ^((\..*)|(.*~)|(#.*)|(.*\.rpmsave)|(.*\.rpmnew))$ - -## Use a host-based security policy. By default CONDOR_HOST and the local machine will be allowed -use SECURITY : HOST_BASED -## To expand your condor pool beyond a single host, set ALLOW_WRITE to match all of the hosts -ALLOW_ADMINISTRATOR = $(CONDOR_HOST) -ALLOW_OWNER = $(FULL_HOSTNAME), $(ALLOW_ADMINISTRATOR) -ALLOW_READ = * -ALLOW_WRITE = * -ALLOW_NEGOTIATOR = $(COLLECTOR_HOST) -ALLOW_NEGOTIATOR_SCHEDD = $(COLLECTOR_HOST), $(FLOCK_NEGOTIATOR_HOSTS) -ALLOW_WRITE_COLLECTOR = $(ALLOW_WRITE), $(FLOCK_FROM) -ALLOW_WRITE_STARTD = $(ALLOW_WRITE), $(FLOCK_FROM) -ALLOW_READ_COLLECTOR = $(ALLOW_READ), $(FLOCK_FROM) -ALLOW_READ_STARTD = $(ALLOW_READ), $(FLOCK_FROM) -ALLOW_CLIENT = * -## FLOCK_FROM defines the machines that grant access to your pool via flocking. (i.e. these machines can join your pool). -#FLOCK_FROM = -## FLOCK_TO defines the central managers that your schedd will advertise itself to (i.e. these pools will give matches to your schedd). -#FLOCK_TO = condor.cs.wisc.edu, cm.example.edu - -##-------------------------------------------------------------------- -## Values set by the debian patch script: -##-------------------------------------------------------------------- - -## For Unix machines, the path and file name of the file containing -## the pool password for password authentication. -#SEC_PASSWORD_FILE = $(LOCAL_DIR)/lib/condor/pool_password - -## Pathnames -RUN = $(LOCAL_DIR)/run/condor -LOG = $(LOCAL_DIR)/log/condor -LOCK = $(LOCAL_DIR)/lock/condor -SPOOL = $(LOCAL_DIR)/lib/condor/spool -EXECUTE = $(LOCAL_DIR)/lib/condor/execute -BIN = $(RELEASE_DIR)/bin -LIB = $(RELEASE_DIR)/lib/condor -INCLUDE = $(RELEASE_DIR)/include/condor -SBIN = $(RELEASE_DIR)/sbin -LIBEXEC = $(RELEASE_DIR)/lib/condor/libexec -SHARE = $(RELEASE_DIR)/share/condor - -PROCD_ADDRESS = $(RUN)/procd_pipe - -## What machine is your central manager? - -CONDOR_HOST = $(FULL_HOSTNAME) - -## Network domain parameters: -## Internet domain of machines sharing a common UID space. If your -## machines don't share a common UID space, set it to -## UID_DOMAIN = $(FULL_HOSTNAME) -## to specify that each machine has its own UID space. -## An administrator could set UID_DOMAIN to *. -## This will match all domains, but it is a gaping security hole. It is not recommended. - -UID_DOMAIN = $(FULL_HOSTNAME) - -## This macro determines what daemons the condor_master will start and keep its watchful eyes on. -## The list is a comma or space separated list of subsystem names - -DAEMON_LIST = COLLECTOR, MASTER, NEGOTIATOR, SCHEDD, STARTD \ No newline at end of file diff --git a/cuneiform-addons/cuneiform-htcondorcre/pom.xml b/cuneiform-addons/cuneiform-htcondorcre/pom.xml deleted file mode 100644 index 9a8c030..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/pom.xml +++ /dev/null @@ -1,71 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform-addons - 2.0.4-RELEASE - - cuneiform-htcondorcre - - - - Björn Groß - grossbjo@informatik.hu-berlin.de - Humboldt-Universität zu Berlin - https://www.hu-berlin.de/ - - - Jörgen Brandt - brandjoe@informatik.hu-berlin.de - Humboldt-Universität zu Berlin - https://www.hu-berlin.de/ - - - Ulf Leser - leser@informatik.hu-berlin.de - Humboldt-Universität zu Berlin - https://www.hu-berlin.de/ - - - - - - de.hu-berlin.wbi.cuneiform - cuneiform-core - ${project.version} - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - - - \ No newline at end of file diff --git a/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/CondorCreActor.java b/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/CondorCreActor.java deleted file mode 100644 index 60070f1..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/CondorCreActor.java +++ /dev/null @@ -1,591 +0,0 @@ -/******************************************************************************* - * The HTCondorCRE provides a execution base for the functional - * workflow language Cuneiform, using the HTCondor software - * (formerly known as 'Condor'). - * - * List of Contributors: - * - * Björn Groß (HU Berlin) - * Jörgen Brandt (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.htcondorcre; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.cre.BaseCreActor; -import de.huberlin.wbi.cuneiform.core.cre.TicketReadyMsg; -import de.huberlin.wbi.cuneiform.core.invoc.Invocation; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketFailedMsg; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketFinishedMsg; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.*; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class CondorCreActor extends BaseCreActor { - public static final String VERSION = "2015-03-05-4"; - - private CondorWatcher watcher; - - private static final String PATH_CENTRALREPO = "repo"; - private static final int WAIT_INTERVAL = 100; - private static final int MAX_TRIALS = 4; - - private static String verbose = null; - private static String unused = null; - - private final Path buildDir; - private final Path centralRepo; - private ExecutorService executor; - - private long maxTransferBytes = Unit.G.bytes; - - //htcondor max file transfer size, T isn't even here because transfering Terabytes of data makes no sense. - public static enum Unit{ - G(1073741824), - M(1048576), - K(1024); - - private final long bytes; - - Unit(long bytes){ - this.bytes = bytes; - } - - protected long convert(double numUnits){ - return Math.round(numUnits * this.bytes); - } - - protected static long convertToBytes(String in){ - try{ - return Long.valueOf(in); - } - catch(Exception e) { - Unit u = valueOf(in.substring(in.length() - 1).toUpperCase()); - double numUnits = Double.valueOf(in.substring(0, in.length() - 1)); - return u.convert(numUnits); - } - } - } - - public CondorCreActor(Path buildDir, String maxTransferBytes) throws Exception{ - this(buildDir); - this.maxTransferBytes = Unit.convertToBytes(maxTransferBytes); - } - - public CondorCreActor(Path buildDir) throws IOException { - // creates an actor for default jobs - if (log.isDebugEnabled()) { - log.debug("Condor CRE actor created. Version " + VERSION); - } - - if (buildDir == null) { - throw new NullPointerException("Build directory must not be null."); - } - if (!Files.exists(buildDir)) { - throw new RuntimeException("Build directory does not exist."); - } - if (!Files.isDirectory(buildDir)) { - throw new RuntimeException("Directory type expected."); - } - this.buildDir = buildDir; - centralRepo = buildDir.resolve(PATH_CENTRALREPO); - if (!Files.exists(centralRepo)) { - Files.createDirectories(centralRepo); - } - - - executor = Executors.newCachedThreadPool(); - // create a new condor watcher to watch for job status changes - watcher = new CondorWatcher(this); - - executor.submit(watcher); - executor.shutdown(); - } - - /** - * Decides whether the condor job should be submitted using the verbose - * flag. The default option is false. - * - * Verbose output - display the created job ClassAd - * - * @param value - * Adds the verbose flag if true, removes the flag if false - */ - public void setVerbose(boolean value) { - if (value) { - verbose = "-verbose"; - } else { - verbose = null; - } - } - - /** - * Sets the unused flag settings for the condor job. As a default, causes no - * warnings to be issued about user-defined macros not being used within the - * submit description file. The meaning reverses (toggles) when the - * configuration variable WARN_ON_UNUSED_SUBMIT_FILE_MACROS is set to the - * non default value of False. Printing the warnings can help identify - * spelling errors of submit description file commands. The warnings are - * sent to stderr. - * - * @param value - * True to activate the flag, false to delete the flag. - */ - public void setUnused(boolean value) { - if (value) { - unused = "-unused"; - } else { - unused = null; - } - } - - /** - * Used for status messages received from the watcher - * - * @param msg - */ - protected void processMsg(StatusMessage msg) { - - Actor sender; - TicketSrcActor ticketSrc; - Ticket ticket; - - if (log.isDebugEnabled()) { - log.debug("CondorCRE received status message with status code " - + msg.getStatusCode()); - } - - sender = msg.getSender(); - if (sender != watcher) { - throw new RuntimeException( - "Status message source different from designated watcher."); - } - - ticket = msg.getTicket(); - ticketSrc = (TicketSrcActor) msg.getOriginalSender(); - Set entry = this.gatherReport(msg); - - if (msg.getStatusCode() == StatusMessage.CODEJobTerminated) { - // create link in central data repository - Invocation invoc = Invocation.createInvocation(ticket); - Path location = buildDir.resolve(String.valueOf(invoc.getTicketId())); - - try{ - invoc.evalReport(entry); - if (log.isDebugEnabled()) { - // for testing purpose only, show the first staged file - for( NameExpr nameExpr : ticket.getOutputList()){ - log.debug("Ticket "+ticket.getTicketId()+" has at least one output file: "+nameExpr.getId()); - //only display the first file - log.debug("Total number of output files: "+ticket.getOutputList().size()); - break; - } - } - - for( String f : invoc.getStageOutList() ) { - - Path srcPath = location.resolve( f ); - Path destPath = centralRepo.resolve( f ); - - if( Files.exists( destPath ) ) - continue; - - if( log.isTraceEnabled() ) - log.trace( "Creating link from "+srcPath+" to "+destPath+"." ); - - Files.createSymbolicLink( destPath, srcPath ); - } - }catch (Exception e){ - if (log.isTraceEnabled()) { - log.trace("Something went wrong."); - } - if (log.isDebugEnabled()) { - // Show full stacktrace - log.debug(e.getMessage()); - } - ticketSrc.sendMsg(new TicketFailedMsg(this, ticket, e, null, null, null)); - } - - //notify of finished job - ticketSrc.sendMsg(new TicketFinishedMsg(this, ticket, entry)); - - return; - } - - if (msg.getStatusCode() == StatusMessage.CODEJobAborted) { - ticketSrc.sendMsg(new TicketFailedMsg(this, ticket, null, null, - null, null)); - - return; - } - - } - - @Override - protected void processMsg(TicketReadyMsg msg) { - - Actor sender; - TicketSrcActor ticketSrc; - Ticket ticket; - - sender = msg.getSender(); - if (!(sender instanceof TicketSrcActor)) { - throw new RuntimeException("Ticket source actor expected."); - } - - ticketSrc = (TicketSrcActor) sender; - - ticket = msg.getTicket(); - - if (!ticket.isNormal()) - throw new RuntimeException("Ticket " + ticket.getTicketId() - + ": Trying to evaluate ticket that is not ready."); - - if (ticket.isEvaluated()) - throw new RuntimeException( - "Ticket " - + ticket.getTicketId() - + ": Trying to evaluate ticket that has already been evaluated."); - - try { - submitJob(ticketSrc, ticket); - } catch (InterruptedException e) { - - if (log.isTraceEnabled()) - log.trace("CondorCRE has been interrupted."); - } catch (Exception e) { - Exception ex = e; - if (log.isTraceEnabled()) { - log.trace("Something went wrong."); - } - if (log.isDebugEnabled()) { - // Show full stacktrace - log.debug(ex.getMessage()); - } - - ticketSrc.sendMsg(new TicketFailedMsg(this, ticket, ex, null, null, - null)); - - } - - } - - @Override - protected void shutdown() { - executor.shutdownNow(); - } - - private Set gatherReport(StatusMessage msg) { - Set report; - JsonReportEntry entry; - Charset cs; - String line; - Invocation invoc = Invocation.createInvocation(msg.getTicket()); - Path location = buildDir.resolve(String.valueOf(invoc.getTicketId())); - Path reportFile = location.resolve(Invocation.REPORT_FILENAME); - - report = new HashSet<>(); - cs = Charset.forName( "UTF-8" ); - try (BufferedReader reader = Files.newBufferedReader(reportFile, cs)) { - - while ((line = reader.readLine()) != null) { - - line = line.trim(); - - if (line.isEmpty()) - continue; - - entry = new JsonReportEntry(line); - - // If the report comes from the hard cache then the run id - // is different from the run id of this invocation. This is - // corrected here. - entry.setRunId(invoc.getRunId()); - - report.add(entry); - } - invoc.evalReport(report); - - } catch (Exception e) { - if (log.isDebugEnabled()) { - log.debug("Exception while gathering json report data: " + e.getMessage() + "."); - } - } - - return report; - - } - - /** - * Submits a job to condor - */ - private void submitJob(TicketSrcActor ticketSrc, Ticket ticket) - throws Exception { - - if (ticketSrc == null) { - throw new NullPointerException("Ticket source must not be null."); - } - if (ticket == null) { - throw new NullPointerException("Ticket must not be null."); - } - - // Setup - Invocation invoc = Invocation.createInvocation(ticket); - Path location = buildDir.resolve(String.valueOf(invoc.getTicketId())); - Files.createDirectories( location ); - if (log.isDebugEnabled()) { - log.debug("Build directory for ticket: " + location.toString()); - } - Path submitFile = location.resolve("cfsubmitfile"); - String script = invoc.toScript(); - Charset cs = Charset.forName("UTF-8"); - Path scriptFile = invoc.getExecutablePath( location ); - Path reportFile = location.resolve(Invocation.REPORT_FILENAME); - - /** - * Log created by the condor job, used to monitor the job. Each job - * should have it's own log file for monitoring */ - java.util.Date date = new java.util.Date(); - Path cjLogFile = location.resolve((date.getTime()) + "cjlog.txt"); - - if (log.isDebugEnabled()) { - log.debug("Starting up condor_submit for ticket " - + invoc.getTicketId() + "."); - } - - // add input files - Set inputs = new HashSet<>(); - long total_size = 0; - for (String filename : invoc.getStageInList()) { - if( filename.charAt( 0 ) == '/' ){ - throw new UnsupportedOperationException( "Absolute path encountered '"+filename+"'." ); - } - Path srcPath = centralRepo.resolve( filename ); - Path destPath = location.resolve( filename ); - Path callLocation = Paths.get( System.getProperty( "user.dir" ) ); - - if( !Files.exists( srcPath ) ) { - - srcPath = callLocation.resolve( filename ); - if( log.isTraceEnabled() ) - log.trace( "Resolving relative path '"+srcPath+"'." ); - } - else - - if( log.isTraceEnabled() ) - log.trace( "Resolving path to central repository '"+srcPath+"'." ); - - if( log.isTraceEnabled() ) - log.trace( "Trying to create symbolic link from '"+srcPath+"' to '"+destPath+"'." ); - - if( !Files.exists( destPath.getParent() ) ) - Files.createDirectories( destPath.getParent() ); - - Files.createSymbolicLink( destPath, srcPath ); - //add the path to the set to insert it in the submitfile later on - inputs.add(destPath.toString()); - - Path target = srcPath; - int infLoopGuard = 10; // I think symbolic links can get self-referential - while(Files.isSymbolicLink(target) && infLoopGuard > 0){ - infLoopGuard--; - target = Files.readSymbolicLink(target); -; } - total_size += Files.size(target); - } - - try { - Files.createFile(scriptFile, - PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString("rwxr-xrwx" ) ) ); - if (log.isDebugEnabled()) { - log.debug("Scriptfile for ticket " + invoc.getTicketId() + " has successfully been created at "+scriptFile.toString()); - } - scriptFile.toFile().setWritable(true, false); - } catch (FileAlreadyExistsException faee) { - // if file already exists do nothing - if (log.isDebugEnabled()) { - log.debug("Scriptfile for ticket " + invoc.getTicketId() + " already exists."); - } - } - - // write executable script - try (BufferedWriter writer = Files.newBufferedWriter(scriptFile, cs, - StandardOpenOption.CREATE)) { - writer.write(script); - } - - // write executable log entry - try (BufferedWriter writer = Files.newBufferedWriter(reportFile, cs, - StandardOpenOption.CREATE)) { - writer.write(ticket.getExecutableLogEntry().toString()); - writer.write('\n'); - } - - try { - Files.createFile(submitFile, PosixFilePermissions - .asFileAttribute(PosixFilePermissions - .fromString("rwxr-x---"))); - } catch (FileAlreadyExistsException faee) { - if (log.isDebugEnabled()) { - log.debug("submitFile " + submitFile.toString() - + " already exists."); - } - } - - Path condorError = location.resolve("condor_stderr.txt"); - Path condorOutput = location.resolve("condor_stdout.txt"); - try { - condorError = Files.createFile(condorError, PosixFilePermissions - .asFileAttribute(PosixFilePermissions - .fromString("rwxr-xrw-"))); - condorError.toFile().setWritable(true, false); - condorOutput = Files.createFile(condorOutput, PosixFilePermissions - .asFileAttribute(PosixFilePermissions - .fromString("rwxr-xrw-"))); - condorOutput.toFile().setWritable(true, false); - } catch (FileAlreadyExistsException faee) { - if (log.isDebugEnabled()) { - log.debug("condorError or condorOutput for "+ ticket.getTicketId() +" already exists."); - } - } - - String universe="vanilla"; - String should_transfer_files="YES"; - String when_to_transfer_output = "when_to_transfer_output = ON_EXIT \n"; - if (total_size >= maxTransferBytes){ - log.info("Total size of files to be transferred exceeds the max transfer limit of " + maxTransferBytes + ". Running in the local universe..."); - universe="local"; - should_transfer_files="NO"; - when_to_transfer_output = "\n"; - } - - try (BufferedWriter writer = Files.newBufferedWriter(submitFile, cs, - StandardOpenOption.CREATE)) { - // name of the executable script - writer.write("executable = " + scriptFile.toString()); - writer.write('\n'); - writer.write("universe = " + universe); - writer.write('\n'); - writer.write("run_as_owner = True"); - writer.write('\n'); - writer.write("log = " + cjLogFile.toString()); - writer.write('\n'); - writer.write("output = " + condorOutput.toString()); - writer.write('\n'); - writer.write("error = " + condorError.toString()); - writer.write('\n'); - //TODO: Transfer files or not? - writer.write("should_transfer_files = " + should_transfer_files + " \n"); - writer.write(when_to_transfer_output); - // inputfiles - if (!inputs.isEmpty() && total_size < maxTransferBytes) { - writer.write("transfer_input_files = "); - boolean successor = false; - for (String file : inputs) { - if(successor){ - writer.write("," ); - }else{ - successor = true; - } - writer.write(file); - } - // TODO: is the ',' causing problems? - writer.write('\n'); - } - // at last add the job to the queue - writer.write('\n'); - writer.write("initialdir = "+location.toString()); - writer.write('\n'); - writer.write("queue"); - } - // run script - // TODO: Add command line arguments here - ArrayList command = new ArrayList(); - command.add("condor_submit"); - command.add(submitFile.toString()); - if (verbose != null) { - command.add(verbose); - } - ProcessBuilder processBuilder = new ProcessBuilder( - command.toArray(new String[command.size()])); - /* - * Sets this process builder's working directory. - * Subprocesses subsequently started by this object's start() method will - * use this as their working directory. The argument may be null -- this - * means to use the working directory of the current Java process, usually the - * directory named by the system property user.dir, as the working directory - * of the child process. */ - processBuilder.directory(location.toFile()); - - Path stdOutFile = location.resolve( Invocation.STDOUT_FILENAME ); - Path stdErrFile = location.resolve( Invocation.STDERR_FILENAME ); - - processBuilder.redirectOutput( stdOutFile.toFile() ); - processBuilder.redirectError( stdErrFile.toFile() ); - - int trial = 1; - boolean suc = false; - Exception ex = null; - - do { - try { - processBuilder.start(); - suc = true; - } catch (IOException e) { - ex = e; - if (log.isWarnEnabled() && trial > 1) - log.warn("Retrying " + (++trial) + "th time." - + " Waiting " + WAIT_INTERVAL + "ms: " - + e.getMessage()); - Thread.sleep(WAIT_INTERVAL); - } - } while (suc == false && trial <= MAX_TRIALS); - - if (suc == false) { - // Couldn't find condor_submit or another error occured - end - // execution - ticketSrc.sendMsg(new TicketFailedMsg(this, ticket, ex, script, - null, null)); - return; - } - - // worked, add job to the watchlist - StatusMessage sm = new StatusMessage(this, ticket.getRunId(), - cjLogFile, ticket, ticketSrc); - watcher.processMsg(sm); - - } - -} diff --git a/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/CondorWatcher.java b/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/CondorWatcher.java deleted file mode 100644 index 5b82ef6..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/CondorWatcher.java +++ /dev/null @@ -1,136 +0,0 @@ -package de.huberlin.wbi.cuneiform.htcondorcre; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.actormodel.Message; - -import java.io.BufferedReader; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.util.ArrayList; - - -public class CondorWatcher extends Actor { - public static final String VERSION = "2015-02-17-1"; - - private CondorCreActor caller = null; - private ArrayList messages = new ArrayList(); - - private static final int WAIT_INTERVAL = 100; - private static final int MAX_TRIALS = 4; - - //polled in intervall condor_wait für alle angegebenen jobs und sendet nachricht - //an cre, sobald ein job finished oder failed ist - public CondorWatcher(CondorCreActor cre){ - if( log.isDebugEnabled() ){ - log.debug( "Condor Watcher actor created, version "+VERSION ); - } - caller = cre; - } - - - - @Override - protected synchronized void processMsg( Message msg ){ - if( msg instanceof StatusMessage ) { - StatusMessage sm = (StatusMessage) msg; - messages.add(sm); - - debug( "Condor Watcher received a new status message with job path "+sm.getJobpath().toString() ); - - return; - } - - throw new RuntimeException( "Message type not recognized." ); - } - - //executed before processQueue (contains processMsg) call - @Override - protected void preRec(){ - int result; - if(!messages.isEmpty()){ - //will hold all messages that are removable after the loop - ArrayList removable = new ArrayList(); - //check all messages in waitline - for(StatusMessage sm : messages){ - result = checkJobStatus(sm); - if(result == 5 || result == 2 || result == 9){ - debug("Condor Watcher found a finished or aborted job: "+sm.getJobpath()); - StatusMessage answer = new StatusMessage(this, sm.getQueryId(), sm.getJobpath(), sm.getTicket(), sm.getOriginalSender()); - answer.setStatusCode(result); - caller.processMsg(answer); - removable.add(sm); - } - } - //remove finished jobs - for(StatusMessage sm : removable){ - messages.remove(sm); - } - removable.clear(); - } - } - - @Override - protected void shutdown() { - System.exit(0); - } - - private int checkJobStatus(StatusMessage sm){ - String jobfile = sm.getJobpath().toString(); - - int trial = 1; - boolean suc = false; - Exception ex = null; - - do { - try { - FileInputStream in = new FileInputStream(jobfile); - BufferedReader br = new BufferedReader(new InputStreamReader(in)); - String tmp; - int tempId = -1; - int lastIdLine = -2; - /* The following will search for the last state id of the job. - * This assumes that only one job uses the log file, so log files - * for multiple jobs will behave unexpected. - * The last id is the current state id. - */ - while ((tmp = br.readLine()) != null) { - try{ - tempId = Integer.parseInt(tmp.substring(0, 3)); - }catch(NumberFormatException nfe){ - continue; - } - //debug(tmp.substring(0, 3) + " and integer "+tempId); - lastIdLine = tempId; - } - in.close(); - - int shellExitStatus = lastIdLine; - suc = true; - - //debug( "CondorWatcher checked job '"+jobfile+"' and got return status "+shellExitStatus+"." ); - return shellExitStatus; - } - catch( IOException e ) { - ex = e; - if( log.isWarnEnabled() && trial > 1) { - log.warn( "Retrying "+( ++trial )+"th time. Waiting "+WAIT_INTERVAL+"ms: "+ex.getMessage() ); - } - try { - Thread.sleep( WAIT_INTERVAL ); - }catch(Exception exept){ - log.warn( "Wait time canceled due to exception "+exept.getMessage() ); - } - } - } while( suc == false && trial <= MAX_TRIALS ); - - return -1; - } - - private void debug(String message){ - if( log.isDebugEnabled() ){ - log.debug( message ); - } - } - -} diff --git a/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/StatusMessage.java b/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/StatusMessage.java deleted file mode 100644 index a8285ce..0000000 --- a/cuneiform-addons/cuneiform-htcondorcre/src/main/java/de/huberlin/wbi/cuneiform/htcondorcre/StatusMessage.java +++ /dev/null @@ -1,96 +0,0 @@ -package de.huberlin.wbi.cuneiform.htcondorcre; - -import java.nio.file.Path; -import java.util.UUID; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.actormodel.Message; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class StatusMessage extends Message { - public static final int CODESubmit = 0; - public static final int CODEExecute = 1; - public static final int CODEExecutableError = 2; - public static final int CODECheckpointed = 3; - public static final int CODEJobEvicted = 4; - public static final int CODEJobTerminated = 5; - public static final int CODEImageSize = 6; - public static final int CODEShadowException = 7; - public static final int CODEGeneric = 8; - public static final int CODEJobAborted = 9; - public static final int CODEJobSuspended = 10; - public static final int CODEJobUnsuspended = 11; - public static final int CODEJobHeld = 12; - public static final int CODEJobReleased = 13; - public static final int CODENodeExecute = 14; - public static final int CODENodeTerminated = 15; - public static final int CODEPostScriptTerminated = 16; - public static final int CODEGlobusSubmit = 17; - public static final int CODEGlobusSubmitFailed = 18; - public static final int CODEGlobusResourceUp = 19; - public static final int CODEGlobusResourceDown = 20; - public static final int CODERemoteError = 21; - - private final Path jobpath; - private final Actor originalSender; - private final UUID queryId; - private final Ticket ticket; - private int status = 0; - - public StatusMessage(Actor sender, UUID queryId, Path jobpath, Ticket ticket, Actor originalSender) { - super(sender); - - if( jobpath == null ){ - throw new NullPointerException( "Jobpath must not be null." ); - } - - if( queryId == null ){ - throw new NullPointerException( "Run ID must not be null." ); - } - - if( ticket == null ){ - throw new NullPointerException( "Ticket must not be null." ); - } - - if( originalSender == null ){ - throw new NullPointerException( "Original sender must not be null." ); - } - - this.jobpath = jobpath; - this.queryId = queryId; - this.ticket = ticket; - this.originalSender = originalSender; - } - - public Path getJobpath(){ - return this.jobpath; - } - - public Actor getOriginalSender(){ - return this.originalSender; - } - - public int getStatusCode(){ - return this.status; - } - - public void setStatusCode(int status){ - if(status >= 0 && status <= 21){ - this.status = status; - } - } - - public UUID getQueryId() { - return this.queryId; - } - - public Ticket getTicket() { - return this.ticket; - } - - @Override - public String toString() { - return "{ statusMessage, \""+queryId+"\", "+jobpath+"\" }"; - } - -} diff --git a/cuneiform-addons/cuneiform-logview/.gitignore b/cuneiform-addons/cuneiform-logview/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-addons/cuneiform-logview/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-addons/cuneiform-logview/pom.xml b/cuneiform-addons/cuneiform-logview/pom.xml deleted file mode 100644 index 810bfe4..0000000 --- a/cuneiform-addons/cuneiform-logview/pom.xml +++ /dev/null @@ -1,63 +0,0 @@ - - 4.0.0 - - cuneiform-logview - - de.hu-berlin.wbi.cuneiform - cuneiform-addons - 2.0.4-RELEASE - - - - UTF-8 - - - - - - de.hu-berlin.wbi.cuneiform - cuneiform-core - ${project.version} - - - - jfree - jfreechart - 1.0.13 - - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - - - diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/common/Visualizable.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/common/Visualizable.java deleted file mode 100644 index fc984bf..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/common/Visualizable.java +++ /dev/null @@ -1,15 +0,0 @@ -package de.huberlin.wbi.cuneiform.logview.common; - -import javax.swing.JPanel; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; - -public abstract class Visualizable extends JPanel { - - private static final long serialVersionUID = -1940465202761413892L; - - public abstract void register( JsonReportEntry entry ) throws Exception; - public abstract void clear(); - public abstract void updateView(); - -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/graphview/CfEdge.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/graphview/CfEdge.java deleted file mode 100644 index 991ac5e..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/graphview/CfEdge.java +++ /dev/null @@ -1,53 +0,0 @@ -package de.huberlin.wbi.cuneiform.logview.graphview; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -public class CfEdge { - - private final String title; - private Long producer; - private final Set consumerIdSet; - - public CfEdge( String title ) { - this( title, null ); - } - - public CfEdge( String title, Long producer ) { - - if( title == null ) - throw new NullPointerException( "Vertex title must not be null." ); - - if( title.isEmpty() ) - throw new RuntimeException( "Vertex title must not be empty." ); - - this.title = title; - this.producer = producer; - consumerIdSet = new HashSet<>(); - } - - public void addConsumerId( long invocId ) { - consumerIdSet.add( invocId ); - } - - public boolean isConsumerIdSetEmpty() { - return consumerIdSet.isEmpty(); - } - - public Set getConsumerIdSet() { - return Collections.unmodifiableSet( consumerIdSet ); - } - - public String getTitle() { - return title; - } - - public boolean hasProducer() { - return producer != null; - } - - public Long getProducerId() { - return producer; - } -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/graphview/GraphView.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/graphview/GraphView.java deleted file mode 100644 index 0fdfb94..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/graphview/GraphView.java +++ /dev/null @@ -1,163 +0,0 @@ -package de.huberlin.wbi.cuneiform.logview.graphview; - -import java.awt.BorderLayout; -import java.awt.Font; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; - -import javax.swing.JScrollPane; -import javax.swing.JTextArea; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.logview.common.Visualizable; - -public class GraphView extends Visualizable { - - private static final long serialVersionUID = 9059116172334909768L; - private static final String COLOR_FILE = "darkgoldenrod"; - private static final String COLOR_INVOC = "blue"; - private static final double WIDTH_VERTEX = .01; - private static final double NODESEP = .03; - - private final Map edgeMap; - private final Map> vertexMap; - private final JTextArea dotArea; - - public GraphView() { - - edgeMap = new HashMap<>(); - vertexMap = new HashMap<>(); - - setLayout( new BorderLayout() ); - - dotArea = new JTextArea(); - dotArea.setEditable( false ); - dotArea.setFont( new Font( Font.MONOSPACED, Font.PLAIN, 11 ) ); - add( new JScrollPane( dotArea ), BorderLayout.CENTER ); - } - - @Override - public void register( JsonReportEntry entry ) throws Exception { - - long invocId; - String filename; - CfEdge edge; - String taskName; - Set vertexSet; - - if( !entry.hasInvocId() ) - return; - - invocId = entry.getInvocId(); - - taskName = entry.getTaskName(); - if( taskName == null ) - taskName = "[lambda]"; - - vertexSet = vertexMap.get( taskName ); - if( vertexSet == null ) { - vertexSet = new HashSet<>(); - vertexMap.put( taskName, vertexSet ); - } - - vertexSet.add( invocId ); - - if( entry.isKeyFileSizeStageIn() ) { - - filename = entry.getFile(); - - edge = edgeMap.get( filename ); - if( edge == null ) { - edge = new CfEdge( filename ); - edgeMap.put( filename, edge ); - } - - edge.addConsumerId( invocId ); - } - else if( entry.isKeyFileSizeStageOut() ) { - - filename = entry.getFile(); - edgeMap.put( filename, new CfEdge( filename, invocId ) ); - } - } - - @Override - public void clear() { - vertexMap.clear(); - edgeMap.clear(); - } - - @Override - public void updateView() { - - CfEdge edge; - StringBuffer buf; - int sg; - long producerId; - int hc; - - buf = new StringBuffer(); - buf.append( "digraph G {\n nodesep=" ).append( NODESEP ).append( ";\n" ); - - - // add vertices to subgraphs - sg = 0; - for( String taskName : vertexMap.keySet() ) { - - buf.append( " subgraph cluster" ).append( sg++ ).append( " {\n" ); - buf.append( " label=\"" ).append( taskName ).append( "\";\n" ); - - for( Long invocId : vertexMap.get( taskName ) ) - buf.append( " node_invoc" ).append( invocId ) - .append( " [shape=box,color=" ).append( COLOR_INVOC ) - .append( ",width=" ).append( WIDTH_VERTEX ).append( ",label=\"\"];\n" ); - - buf.append( " }\n" ); - } - - for( String filename : edgeMap.keySet() ) { - - edge = edgeMap.get( filename ); - hc = Math.abs( filename.hashCode() ); - - - if( !edge.hasProducer() ) { - - // edge is an input file - buf.append( " node_infile" ).append( hc ) - .append( " [shape=box,color=" ).append( COLOR_FILE ) - .append( ",width=" ).append( WIDTH_VERTEX ).append( ",label=\"\"];\n" ); - - for( long consumerId : edge.getConsumerIdSet() ) - - buf.append( " node_infile" ).append( hc ).append( ":s -> " ) - .append( "node_invoc" ).append( consumerId ).append( ":n;\n" ); - } - else { - - // edge has a producer - producerId = edge.getProducerId(); - - if( edge.isConsumerIdSetEmpty() ) - buf.append( " node_outfile" ).append( hc ).append( " [shape=box,color=" ).append( COLOR_FILE ) - .append( ",width=" ).append( WIDTH_VERTEX ).append( ",label=\"\"];\n" ) - .append( " node_invoc" ).append( producerId ) - .append( ":s -> node_outfile" ).append( hc ).append( ":n;\n" ); - - else - for( long consumerId : edge.getConsumerIdSet() ) - - buf.append( " node_invoc" ).append( producerId ).append( ":s -> " ) - .append( "node_invoc" ).append( consumerId ).append( ":n;\n" ); - - } - } - - buf.append( "}\n" ); - - dotArea.setText( buf.toString() ); - } - -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/main/Main.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/main/Main.java deleted file mode 100644 index d5eea5f..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/main/Main.java +++ /dev/null @@ -1,118 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.logview.main; - - -import java.awt.BorderLayout; -import java.io.BufferedReader; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import javax.swing.JFrame; -import javax.swing.JSplitPane; -import javax.swing.JTabbedPane; -import javax.swing.WindowConstants; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.logview.graphview.GraphView; -import de.huberlin.wbi.cuneiform.logview.parallelismview.ParallelismView; -import de.huberlin.wbi.cuneiform.taskview.TaskBrowser; -import de.huberlin.wbi.cuneiform.taskview.TaskView; - -public class Main { - - public static void main( String[] args ) throws Exception { - - - JFrame frame; - JSplitPane splitPane; - TaskBrowser taskBrowser; - TaskView taskView; - Path logPath; - String line; - JTabbedPane tabbedPane; - ParallelismView parallelismView; - JsonReportEntry entry; - GraphView graphView; - Path buildPath; - - buildPath = Paths.get( System.getProperty( "user.home" ) ).resolve( ".cuneiform" ); - if( args.length > 0 ) - buildPath = Paths.get( args[ 0 ] ); - - frame = new JFrame( "Cuneiform Log View" ); - - frame.setSize( 1024, 768 ); - frame.setDefaultCloseOperation( WindowConstants.DISPOSE_ON_CLOSE ); - frame.setLayout( new BorderLayout() ); - - - taskView = new TaskView( buildPath ); - taskBrowser = new TaskBrowser( taskView ); - splitPane = new JSplitPane( JSplitPane.HORIZONTAL_SPLIT, taskBrowser, taskView ); - splitPane.setDividerLocation( 200 ); - - parallelismView = new ParallelismView(); - - graphView = new GraphView(); - - tabbedPane = new JTabbedPane(); - tabbedPane.addTab( "Task browser", splitPane ); - tabbedPane.addTab( "Parallelism", parallelismView ); - tabbedPane.addTab( "Invocation graph", graphView ); - - frame.add( tabbedPane, BorderLayout.CENTER ); - - logPath = Paths.get( "/tmp/cuneiform-stat.log" ); - try( BufferedReader reader = Files.newBufferedReader( logPath, Charset.forName( "UTF-8" ) ) ) { - - while( ( line = reader.readLine() ) != null ) { - entry = new JsonReportEntry( line ); - taskBrowser.register( entry ); - parallelismView.register( entry ); - graphView.register( entry ); - } - - } - - taskBrowser.updateView(); - parallelismView.updateView(); - graphView.updateView(); - - - frame.setVisible( true ); - - } -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/parallelismview/ParallelismView.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/parallelismview/ParallelismView.java deleted file mode 100644 index 61b46e8..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/logview/parallelismview/ParallelismView.java +++ /dev/null @@ -1,189 +0,0 @@ -package de.huberlin.wbi.cuneiform.logview.parallelismview; - -import java.awt.BorderLayout; -import java.util.HashMap; -import java.util.Map; - -import org.jfree.chart.ChartFactory; -import org.jfree.chart.ChartPanel; -import org.jfree.chart.JFreeChart; -import org.jfree.chart.plot.PlotOrientation; -import org.jfree.data.xy.DefaultTableXYDataset; -import org.jfree.data.xy.XYSeries; -import org.json.JSONException; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.logview.common.Visualizable; - -public class ParallelismView extends Visualizable { - - private static final long serialVersionUID = -5167316615121741821L; - private static final int NSAMPLE = 1024; - - private long firstBegin; - private long lastBegin; - private long lastDur; - private final Map beginMap; - private final Map durMap; - private final Map taskNameMap; - - public ParallelismView() { - - beginMap = new HashMap<>(); - durMap = new HashMap<>(); - taskNameMap = new HashMap<>(); - clear(); - - setLayout( new BorderLayout() ); - } - - @Override - public void clear() { - - firstBegin = Long.MAX_VALUE; - lastDur = Long.MIN_VALUE; - - beginMap.clear(); - durMap.clear(); - taskNameMap.clear(); - } - - @Override - public void register( JsonReportEntry entry ) throws JSONException { - - long begin, dur; - long invocId; - String taskName; - - if( entry == null ) - throw new NullPointerException( "JSON report entry must not be null." ); - - if( !entry.isKeyInvocTime() ) - return; - - invocId = entry.getInvocId(); - taskName = entry.getTaskName(); - - dur = entry.getValueJsonObj().getLong( JsonReportEntry.LABEL_REALTIME ); - begin = entry.getTimestamp(); - if( begin < firstBegin ) - firstBegin = begin; - else - if( begin+dur > lastBegin+lastDur ) { - lastBegin = begin; - lastDur = dur; - } - - beginMap.put( invocId, begin ); - durMap.put( invocId, dur ); - taskNameMap.put( invocId, taskName ); - } - - @Override - public void updateView() { - - JFreeChart chart; - DefaultTableXYDataset dataset; - Map seriesMap; - String taskName; - int[] cvec; - int i; - long begin, stop, t; - long[] tvec; - long delta; - XYSeries series; - long dur; - String uname; - int udiv; - - seriesMap = new HashMap<>(); - tvec = new long[ NSAMPLE ]; - dur = lastBegin+lastDur-firstBegin; - delta = dur/( NSAMPLE-1 )+1; - - t = firstBegin; - for( i = 0; i < NSAMPLE; i++ ) { - - tvec[ i ] = t; - - for( Long invocId : beginMap.keySet() ) { - - begin = beginMap.get( invocId ); - if( begin > t ) - continue; - - stop = begin+durMap.get( invocId ); - if( stop < t ) - continue; - - taskName = taskNameMap.get( invocId ); - if( taskName == null ) - taskName = "[lambda]"; - - cvec = seriesMap.get( taskName ); - if( cvec == null ) { - cvec = new int[ NSAMPLE ]; - seriesMap.put( taskName, cvec ); - } - - cvec[ i ]++; - - } - - t += delta; - } - - udiv = 3600000*24; - uname = "d"; - - if( dur < 2*3600000*24 ) { - - udiv = 3600000; - uname = "h"; - } - - if( dur < 2*3600000 ) { - - udiv = 60000; - uname = "min"; - } - - if( dur < 2*60000 ) { - - udiv = 1000; - uname = "s"; - } - - if( dur < 2000 ) { - - udiv = 1; - uname = "ms"; - } - - dataset = new DefaultTableXYDataset(); - for( String n : seriesMap.keySet() ) { - - cvec = seriesMap.get( n ); - series = new XYSeries( n, true, false ); - for( i = 0; i < NSAMPLE; i++ ) - series.add( ( ( double )( tvec[ i ]-firstBegin ) )/udiv, cvec[ i ] ); - - dataset.addSeries( series ); - } - - chart = ChartFactory.createStackedXYAreaChart( - null, // title - "Time ["+uname+"]", // xAxisLabel - "N invocations", // yAxisLabel - dataset, // dataset - PlotOrientation.VERTICAL, // orientation - true, // legend - false, // tooltips - true // urls - ); - - add( new ChartPanel( chart ), BorderLayout.CENTER ); - - } - -} \ No newline at end of file diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/FileBrowser.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/FileBrowser.java deleted file mode 100644 index 8084566..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/FileBrowser.java +++ /dev/null @@ -1,156 +0,0 @@ -package de.huberlin.wbi.cuneiform.taskview; - -import java.awt.BorderLayout; -import java.awt.Font; -import java.io.BufferedReader; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; - -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JSplitPane; -import javax.swing.JTextArea; -import javax.swing.JTree; -import javax.swing.ScrollPaneConstants; -import javax.swing.event.TreeSelectionEvent; -import javax.swing.event.TreeSelectionListener; -import javax.swing.tree.DefaultMutableTreeNode; -import javax.swing.tree.DefaultTreeModel; -import javax.swing.tree.TreePath; -import javax.swing.tree.TreeSelectionModel; - -public class FileBrowser extends JPanel implements TreeSelectionListener { - - private static final long serialVersionUID = 3161764896005980182L; - - private final JTree tree; - private final DefaultMutableTreeNode top; - private final DefaultTreeModel treeModel; - private final JTextArea contentArea; - private final Path buildPath; - - public FileBrowser( Path buildPath ) { - - JSplitPane splitPane; - JScrollPane scrollPane; - - if( buildPath == null ) - throw new IllegalArgumentException( "Build path must not be null." ); - - this.buildPath = buildPath; - top = new DefaultMutableTreeNode( "[Container]" ); - treeModel = new DefaultTreeModel( top ); - tree = new JTree( treeModel ); - tree.getSelectionModel().setSelectionMode( TreeSelectionModel.SINGLE_TREE_SELECTION ); - tree.addTreeSelectionListener( this ); - - - setLayout( new BorderLayout() ); - - contentArea = new JTextArea(); - contentArea.setEditable( false ); - contentArea.setFont( new Font( Font.MONOSPACED, Font.PLAIN, 11 ) ); - - scrollPane = new JScrollPane( tree ); - scrollPane.setHorizontalScrollBarPolicy( ScrollPaneConstants.HORIZONTAL_SCROLLBAR_NEVER ); - - splitPane = new JSplitPane( - JSplitPane.HORIZONTAL_SPLIT, - scrollPane, - new JScrollPane ( contentArea ) ); - splitPane.setDividerLocation( 300 ); - - add( splitPane, BorderLayout.CENTER ); - } - - public void blank() { - - int i; - - for( i = top.getChildCount()-1; i >= 0; i-- ) - treeModel.removeNodeFromParent( ( DefaultMutableTreeNode )top.getChildAt( i ) ); - - - - contentArea.setText( null ); - } - - public void setInvocId( long invocId ) { - - Path currentPath; - - try { - - blank(); - currentPath = buildPath.resolve( String.valueOf( invocId ) ); - ls( currentPath, top ); - tree.expandPath( new TreePath( top.getPath() ) ); - } - catch( IOException e ) { - e.printStackTrace(); - } - } - - private void ls( Path currentPath, DefaultMutableTreeNode currentNode ) throws IOException { - - DefaultMutableTreeNode node; - - for( Path path : Files.newDirectoryStream( currentPath ) ) { - - node = new FileMutableTreeNode( path ); - treeModel.insertNodeInto( - node, - currentNode, - treeModel.getChildCount( currentNode ) ); - - if( Files.isDirectory( path ) ) - ls( path, node ); - } - - } - - @Override - public void valueChanged( TreeSelectionEvent e ) { - - JTree t; - DefaultMutableTreeNode node; - int i; - StringBuffer buf; - String line; - - t = ( JTree )e.getSource(); - node = ( DefaultMutableTreeNode )t.getLastSelectedPathComponent(); - - if( node.isLeaf() ) - - if( node != top ) { - - - try( BufferedReader reader = - Files.newBufferedReader( ( ( FileMutableTreeNode )node ).getFilePath(), Charset.forName( "UTF-8" ) ) ) { - - buf = new StringBuffer(); - for( i = 0; i < 1024; i++ ) { - - line = reader.readLine(); - if( line == null ) - break; - - buf.append( line ).append( '\n' ); - } - contentArea.setText( buf.toString() ); - - return; - } - catch( IOException x ) { - contentArea.setText( "[binary]" ); - return; - } - } - - contentArea.setText( null ); - - } -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/FileMutableTreeNode.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/FileMutableTreeNode.java deleted file mode 100644 index 6a16daf..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/FileMutableTreeNode.java +++ /dev/null @@ -1,21 +0,0 @@ -package de.huberlin.wbi.cuneiform.taskview; - -import java.nio.file.Path; - -import javax.swing.tree.DefaultMutableTreeNode; - -public class FileMutableTreeNode extends DefaultMutableTreeNode { - - private static final long serialVersionUID = -3967012888405802072L; - - private Path filePath; - - public FileMutableTreeNode( Path filePath ) { - super( filePath.getFileName() ); - this.filePath = filePath; - } - - public Path getFilePath() { - return filePath; - } -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/InvocationItem.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/InvocationItem.java deleted file mode 100644 index 5c932a7..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/InvocationItem.java +++ /dev/null @@ -1,44 +0,0 @@ -package de.huberlin.wbi.cuneiform.taskview; - -import javax.swing.tree.DefaultMutableTreeNode; - -public class InvocationItem extends DefaultMutableTreeNode { - - private static final long serialVersionUID = -381630579332316845L; - - private final long invocId; - private final String taskName; - private String stdErr; - private String stdOut; - - public InvocationItem( long invocId, String taskName ) { - super( String.valueOf( invocId ) ); - - this.invocId = invocId; - this.taskName = taskName; - } - - public long getInvocId() { - return invocId; - } - - public String getStdOut() { - return stdOut; - } - - public String getStdErr() { - return stdErr; - } - - public String getTaskName() { - return taskName; - } - - public void setStdOut( String stdOut ) { - this.stdOut = stdOut; - } - - public void setStdErr( String stdErr ) { - this.stdErr = stdErr; - } -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/TaskBrowser.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/TaskBrowser.java deleted file mode 100644 index 55cc133..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/TaskBrowser.java +++ /dev/null @@ -1,123 +0,0 @@ -package de.huberlin.wbi.cuneiform.taskview; - -import java.awt.BorderLayout; -import java.util.HashMap; -import java.util.Map; - -import javax.swing.JScrollPane; -import javax.swing.JTree; -import javax.swing.ScrollPaneConstants; -import javax.swing.tree.DefaultMutableTreeNode; -import javax.swing.tree.DefaultTreeModel; -import javax.swing.tree.TreePath; -import javax.swing.tree.TreeSelectionModel; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.logview.common.Visualizable; - -public class TaskBrowser extends Visualizable { - - private static final long serialVersionUID = 2782394011894606586L; - - private final DefaultMutableTreeNode top; - private DefaultMutableTreeNode unnamed; - private final Map invocMap; - private final Map taskMap; - private final JTree tree; - private final DefaultTreeModel treeModel; - - public TaskBrowser( TaskView taskView ) { - - JScrollPane scrollPane; - - if( taskView == null ) - throw new NullPointerException( "Task view must not be null." ); - - invocMap = new HashMap<>(); - taskMap = new HashMap<>(); - - setLayout( new BorderLayout() ); - - top = new DefaultMutableTreeNode( "Cuneiform tasks" ); - - treeModel = new DefaultTreeModel( top ); - tree = new JTree( treeModel ); - tree.getSelectionModel().setSelectionMode( TreeSelectionModel.SINGLE_TREE_SELECTION ); - - scrollPane = new JScrollPane( tree ); - scrollPane.setHorizontalScrollBarPolicy( ScrollPaneConstants.HORIZONTAL_SCROLLBAR_NEVER ); - - add( scrollPane, BorderLayout.CENTER ); - - tree.addTreeSelectionListener( taskView ); - - } - - @Override - public void register( JsonReportEntry entry ) { - - long invocId; - InvocationItem invocItem; - String taskName; - DefaultMutableTreeNode taskItem; - - if( !entry.hasInvocId() ) - return; - - tree.expandPath( new TreePath( top.getPath() ) ); - - - invocId = entry.getInvocId(); - - invocItem = invocMap.get( invocId ); - if( invocItem == null ) { - - - taskName = null; - if( entry.hasTaskname() ) { - - taskName = entry.getTaskName(); - taskItem = taskMap.get( taskName ); - if( taskItem == null ) { - taskItem = new DefaultMutableTreeNode( taskName ); - taskMap.put( taskName, taskItem ); - treeModel.insertNodeInto( taskItem, top, top.getChildCount() ); - top.add( taskItem ); - } - } - else { - - if( unnamed == null ) { - unnamed = new DefaultMutableTreeNode( "[lambda]" ); - treeModel.insertNodeInto( unnamed, top, 0 ); - } - taskItem = unnamed; - } - invocItem = new InvocationItem( invocId, taskName ); - invocMap.put( invocId, invocItem ); - - treeModel.insertNodeInto( invocItem, taskItem, taskItem.getChildCount() ); - } - - if( entry.isKeyInvocStdErr() ) - invocItem.setStdErr( entry.getValueRawString() ); - - if( entry.isKeyInvocStdOut() ) - invocItem.setStdOut( entry.getValueRawString() ); - - } - - @Override - public void clear() { - - int i; - - for( i = treeModel.getChildCount( top )-1; i >= 0; i-- ) - treeModel.removeNodeFromParent( ( DefaultMutableTreeNode )top.getChildAt( i ) ); - } - - @Override - public void updateView() { - // nothing to do - } -} diff --git a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/TaskView.java b/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/TaskView.java deleted file mode 100644 index 05cff4f..0000000 --- a/cuneiform-addons/cuneiform-logview/src/main/java/de/huberlin/wbi/cuneiform/taskview/TaskView.java +++ /dev/null @@ -1,109 +0,0 @@ -package de.huberlin.wbi.cuneiform.taskview; - -import java.awt.BorderLayout; -import java.awt.Font; -import java.nio.file.Path; - -import javax.swing.BoxLayout; -import javax.swing.JLabel; -import javax.swing.JPanel; -import javax.swing.JScrollPane; -import javax.swing.JTabbedPane; -import javax.swing.JTextArea; -import javax.swing.JTextField; -import javax.swing.JTree; -import javax.swing.event.TreeSelectionEvent; -import javax.swing.event.TreeSelectionListener; -import javax.swing.tree.DefaultMutableTreeNode; - -public class TaskView extends JPanel implements TreeSelectionListener { - - private static final long serialVersionUID = -5409910527314838886L; - - private final JTextField invocIdField; - private final JTextField taskNameField; - private final JTextArea stdOutArea; - private final JTextArea stdErrArea; - private final FileBrowser fileBrowser; - - public TaskView( Path buildPath ) { - - JPanel infoBox; - JPanel col; - JTabbedPane tabbedPane; - Font font; - - setLayout( new BorderLayout() ); - - infoBox = new JPanel(); - infoBox.setLayout( new BorderLayout() ); - add( infoBox, BorderLayout.NORTH ); - - col = new JPanel(); - col.setLayout( new BoxLayout( col, BoxLayout.Y_AXIS ) ); - infoBox.add( col, BorderLayout.WEST ); - - col.add( new JLabel( "Invoc. ID:" ) ); - col.add( new JLabel( "Task name:" ) ); - - - col = new JPanel(); - col.setLayout( new BoxLayout( col, BoxLayout.Y_AXIS ) ); - infoBox.add( col, BorderLayout.CENTER ); - - invocIdField = new JTextField(); - invocIdField.setEditable( false ); - taskNameField = new JTextField(); - taskNameField.setEditable( false ); - - col.add( invocIdField ); - col.add( taskNameField ); - - font = new Font( Font.MONOSPACED, Font.PLAIN, 11 ); - - stdOutArea = new JTextArea(); - stdOutArea.setEditable( false ); - stdOutArea.setFont( font ); - stdErrArea = new JTextArea(); - stdErrArea.setEditable( false ); - stdErrArea.setFont( font ); - - fileBrowser = new FileBrowser( buildPath ); - - tabbedPane = new JTabbedPane(); - tabbedPane.addTab( "Stdout", new JScrollPane( stdOutArea ) ); - tabbedPane.addTab( "Stderr", new JScrollPane( stdErrArea ) ); - tabbedPane.addTab( "Container", fileBrowser ); - add( tabbedPane, BorderLayout.CENTER ); - } - - @Override - public void valueChanged( TreeSelectionEvent e ) { - - JTree tree; - DefaultMutableTreeNode node; - InvocationItem invocItem; - - tree = ( JTree )e.getSource(); - node = ( DefaultMutableTreeNode )tree.getLastSelectedPathComponent(); - - if( node instanceof InvocationItem ) { - - invocItem = ( InvocationItem )node; - invocIdField.setText( String.valueOf( invocItem.getInvocId() ) ); - taskNameField.setText( invocItem.getTaskName() ); - stdOutArea.setText( invocItem.getStdOut() ); - stdErrArea.setText( invocItem.getStdErr() ); - fileBrowser.setInvocId( invocItem.getInvocId() ); - } - else { - invocIdField.setText( null ); - taskNameField.setText( null ); - stdOutArea.setText( null ); - stdErrArea.setText( null ); - fileBrowser.blank(); - } - - } - -} diff --git a/cuneiform-addons/cuneiform-starlinger/.gitignore b/cuneiform-addons/cuneiform-starlinger/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-addons/cuneiform-starlinger/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-addons/cuneiform-starlinger/pom.xml b/cuneiform-addons/cuneiform-starlinger/pom.xml deleted file mode 100644 index 2941c55..0000000 --- a/cuneiform-addons/cuneiform-starlinger/pom.xml +++ /dev/null @@ -1,49 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform-addons - 2.0.4-RELEASE - - cuneiform-starlinger - - - - de.hu-berlin.wbi.cuneiform - cuneiform-core - ${project.version} - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - - \ No newline at end of file diff --git a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/JsonMap.java b/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/JsonMap.java deleted file mode 100644 index 72aeb82..0000000 --- a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/JsonMap.java +++ /dev/null @@ -1,85 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.starlinger; - -import java.util.HashMap; -import java.util.Map; - -public class JsonMap { - - private Map attributeMap; - - public JsonMap() { - attributeMap = new HashMap<>(); - } - - public String getAttribute( String key ) { - return attributeMap.get( key ); - } - - public void putAttribute( String key, String value ) { - - if( key == null ) - throw new NullPointerException( "Key must not be null." ); - - attributeMap.put( key, value ); - } - - @Override - public String toString() { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "{\n" ); - - comma = false; - for( String key : attributeMap.keySet() ) { - - if( comma ) - buf.append( ",\n" ); - comma = true; - - buf.append( '\'' ).append( key ).append( "'=>" ) - .append( attributeMap.get( key ) ); - } - - buf.append( "\n}" ); - - return buf.toString(); - } - - -} diff --git a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerEdgeSet.java b/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerEdgeSet.java deleted file mode 100644 index 0f815fb..0000000 --- a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerEdgeSet.java +++ /dev/null @@ -1,97 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.starlinger; - -import java.util.HashMap; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; - -public class StarlingerEdgeSet { - - private Map> edgeMap; - - public StarlingerEdgeSet() { - - edgeMap = new HashMap<>(); - } - - public void addEdge( String src, String sink ) { - - List sinkList; - - sinkList = edgeMap.get( src ); - - if( sinkList == null ) { - - sinkList = new LinkedList<>(); - edgeMap.put( src, sinkList ); - } - - sinkList.add( sink ); - } - - @Override - public String toString() { - - JsonMap jsonMap; - StringBuffer buf; - boolean comma; - - jsonMap = new JsonMap(); - - for( String key : edgeMap.keySet() ) { - - - buf = new StringBuffer(); - - buf.append( '[' ); - - comma = false; - for( String value : edgeMap.get( key ) ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( '\'' ).append( value ).append( '\'' ); - } - - buf.append( ']' ); - - jsonMap.putAttribute( key, buf.toString() ); - } - - return jsonMap.toString(); - } -} diff --git a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNode.java b/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNode.java deleted file mode 100644 index 33c8957..0000000 --- a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNode.java +++ /dev/null @@ -1,141 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.starlinger; - -public class StarlingerNode extends JsonMap { - - public static final String ATT_TYPE = "type"; - public static final String ATT_ID = "id"; - public static final String ATT_NAME = "name"; - public static final String ATT_FOREIGN_SRC = "foreign-src"; - public static final String ATT_STRING_CONTENT = "string-content"; - - public static final String LABEL_NIL = "nil"; - public static final String LABEL_APPEND = "append"; - public static final String LABEL_NATIVE_LAMBDA = "\\\\{}"; - public static final String LABEL_CURRY = "curry"; - public static final String LABEL_FOREIGN_LAMBDA = "\\\\*{}*"; - public static final String LABEL_COND = "cond"; - public static final String LABEL_APPLY = "apply"; - - public static final String TYPE_APPEND = "Append"; - public static final String TYPE_NIL = "Nil"; - public static final String TYPE_STRING = "StringExpr"; - public static final String TYPE_APPLY = "ApplyExpr"; - public static final String TYPE_CURRY = "CurryExpr"; - public static final String TYPE_FOREIGN_LAMBDA = "ForeignLambdaExpr"; - public static final String TYPE_NATIVE_LAMBDA = "NativeLambdaExpr"; - public static final String TYPE_COND = "CondExpr"; - public static final String TYPE_NAME = "NameExpr"; - - public static StarlingerNode createAppendNode( String id ) { - return new StarlingerNode( id, LABEL_APPEND, TYPE_APPEND ); - } - - public static StarlingerNode createNilNode( String id ) { - return new StarlingerNode( id, LABEL_NIL, TYPE_NIL ); - } - - public static StarlingerNode createStringNode( String id, String label ) { - - StarlingerNode node; - - node = new StarlingerNode( id, label, TYPE_STRING ); - node.setStringContent( label ); - - return node; - } - - public static StarlingerNode createCurryNode( String id ) { - return new StarlingerNode( id, LABEL_CURRY, TYPE_CURRY ); - } - - public static StarlingerNode createForeignLambdaNode( String id ) { - return new StarlingerNode( id, LABEL_FOREIGN_LAMBDA, TYPE_FOREIGN_LAMBDA ); - } - - public static StarlingerNode createNativeLambdaNode( String id ) { - return new StarlingerNode( id, LABEL_NATIVE_LAMBDA, TYPE_NATIVE_LAMBDA ); - } - - public static StarlingerNode createCondNode( String id ) { - return new StarlingerNode( id, LABEL_COND, TYPE_COND ); - } - - public static StarlingerNode createApplyNode( String id ) { - return new StarlingerNode( id, LABEL_APPLY, TYPE_APPLY ); - } - - public static StarlingerNode createApplyNode( String id, String label ) { - return new StarlingerNode( id, label, TYPE_APPLY ); - } - - public static StarlingerNode createNameNode( String id, String label ) { - return new StarlingerNode( id, label, TYPE_NAME ); - } - - - - - public StarlingerNode( String id, String label, String type ) { - setId( id ); - setLabel( label ); - setType( type ); - } - - public String getId() { - - String s; - - s = getAttribute( ATT_ID ); - - return s.substring( 1, s.length()-1 ); - } - - public void setType( String type ) { - putAttribute( ATT_TYPE, "'"+type+"'" ); - } - - public void setLabel( String label ) { - putAttribute( ATT_NAME, "'"+label+"'" ); - } - - public void setId( String id ) { - putAttribute( ATT_ID, "'"+id+"'" ); - } - - public void setStringContent( String content ) { - putAttribute( ATT_STRING_CONTENT, "'"+content+"'" ); - } - -} diff --git a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNodeSet.java b/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNodeSet.java deleted file mode 100644 index 1d9a9b9..0000000 --- a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNodeSet.java +++ /dev/null @@ -1,40 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.starlinger; - -public class StarlingerNodeSet extends JsonMap { - - public void addNode( StarlingerNode node ) { - putAttribute( node.getId(), node.toString() ); - } -} diff --git a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNodeVisitor.java b/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNodeVisitor.java deleted file mode 100644 index 2834c71..0000000 --- a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerNodeVisitor.java +++ /dev/null @@ -1,372 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.starlinger; - -import java.util.HashMap; -import java.util.Map; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Block; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CondExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CorrelParam; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CurryExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.DataType; -import de.huberlin.wbi.cuneiform.core.semanticmodel.DrawParam; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.LambdaType; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NativeLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfNode; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NodeVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ReduceVar; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SingleExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; - -public class StarlingerNodeVisitor extends StarlingerWorkflow implements NodeVisitor { - - private Map exprMap; - private int nextNodeId; - - public StarlingerNodeVisitor() { - exprMap = new HashMap<>(); - nextNodeId = 0; - } - - @Override - public String accept(StringExpr stringExpr) { - - String nodeId; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( stringExpr ) ) - return exprMap.get( stringExpr ); - - nodeId = popNodeId(); - - starlingerNode = StarlingerNode.createStringNode( nodeId, stringExpr.getContent() ); - - addNode( starlingerNode ); - - exprMap.put( stringExpr, nodeId ); - - return nodeId; - } - - @Override - public String accept( QualifiedTicket qualifiedTicket ) { - throw new RuntimeException( "invalid qualified ticket." ); - } - - @Override - public String accept(NativeLambdaExpr nativeLambdaExpr) { - - String nodeId; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( nativeLambdaExpr ) ) - return exprMap.get( nativeLambdaExpr ); - - nodeId = popNodeId(); - - starlingerNode = StarlingerNode.createNativeLambdaNode( nodeId ); - addNode( starlingerNode ); - - exprMap.put( nativeLambdaExpr, nodeId ); - - return nodeId; - } - - @Override - public String accept(NameExpr nameExpr) { - - String nodeId; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( nameExpr ) ) - return exprMap.get( nameExpr ); - - nodeId = popNodeId(); - - starlingerNode = StarlingerNode.createNameNode( nodeId, nameExpr.getId() ); - - addNode( starlingerNode ); - - exprMap.put( nameExpr, nodeId ); - - return nodeId; - } - - @Override - public String accept( LambdaType lambdaType ) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept(DrawParam drawParam) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept(DataType dataType) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept(CondExpr condExpr) throws HasFailedException, NotBoundException { - - String nodeId; - String childId; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( condExpr ) ) - return exprMap.get( condExpr ); - - nodeId = popNodeId(); - - starlingerNode = StarlingerNode.createCondNode( nodeId ); - - addNode( starlingerNode ); - - childId = condExpr.getIfExpr().visit( this ); - addEdge( childId, nodeId ); - - childId = condExpr.getThenExpr().visit( this ); - addEdge( childId, nodeId ); - - childId = condExpr.getElseExpr().visit( this ); - addEdge( childId, nodeId ); - - exprMap.put( condExpr, nodeId ); - - return nodeId; - } - - @Override - public String accept(Block block) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept(ApplyExpr applyExpr) throws HasFailedException, NotBoundException { - String taskNodeId, refNodeId, taskName; - SingleExpr se; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( applyExpr ) ) - return exprMap.get( applyExpr ); - - taskNodeId = popNodeId(); - - if( applyExpr.getTaskExpr().getNumSingleExpr() == 0 ) - throw new RuntimeException( "Task expression must not be nil." ); - - se = applyExpr.getTaskExpr().getSingleExpr( 0 ); - - if( applyExpr.getTaskExpr().getNumSingleExpr() != 1 || se instanceof CurryExpr ) { - - refNodeId = applyExpr.getTaskExpr().visit( this ); - starlingerNode = StarlingerNode.createApplyNode( taskNodeId ); - - addNode( starlingerNode ); - addEdge( refNodeId, taskNodeId ); - } - else { - - if( se instanceof NativeLambdaExpr ) - taskName = StarlingerNode.LABEL_NATIVE_LAMBDA; - else if( se instanceof ForeignLambdaExpr ) - taskName = StarlingerNode.LABEL_FOREIGN_LAMBDA; - else if( se instanceof NameExpr ) - taskName = ( ( NameExpr)se ).getId(); - else - throw new RuntimeException( "Task type not recognized." ); - - starlingerNode = StarlingerNode.createApplyNode( taskNodeId, taskName ); - - addNode( starlingerNode ); - } - - - for( NameExpr nameExpr : applyExpr.getNameSet() ) - try { - - refNodeId = applyExpr.getExpr( nameExpr ).visit( this ); - - addEdge( refNodeId, taskNodeId ); - } - catch( NotBoundException e ) { - // cannot happen, because we only ask for what is there - throw new RuntimeException( e.getMessage() ); - } - - exprMap.put( applyExpr, taskNodeId ); - - return taskNodeId; - } - - @Override - public String accept(CompoundExpr compoundExpr) throws HasFailedException, NotBoundException { - - String nodeId; - int n; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( compoundExpr ) ) - return exprMap.get( compoundExpr ); - - n = compoundExpr.getNumSingleExpr(); - - if( n == 0 ) { - - nodeId = popNodeId(); - - starlingerNode = StarlingerNode.createNilNode( nodeId ); - - addNode( starlingerNode ); - exprMap.put( compoundExpr, nodeId ); - - return nodeId; - } - - if( n > 1 ) { - - nodeId = popNodeId(); - - - starlingerNode = StarlingerNode.createAppendNode( nodeId ); - addNode( starlingerNode ); - - - for( SingleExpr se : compoundExpr.getSingleExprList() ) - addEdge( se.visit( this ), nodeId ); - - exprMap.put( compoundExpr, nodeId ); - - return nodeId; - } - - - return compoundExpr.getSingleExpr( 0 ).visit( this ); - } - - @Override - public String accept(CorrelParam correlParam) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept(ForeignLambdaExpr foreignLambdaExpr) { - - String nodeId; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( foreignLambdaExpr ) ) - return exprMap.get( foreignLambdaExpr ); - - nodeId = popNodeId(); - - starlingerNode = StarlingerNode.createForeignLambdaNode( nodeId ); - - addNode( starlingerNode ); - - exprMap.put( foreignLambdaExpr, nodeId ); - - return nodeId; - } - - @Override - public String accept(CurryExpr curryExpr) throws HasFailedException, NotBoundException { - String taskNodeId, refNodeId; - StarlingerNode starlingerNode; - - if( exprMap.containsKey( curryExpr ) ) - return exprMap.get( curryExpr ); - - taskNodeId = popNodeId(); - - - refNodeId = curryExpr.getTaskExpr().visit( this ); - starlingerNode = StarlingerNode.createCurryNode( taskNodeId ); - - addNode( starlingerNode ); - addEdge( refNodeId, taskNodeId ); - - - for( NameExpr nameExpr : curryExpr.getNameSet() ) - try { - - refNodeId = curryExpr.getExpr( nameExpr ).visit( this ); - - addEdge( refNodeId, taskNodeId ); - } - catch( NotBoundException e ) { - // cannot happen, because we only ask for what is there - throw new RuntimeException( e.getMessage() ); - } - - exprMap.put( curryExpr, taskNodeId ); - - return taskNodeId; - - } - - @Override - public String accept(Prototype prototype) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept(ReduceVar reduceVar) { - throw new RuntimeException( "Not an expression." ); - } - - @Override - public String accept( TopLevelContext tlc ) throws HasFailedException, NotBoundException { - - for( CompoundExpr ce : tlc.getTargetList() ) - ce.visit( this ); - - return null; - } - - private String popNodeId() { - return "node"+( nextNodeId++ ); - } - -} diff --git a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerWorkflow.java b/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerWorkflow.java deleted file mode 100644 index 20ca4ed..0000000 --- a/cuneiform-addons/cuneiform-starlinger/src/main/java/de/huberlin/wbi/cuneiform/starlinger/StarlingerWorkflow.java +++ /dev/null @@ -1,90 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.starlinger; - -public class StarlingerWorkflow extends JsonMap { - - private static final String LABEL_FILENAME = "filename"; - private static final String LABEL_RESOURCE_URI = "resource_uri"; - private static final String LABEL_TITLE = "title"; - private static final String LABEL_FILEPATH = "filepath"; - private static final String LABEL_DESCRIPTION = "description"; - private static final String LABEL_NODES = "nodes"; - private static final String LABEL_EDGES = "edges"; - - private StarlingerNodeSet nodeSet; - private StarlingerEdgeSet edgeSet; - - public StarlingerWorkflow() { - nodeSet = new StarlingerNodeSet(); - edgeSet = new StarlingerEdgeSet(); - } - - public void addEdge( String src, String sink ) { - edgeSet.addEdge( src, sink ); - } - - public void addNode( StarlingerNode node ) { - nodeSet.addNode( node ); - } - - public void setFilename( String filename ) { - putAttribute( LABEL_FILENAME, "'"+filename+"'" ); - } - - public void setResourceUri( String resourceUri ) { - putAttribute( LABEL_RESOURCE_URI, "'"+resourceUri+"'" ); - } - - public void setTitle( String title ) { - putAttribute( LABEL_TITLE, "'"+title+"'" ); - } - - public void setFilepath( String filepath ) { - putAttribute( LABEL_FILEPATH, "'"+filepath+"'" ); - } - - public void setDescription( String description ) { - putAttribute( LABEL_DESCRIPTION, "'"+description+"'" ); - } - - @Override - public String toString() { - - putAttribute( LABEL_EDGES, edgeSet.toString() ); - putAttribute( LABEL_NODES, nodeSet.toString() ); - - return super.toString(); - } - -} diff --git a/cuneiform-addons/pom.xml b/cuneiform-addons/pom.xml deleted file mode 100644 index 4c9e19e..0000000 --- a/cuneiform-addons/pom.xml +++ /dev/null @@ -1,17 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform - 2.0.4-RELEASE - - cuneiform-addons - pom - - cuneiform-starlinger - cuneiform-dax - cuneiform-cfide - cuneiform-logview - cuneiform-htcondorcre - - \ No newline at end of file diff --git a/cuneiform-cmdline/.gitignore b/cuneiform-cmdline/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-cmdline/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-cmdline/pom.xml b/cuneiform-cmdline/pom.xml deleted file mode 100644 index dbe85f7..0000000 --- a/cuneiform-cmdline/pom.xml +++ /dev/null @@ -1,70 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform - 2.0.4-RELEASE - - cuneiform-cmdline - cuneiform-cmdline - - - - - - de.hu-berlin.wbi.cuneiform - cuneiform-core - ${project.version} - - - - de.hu-berlin.wbi.cuneiform - cuneiform-dax - ${project.version} - - - - de.hu-berlin.wbi.cuneiform - cuneiform-htcondorcre - ${project.version} - - - - commons-cli - commons-cli - 1.2 - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - - \ No newline at end of file diff --git a/cuneiform-cmdline/src/main/java/de/huberlin/wbi/cuneiform/cmdline/main/JsonSummary.java b/cuneiform-cmdline/src/main/java/de/huberlin/wbi/cuneiform/cmdline/main/JsonSummary.java deleted file mode 100644 index 98c904c..0000000 --- a/cuneiform-cmdline/src/main/java/de/huberlin/wbi/cuneiform/cmdline/main/JsonSummary.java +++ /dev/null @@ -1,90 +0,0 @@ -package de.huberlin.wbi.cuneiform.cmdline.main; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.List; -import java.util.UUID; - -import de.huberlin.wbi.cuneiform.core.cre.LocalCreActor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; - -public class JsonSummary { - - private final UUID runId; - private final Path buildDir; - private final List output; - - public JsonSummary( UUID runId, Path buildDir, CompoundExpr output ) - throws NotDerivableException { - - if( runId == null ) - throw new NullPointerException( "Run Id must not be null." ); - - if( buildDir == null ) - throw new NullPointerException( "Build directory must not be null." ); - - if( output == null ) - this.output = new ArrayList<>(); - else - this.output = output.normalize(); - - this.runId = runId; - this.buildDir = buildDir; - } - - @Override - public String toString() { - - StringBuffer buf; - String s; - int i, n; - boolean res, comma; - Path candidate; - - buf = new StringBuffer(); - - buf.append( "{\n" ); - - buf.append( " \"runId\":\"" ).append( runId ).append( "\",\n" ); - - buf.append( " \"output\":[" ); - res = false; - if( !output.isEmpty() ) { - - n = output.size(); - comma = false; - - for( i = 0; i < n; i++ ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - s = output.get( i ); - candidate = buildDir.resolve( LocalCreActor.PATH_CENTRALREPO ).resolve( s ); - - if( Files.exists( candidate ) ) { - s = candidate.toAbsolutePath().toString(); - res = true; - } - - buf.append( '"' ).append( s ).append( '"' ); - } - } - buf.append( "],\n" ); - - buf.append( " \"type\":\"" ); - if( res ) - buf.append( "File" ); - else - buf.append( "String" ); - buf.append( "\"\n" ); - - - buf.append( "}\n" ); - - return buf.toString(); - } -} diff --git a/cuneiform-cmdline/src/main/java/de/huberlin/wbi/cuneiform/cmdline/main/Main.java b/cuneiform-cmdline/src/main/java/de/huberlin/wbi/cuneiform/cmdline/main/Main.java deleted file mode 100644 index c907523..0000000 --- a/cuneiform-cmdline/src/main/java/de/huberlin/wbi/cuneiform/cmdline/main/Main.java +++ /dev/null @@ -1,356 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.cmdline.main; - -import de.huberlin.cuneiform.dax.repl.DaxRepl; -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.cre.BaseCreActor; -import de.huberlin.wbi.cuneiform.core.cre.LocalCreActor; -import de.huberlin.wbi.cuneiform.core.cre.LocalThread; -import de.huberlin.wbi.cuneiform.core.invoc.Invocation; -import de.huberlin.wbi.cuneiform.core.repl.BaseRepl; -import de.huberlin.wbi.cuneiform.core.repl.CmdlineRepl; -import de.huberlin.wbi.cuneiform.core.repl.InteractiveRepl; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor; -import de.huberlin.wbi.cuneiform.htcondorcre.CondorCreActor; -import org.apache.commons.cli.*; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.io.*; -import java.nio.charset.Charset; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -public class Main { - - private static final String FORMAT_CF = "cf"; - private static final String FORMAT_DAX = "dax"; - private static final String PLATFORM_LOCAL = "local"; - private static final String PLATFORM_HTCONDOR = "htcondor"; - - - public static final int MAX_TRANSFER = 1073741824; // no more than 1G of total transfer - - private static String platform; - private static String format; - private static Path[] inputFileVector; - - - public static void main( String[] args ) throws IOException, ParseException, InterruptedException, NotDerivableException { - - CommandLine cmd; - Options opt; - BaseRepl repl; - BaseCreActor cre; - Path sandbox; - ExecutorService executor; - TicketSrcActor ticketSrc; - JsonSummary summary; - Path summaryPath; - Log statLog; - int nthread; - Path workDir; - - statLog = LogFactory.getLog( "statLogger" ); - - executor = Executors.newCachedThreadPool(); - try { - - opt = getOptions(); - cmd = parse( args, opt ); - config( cmd ); - - if( cmd.hasOption( 'h' ) ) { - - System.out.println( - "CUNEIFORM - A Functional Workflow Language\nversion " - +BaseRepl.LABEL_VERSION+" build "+BaseRepl.LABEL_BUILD ); - new HelpFormatter().printHelp( - "java -jar cuneiform.jar [OPTION]*", opt ); - - return; - } - - if( cmd.hasOption( 'r' ) ) - Invocation.putLibPath( ForeignLambdaExpr.LANGID_R, cmd.getOptionValue( 'r' ) ); - - if( cmd.hasOption( 'y' ) ) - Invocation.putLibPath( ForeignLambdaExpr.LANGID_PYTHON, cmd.getOptionValue( 'y' ) ); - - if( cmd.hasOption( 'l' ) ) - sandbox = Paths.get( cmd.getOptionValue( "l" ) ); - else - sandbox = Paths.get( System.getProperty( "user.home" ) ).resolve( ".cuneiform" ); - - sandbox = sandbox.toAbsolutePath(); - - if( cmd.hasOption( 'c' ) ) - LocalThread.deleteIfExists( sandbox ); - - if( cmd.hasOption( 't' ) ) - nthread = Integer.valueOf( cmd.getOptionValue( 't' ) ); - else - nthread = Runtime.getRuntime().availableProcessors(); - - if( cmd.hasOption( 'w' ) ) - workDir = Paths.get( cmd.getOptionValue( 'w' ) ); - else - workDir = Paths.get( System.getProperty( "user.dir" ) ); - - workDir = workDir.toAbsolutePath(); - - - - - switch( platform ) { - - case PLATFORM_LOCAL : - - if( !Files.exists( sandbox ) ) - Files.createDirectories( sandbox ); - - - cre = new LocalCreActor( sandbox, workDir, nthread ); - break; - - case PLATFORM_HTCONDOR : - - if( !Files.exists( sandbox ) ) - Files.createDirectories( sandbox ); - - if (cmd.hasOption('m')) { // MAX_TRANSFER SIZE - String maxTransferSize = cmd.getOptionValue('m'); - try { - cre = new CondorCreActor(sandbox, maxTransferSize); - } catch (Exception e) { - System.out.println( - "INVALID '-m' option value: " + maxTransferSize + - "\n\nCUNEIFORM - A Functional Workflow Language\nversion " - + BaseRepl.LABEL_VERSION + " build " + BaseRepl.LABEL_BUILD); - new HelpFormatter().printHelp( - "java -jar cuneiform.jar [OPTION]*", opt); - - return; - } - } - else{ - cre = new CondorCreActor(sandbox); - } - - break; - - default : throw new RuntimeException( "Platform not recognized." ); - } - - executor.submit( cre ); - ticketSrc = new TicketSrcActor( cre ); - executor.submit( ticketSrc ); - executor.shutdown(); - - switch( format ) { - - case FORMAT_CF : - - if( cmd.hasOption( "i" ) ) - repl = new InteractiveRepl( ticketSrc, statLog ); - else - repl = new CmdlineRepl( ticketSrc, statLog ); - break; - - case FORMAT_DAX : - repl = new DaxRepl( ticketSrc, statLog ); - break; - - default : - throw new RuntimeException( "Format not recognized." ); - } - - if( cmd.hasOption( "i" ) ) { - - // run in interactive mode - BaseRepl.run( repl ); - - return; - } - - // run in quiet mode - - if( inputFileVector.length > 0 ) - - for( Path f : inputFileVector ) - repl.interpret( readFile( f ) ); - - else - repl.interpret( readStdIn() ); - - - Thread.sleep( 3*Actor.DELAY ); - while( repl.isBusy() ) - Thread.sleep( Actor.DELAY ); - - if( cmd.hasOption( "s" ) ) { - - summary = new JsonSummary( ticketSrc.getRunId(), sandbox, repl.getAns() ); - summaryPath = Paths.get( cmd.getOptionValue( "s" ) ); - summaryPath = summaryPath.toAbsolutePath(); - try( BufferedWriter writer = Files.newBufferedWriter( summaryPath, Charset.forName( "UTF-8" ) ) ) { - - writer.write( summary.toString() ); - } - - } - - - } - finally { - executor.shutdownNow(); - } - - } - - public static void config( CommandLine cmd ) { - - String[] fileVector; - Path f; - String s; - int i, n; - - fileVector = cmd.getArgs(); - n = fileVector.length; - inputFileVector = new Path[ n ]; - - for( i = 0; i < n; i++ ) { - - s = fileVector[ i ]; - f = Paths.get( s ); - - inputFileVector[ i ] = f; - } - - if( cmd.hasOption( 'p' ) ) - platform = cmd.getOptionValue( 'p' ); - else - platform = PLATFORM_LOCAL; - - if( cmd.hasOption( 'f' ) ) - format = cmd.getOptionValue( 'f' ); - else - format = FORMAT_CF; - } - - public static CommandLine parse( String[] args, Options opt ) throws ParseException { - - GnuParser parser; - CommandLine cmd; - - parser = new GnuParser(); - cmd = parser.parse( opt, args ); - return cmd; - } - - public static Options getOptions() { - - Options opt; - - opt = new Options(); - - opt.addOption( "c", "clean", false, "Clear local cache before start." ); - - opt.addOption( "f", "format", true, - "The format of the input file. Must be either '"+FORMAT_CF+"' for Cuneiform or '"+FORMAT_DAX+"' for Pegasus DAX. Default is '"+FORMAT_CF+"'." ); - - opt.addOption( "h", "help", false, "Output help." ); - - opt.addOption( "i", "interactive", false, "Start an interactive REPL." ); - - opt.addOption( "l", "localcache", true, "Path to the local cache. Defaults to '~/.cuneiform'." ); - - opt.addOption( "p", "platform", true, - "The platform to run. Currently available platforms are '"+PLATFORM_LOCAL+"' and '"+PLATFORM_HTCONDOR+"'. Default is '"+PLATFORM_LOCAL+"'." ); - - opt.addOption("m", "maxtransfer", true, - "Optional. Takes effect only if the platform is '"+PLATFORM_HTCONDOR+"'. Default is 1G. Controls max input file transfer. If total input files exceed this limit, condor job is run in the local universe. Takes a number in bytes by default or specified units [G|M|K], i.e. 250000000 or 2G or 1000M or 234K"); - - opt.addOption( "r", "rlib", true, "Optional. The directory in which custom R libraries are installed. If set, the directory is added to R's libPath list." ); - - opt.addOption( "s", "summary", true, - "The name of a JSON summary file. No file is created if this parameter is not specified." ); - - opt.addOption( "t", "threads", true, "The number of threads to use. Defaults to the number of CPU cores available on this machine. Takes effect only if the platform is '"+PLATFORM_LOCAL+"'." ); - - opt.addOption( "w", "workdir", true, "Optional. The working directory from which to look for local data. The default is the current working directory from which Cuneiform was called." ); - - opt.addOption( "y", "pylib", true, "Optional. The directory in which custom Python libraries are installed. If set, the directory is added to Python's system path list." ); - - - return opt; - - } - - public static String readFile( Path f ) throws FileNotFoundException, IOException { - - String line; - StringBuffer buf; - - buf = new StringBuffer(); - try( BufferedReader reader = Files.newBufferedReader( f, Charset.forName( "UTF-8" ) ) ) { - - while( ( line = reader.readLine() ) != null ) - buf.append( line ).append( '\n' ); - } - - return buf.toString(); - - } - - public static String readStdIn() throws IOException { - - StringBuffer buf; - String line; - - buf = new StringBuffer(); - try( BufferedReader reader = new BufferedReader( new InputStreamReader( System.in ) ) ) { - - while( ( line = reader.readLine() ) != null ) - buf.append( line ).append( '\n' ); - } - - return buf.toString(); - } -} diff --git a/cuneiform-core/.gitignore b/cuneiform-core/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-core/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-core/pom.xml b/cuneiform-core/pom.xml deleted file mode 100644 index 0b544aa..0000000 --- a/cuneiform-core/pom.xml +++ /dev/null @@ -1,99 +0,0 @@ - - 4.0.0 - - de.hu-berlin.wbi.cuneiform - cuneiform - 2.0.4-RELEASE - - cuneiform-core - - - - junit - junit - 4.12 - test - - - - org.antlr - antlr4 - 4.5 - - - - org.json - json - 20090211 - - - - commons-logging - commons-logging - 1.1.3 - - - - log4j - log4j - 1.2.17 - - - - org.mockito - mockito-all - 1.10.19 - - - - - - - - - org.apache.maven.plugins - maven-source-plugin - 2.4 - - - attach-sources - - jar - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - - org.antlr - antlr4-maven-plugin - 4.5 - - ${basedir}/src/main/antlr - true - true - - - - - antlr4 - - - - - - - \ No newline at end of file diff --git a/cuneiform-core/src/main/antlr/de/huberlin/wbi/cuneiform/core/parser/Cuneiform.g4 b/cuneiform-core/src/main/antlr/de/huberlin/wbi/cuneiform/core/parser/Cuneiform.g4 deleted file mode 100644 index 3518c5a..0000000 --- a/cuneiform-core/src/main/antlr/de/huberlin/wbi/cuneiform/core/parser/Cuneiform.g4 +++ /dev/null @@ -1,208 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -grammar Cuneiform ; - - -// NON-TERMINAL SYMBOLS - -script : stat* EOF ; - -// the body -stat : target // top-level - | importFile - | instat - | expr - { notifyErrorListeners( "Dangling expression. Expecting ';' or '-+'." ); } - ; - -instat : assign // allowed also in blocks - | defTask - | danglingExpr - ; - -// imports -importFile : ( IMPORT | INCLUDE )STRING SEMICOLON ; - -// assignments, tasks and lambda expressions -assign : name+ EQUAL expr SEMICOLON ; - -defTask : DEFTASK ID prototype LBRACE instat+ RBRACE # NativeDefTask - | DEFTASK ID prototype foreignBody # ForeignDefTask - | DEFTASK - { notifyErrorListeners( "Incomplete Task definition. Task name expected." ); } # DefTaskErr1 - | DEFTASK ID prototype LBRACE - { notifyErrorListeners( "Missing '}'." ); } # DefTaskErr2 - | DEFTASK ID prototype LBRACE RBRACE - { notifyErrorListeners( "Empty native task block." ); } # DefTaskErr3 - | DEFTASK ID LPAREN - { notifyErrorListeners( "Incomplete task prototype. Expecting at least one output variable declaration." ); } # FnPrototypeErr1 - | DEFTASK ID LPAREN output+ COLON param* - { notifyErrorListeners( "Incomplete task prototype. Expecting input parameter declaration or ')'." ); } # FnPrototypeErr2 - | DEFTASK ID LPAREN output+ COLON param* RPAREN RPAREN+ - { notifyErrorListeners( "Too many ')'." ); } # FnPrototypeErr3 - ; - -// type system -prototype : LPAREN output+ COLON param* RPAREN ; - -name : ID # NameInferredType - | ID LPAREN ID RPAREN # NameDataType - | ID LPAREN COLON RPAREN # NamePlainFnType - | ID prototype # NameDeepFnType - | ID LPAREN COLON? - { notifyErrorListeners( "Missing ')'." ); } # NameErr1 - | ID LPAREN ID RPAREN RPAREN+ - { notifyErrorListeners( "Too many ')'." ); } # NameErr2 - ; - -param : name - | reduceVar - | correlParam - | draw - ; - -draw : LBRACE COMB INT COLON( name )RBRACE # DrawComb - | LBRACE COMBR INT COLON( name )RBRACE # DrawCombr - | LBRACE VAR INT COLON( name )RBRACE # DrawVar - | LBRACE PERM INT COLON( name )RBRACE # DrawPerm - ; - -output : name - | reduceVar - ; - -reduceVar : LTAG name RTAG ; - -correlParam : LSQUAREBR name name+ RSQUAREBR - | LSQUAREBR name RSQUAREBR - { notifyErrorListeners( "Correlated parameter list must have at least two entries." ); } - | LSQUAREBR RSQUAREBR - { notifyErrorListeners( "Empty correlated parameter list." ); } - ; - -// expressions -expr : NIL # NilExpr - | singleExpr+ # CompoundExpr - ; - -danglingExpr : expr TOSTACK ; - -singleExpr : ID # IdExpr - | INT # IntExpr - | STRING # StringExpr - | FROMSTACK # FromStackExpr - | IF expr THEN expr ELSE expr END # CondExpr - | channel? APPLY LPAREN paramBind( COMMA paramBind )* TILDE? RPAREN # ApplyExpr - | channel? ID LPAREN( paramBind( COMMA paramBind )* )? TILDE? RPAREN # CallExpr - | CURRY LPAREN paramBind( COMMA paramBind )* RPAREN # CurryExpr - | LAMBDA prototype block # NativeLambdaExpr - | LAMBDA prototype foreignBody # ForeignLambdaExpr - | APPLY - { notifyErrorListeners( "Incomplete task application. Missing '('." ); } # SingleExprErr1 - | APPLY LPAREN - { notifyErrorListeners( "Incomplete task application. Missing Parameter bindings, e.g. 'param: value'." ); } # SingleExprErr2 - | APPLY LPAREN paramBind( COMMA paramBind )* - { notifyErrorListeners( "Incomplete task application. Missing ')'." ); } # SingleExprErr3 - | APPLY LPAREN ID COLON - { notifyErrorListeners( "Incomplete Parameter binding. Missing value." ); } # ParamBindErr1 - | LAMBDA prototype INLANG OPENBODY - { notifyErrorListeners( "In foreign task definition: Missing '}*'." ); } # ForeignFnBodyErr2 - ; - -channel : LSQUAREBR INT RSQUAREBR ; - -block : LBRACE instat+ RBRACE - | LBRACE RBRACE - { notifyErrorListeners( "Empty block. Expecting target, assignment, or task definition." ); } - ; - -paramBind : ID COLON expr ; - -target : expr SEMICOLON ; - -foreignBody : INLANG BODY ; - -// TERMINAL SYMBOLS - -APPLY : 'apply' ; -COLON : ':' ; -COMMA : ',' ; -COMB : 'comb' ; -COMBR : 'combr' ; -CURRY : 'curry' ; -DEFTASK : 'deftask' ; -ELSE : 'else' ; -END : 'end' ; -EQUAL : '=' ; -FROMSTACK : '<' '-'+ '+' ; -IF : 'if' ; -INLANG : 'in' WSSYMB+ LANGSYMB ; -fragment LANGSYMB: 'bash' | 'r' | 'R' | 'lisp' | 'octave' | 'matlab' | 'perl' - | 'pegasus' | 'python' | 'java' | 'scala' | 'beanshell' ; -IMPORT : 'import' ; -INCLUDE : 'include' ; -LAMBDA : '\\' ; -LBRACE : '{' ; -LPAREN : '(' ; -LSQUAREBR : '[' ; -LTAG : '<' ; -NIL : 'nil' ; -PERM : 'perm' ; -RBRACE : '}' ; -RPAREN : ')' ; -RSQUAREBR : ']' ; -RTAG : '>' ; -SEMICOLON : ';' ; -TILDE : '~' ; -THEN : 'then' ; -TOSTACK : '-'+ '+' ; -VAR : 'var' ; - -INT : [0-9] - | '-'? [1-9][0-9]* - ; -BODY : LMMECB .*? RMMECB ; -OPENBODY : LMMECB .*? ; -fragment LMMECB : '*{' ; -fragment RMMECB : '}*' ; -STRING : '\'' ( '\\\'' | '\\\\' | . )*? '\'' - | '"' ( '\\"' | '\\\\' | . )*? '"' - ; -COMMENT : ( ( '#' | '//' | '%' ) ~'\n'* - | '/*' .*? '*/' - | '' - | '|' ) -> skip - ; -ID : [a-zA-Z0-9\.\-_\+\*/]+ ; -WS : WSSYMB -> channel( HIDDEN ) ; -fragment WSSYMB : [ \n\r\t] ; diff --git a/cuneiform-core/src/main/antlr/de/huberlin/wbi/cuneiform/core/parser/ParseCtlLexer.g4 b/cuneiform-core/src/main/antlr/de/huberlin/wbi/cuneiform/core/parser/ParseCtlLexer.g4 deleted file mode 100644 index d0d0e30..0000000 --- a/cuneiform-core/src/main/antlr/de/huberlin/wbi/cuneiform/core/parser/ParseCtlLexer.g4 +++ /dev/null @@ -1,46 +0,0 @@ -lexer grammar ParseCtlLexer; - -// LEXER RULES - -RC : '#' -> mode( COMMENT ) ; -OC : '%' -> mode( COMMENT ) ; -CC : '//' -> mode( COMMENT ) ; -LCCOMMENT : '/*' -> mode( CCOMMENT ) ; -LPAREN : '(' -> mode( APPLY ) ; -LXCOMMENT : '' -> mode( DEFAULT_MODE ) ; -ANYXC : . -> skip ; - - -mode APPLY ; - -RPAREN : ')' -> mode( DEFAULT_MODE ) ; -ANYA : . -> skip ; - -mode FOREIGN ; - -RMMECB : '}*' -> mode( DEFAULT_MODE ) ; -ANYF : . -> skip ; \ No newline at end of file diff --git a/cuneiform-core/src/main/doc/cuneiform-logo.svg b/cuneiform-core/src/main/doc/cuneiform-logo.svg deleted file mode 100644 index d1e3aca..0000000 --- a/cuneiform-core/src/main/doc/cuneiform-logo.svg +++ /dev/null @@ -1,541 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - image/svg+xml - - - - - - - - - - - - diff --git a/cuneiform-core/src/main/doc/g18225.png b/cuneiform-core/src/main/doc/g18225.png deleted file mode 100644 index 3b8a0bc..0000000 Binary files a/cuneiform-core/src/main/doc/g18225.png and /dev/null differ diff --git a/cuneiform-core/src/main/doc/pap-cf-apply.dia b/cuneiform-core/src/main/doc/pap-cf-apply.dia deleted file mode 100644 index 7a9244f..0000000 Binary files a/cuneiform-core/src/main/doc/pap-cf-apply.dia and /dev/null differ diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/actormodel/Actor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/actormodel/Actor.java deleted file mode 100644 index 7253921..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/actormodel/Actor.java +++ /dev/null @@ -1,139 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.actormodel; - -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -public abstract class Actor implements Runnable { - - public static final int DELAY = 1000; - - protected final Log log; - private final List inbox; - - public Actor() { - inbox = new ArrayList<>(); - log = LogFactory.getLog( Actor.class ); - } - - public synchronized void sendMsg( Message msg ) { - - if( msg == null ) - throw new NullPointerException( "Message must not be null." ); - - if( log.isDebugEnabled() ) - log.debug( "Sending message. "+msg.getSender().getClass().getSimpleName()+" -> "+getClass().getSimpleName()+" "+msg ); - - - inbox.add( msg ); - - } - - private synchronized void processQueue() { - - if( inbox == null ) - throw new NullPointerException( "Inbox must not be null." ); - - for( Message msg : inbox ) { - - if( msg == null ) - throw new NullPointerException( "Message must not be null." ); - - if( log.isDebugEnabled() ) - log.debug( "Receiving message. "+msg.getSender().getClass().getSimpleName()+" -> "+getClass().getSimpleName()+" "+msg ); - - processMsg( msg ); - } - inbox.clear(); - - } - - protected abstract void processMsg( Message msg ); - - @Override - public void run() { - - try { - - if( log.isDebugEnabled() ) - log.debug( "Starting actor: "+getClass().getSimpleName() ); - - while( true ) { - - - Thread.sleep( DELAY ); - - if( log.isDebugEnabled() ) - log.debug( getClass()+" is alive." ); - - preRec(); - processQueue(); - postRec(); - - } - } - catch( InterruptedException e ) { - - if( log.isDebugEnabled() ) - log.debug( getClass()+" caught InterruptedException. Shutting down." ); - - } - catch( Throwable e ) { - - if( log.isDebugEnabled() ) - log.debug( getClass()+" caught unexpected throwable: "+e.getMessage() ); - - e.printStackTrace(); - } - finally { - - if( log.isDebugEnabled() ) - log.debug( "Downing "+getClass()+"." ); - - shutdown(); - } - } - - protected abstract void shutdown(); - protected void preRec() { - // override if necessary - } - protected void postRec() { - // override if necessary - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/actormodel/Message.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/actormodel/Message.java deleted file mode 100644 index c967e3e..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/actormodel/Message.java +++ /dev/null @@ -1,50 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.actormodel; - -public abstract class Message { - - private final Actor sender; - - public Message( Actor sender ) { - - if( sender == null ) - throw new NullPointerException( "Sender must not be null." ); - - this.sender = sender; - } - - public Actor getSender() { - return sender; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/BaseCreActor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/BaseCreActor.java deleted file mode 100644 index 6f177a1..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/BaseCreActor.java +++ /dev/null @@ -1,55 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.cre; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.actormodel.Message; - -public abstract class BaseCreActor extends Actor { - - protected abstract void processMsg( TicketReadyMsg msg ); - - @Override - protected synchronized void processMsg( Message msg ) { - - if( msg instanceof TicketReadyMsg ) { - - processMsg( ( TicketReadyMsg )msg ); - return; - } - - throw new RuntimeException( "Message type not recognized." ); - - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/LocalCreActor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/LocalCreActor.java deleted file mode 100644 index b0b37ec..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/LocalCreActor.java +++ /dev/null @@ -1,130 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.cre; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor; - -public class LocalCreActor extends BaseCreActor { - - public static final String PATH_CENTRALREPO = "repo"; - - - private final ExecutorService executor; - private final Path buildDir; - private final Path centralRepo; - private final Path workDir; - - public LocalCreActor( Path buildDir, Path workDir, int nthread ) throws IOException { - - if( buildDir == null ) - throw new NullPointerException( "Build directory must not be null." ); - - if( !Files.exists( buildDir ) ) - throw new RuntimeException( "Build directory does not exist." ); - - if( !Files.isDirectory( buildDir ) ) - throw new RuntimeException( "Directory expected." ); - - if( nthread < 1 ) - throw new RuntimeException( "Number of threads must at least be 1. It was "+nthread+"." ); - - if( workDir == null ) - throw new IllegalArgumentException( "Working directory must not be null." ); - - if( !Files.exists( workDir ) ) - throw new RuntimeException( "Working directory does not exist." ); - - if( !Files.isDirectory( workDir ) ) - throw new RuntimeException( "Working directory expected to be a directory." ); - - - this.buildDir = buildDir; - this.workDir = workDir; - - executor = Executors.newFixedThreadPool( nthread ); - - centralRepo = buildDir.resolve( PATH_CENTRALREPO ); - if( !Files.exists( centralRepo ) ) - Files.createDirectories( centralRepo ); - - if( log.isDebugEnabled() ) - log.debug( "Local CRE actor created with "+nthread+" threads." ); - } - - public LocalCreActor( Path buildDir, Path workDir ) throws IOException { - this( buildDir, workDir, Runtime.getRuntime().availableProcessors() ); - } - - @Override - protected void processMsg( TicketReadyMsg msg ) { - - LocalThread localThread; - Actor sender; - TicketSrcActor ticketSrc; - Ticket ticket; - - sender = msg.getSender(); - if( !( sender instanceof TicketSrcActor ) ) - throw new RuntimeException( "Ticket source actor expected." ); - - ticketSrc = ( TicketSrcActor )sender; - - ticket = msg.getTicket(); - - if( !ticket.isNormal() ) - throw new RuntimeException( "Ticket "+ticket.getTicketId()+": Trying to evaluate ticket that is not ready." ); - - if( ticket.isEvaluated() ) - throw new RuntimeException( - "Ticket "+ticket.getTicketId()+": Trying to evaluate ticket that has already been evaluated." ); - - localThread = new LocalThread( ticketSrc, this, ticket, buildDir, centralRepo, workDir ); - - executor.submit( localThread ); - - } - - @Override - protected void shutdown() { - executor.shutdownNow(); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/LocalThread.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/LocalThread.java deleted file mode 100644 index 0967cfa..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/LocalThread.java +++ /dev/null @@ -1,434 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.cre; - -import de.huberlin.wbi.cuneiform.core.actormodel.Message; -import de.huberlin.wbi.cuneiform.core.invoc.Invocation; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketFailedMsg; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketFinishedMsg; -import de.huberlin.wbi.cuneiform.core.ticketsrc.TicketSrcActor; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.json.JSONObject; - -import java.io.BufferedReader; -import java.io.BufferedWriter; -import java.io.IOException; -import java.nio.charset.Charset; -import java.nio.file.*; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.HashSet; -import java.util.Set; - -public class LocalThread implements Runnable { - - private static final int WAIT_INTERVAL = 100; - private static final int MAX_TRIALS = 4; - - private final Invocation invoc; - private final Log log; - private final Path buildDir; - private final TicketSrcActor ticketSrc; - private final BaseCreActor cre; - private final Path centralRepo; - private final Path workDir; - - public LocalThread(TicketSrcActor ticketSrc, BaseCreActor cre, - Ticket ticket, Path buildDir, Path centralRepo, Path workDir ) { - - if( buildDir == null ) - throw new NullPointerException( "Build directory must not be null." ); - - if( !Files.exists( buildDir ) ) - throw new RuntimeException( "Build directory does not exist." ); - - if( !Files.isDirectory( buildDir ) ) - throw new RuntimeException( "Build path expected to be a directory." ); - - if( cre == null ) - throw new NullPointerException( "CRE actor must not be null." ); - - if( ticketSrc == null ) - throw new NullPointerException( "Ticket source must not be null." ); - - if( centralRepo == null ) - throw new NullPointerException( "Central repository must not be null." ); - - if( !Files.exists( centralRepo ) ) - throw new RuntimeException( "Central repository does not exist." ); - - if( !Files.isDirectory( centralRepo ) ) - throw new RuntimeException( "Central repository path expected to be a directory." ); - - if( workDir == null ) - throw new IllegalArgumentException( "Working directory must not be null." ); - - if( !Files.exists( workDir ) ) - throw new RuntimeException( "Working directory does not exist." ); - - if( !Files.isDirectory( workDir ) ) - throw new RuntimeException( "Working directory expected to be a directory." ); - - this.ticketSrc = ticketSrc; - this.cre = cre; - this.buildDir = buildDir; - this.centralRepo = centralRepo; - this.workDir = workDir; - - invoc = Invocation.createInvocation( ticket ); - log = LogFactory.getLog( LocalThread.class ); - } - - @Override - public void run() { - - Path scriptFile, location, successMarker, reportFile, stdErrFile, stdOutFile; - // Path lockMarker; - Process process; - int exitValue; - Set report; - JsonReportEntry entry; - String line; - StringBuffer buf; - Path srcPath, destPath; - ProcessBuilder processBuilder; - Ticket ticket; - String script, stdOut, stdErr; - long tic, toc; - JSONObject obj; - Message msg; - Charset cs; - int trial; - boolean suc; - Exception ex; - Path parent; - - - if( log.isDebugEnabled() ) - log.debug( "Starting up local thread for ticket "+invoc.getTicketId()+"." ); - - - if( invoc == null ) - throw new NullPointerException( "Invocation must not be null." ); - - - ticket = invoc.getTicket(); - process = null; - stdOut = null; - stdErr = null; - // lockMarker = null; - script = null; - successMarker = null; - cs = Charset.forName( "UTF-8" ); - - try { - - - - // callLocation = Paths.get( System.getProperty( "user.dir" ) ); - location = buildDir.resolve( String.valueOf( invoc.getTicketId() ) ); - // lockMarker = location.resolve( Invocation.LOCK_FILENAME ); - successMarker = location.resolve( Invocation.SUCCESS_FILENAME ); - reportFile = location.resolve( Invocation.REPORT_FILENAME ); - script = invoc.toScript(); - - // if( Files.exists( lockMarker ) ) - // throw new IOException( "Lock held on ticket "+invoc.getTicketId() ); - - if( !Files.exists( successMarker ) ) { - - deleteIfExists( location ); - Files.createDirectories( location ); - - // Files.createFile( lockMarker ); - - scriptFile = invoc.getExecutablePath( location ); - - Files.createFile( scriptFile, - PosixFilePermissions.asFileAttribute( - PosixFilePermissions.fromString( "rwxr-x---" ) ) ); - - // write executable script - try( BufferedWriter writer = Files.newBufferedWriter( scriptFile, cs, StandardOpenOption.CREATE ) ) { - writer.write( script ); - } - - // write executable log entry - try( BufferedWriter writer = Files.newBufferedWriter( reportFile, cs, StandardOpenOption.CREATE ) ) { - writer.write( ticket.getExecutableLogEntry().toString() ); - writer.write( '\n' ); - } - - for( String filename : invoc.getStageInList() ) { - - - - if( filename.charAt( 0 ) == '/' ) - throw new UnsupportedOperationException( "Absolute path encountered '"+filename+"'." ); - - srcPath = centralRepo.resolve( filename ); - destPath = location.resolve( filename ); - if( destPath == null ) - throw new NullPointerException( "Link destination path must not be null." ); - - if( !Files.exists( srcPath ) ) { - - srcPath = workDir.resolve( filename ); - if( log.isTraceEnabled() ) - log.trace( "Resolving relative path '"+srcPath+"'." ); - } - else - - if( log.isTraceEnabled() ) - log.trace( "Resolving path to central repository '"+srcPath+"'." ); - - if( log.isTraceEnabled() ) - log.trace( "Trying to create symbolic link from '"+srcPath+"' to '"+destPath+"'." ); - - parent = destPath.getParent(); - if( parent == null ) - throw new NullPointerException( "Parent path of destination path must not be null." ); - - if( !Files.exists( parent ) ) - Files.createDirectories( parent ); - - Files.createSymbolicLink( destPath, srcPath ); - - } - - // run script - processBuilder = new ProcessBuilder( invoc.getCmd() ); - processBuilder.directory( location.toFile() ); - - stdOutFile = location.resolve( Invocation.STDOUT_FILENAME ); - stdErrFile = location.resolve( Invocation.STDERR_FILENAME ); - - processBuilder.redirectOutput( stdOutFile.toFile() ); - processBuilder.redirectError( stdErrFile.toFile() ); - - trial = 1; - suc = false; - ex = null; - tic = System.currentTimeMillis(); - do { - try { - process = processBuilder.start(); - - suc = true; - } - catch( IOException e ) { - - ex = e; - if( log.isWarnEnabled() && trial > 1) // one retry is normal, 2 is a WARN - log.warn( "Retrying "+ ( ++trial ) +"th time. Waiting "+WAIT_INTERVAL+"ms: "+e.getMessage() ); - Thread.sleep( WAIT_INTERVAL ); - } - } while( suc == false && trial <= MAX_TRIALS ); - - if( process == null ) { - - if( log.isInfoEnabled() ) - log.info( "Sending ticket failed message because the received process object was null." ); - - ticketSrc.sendMsg( new TicketFailedMsg( cre, ticket, ex, script, null, null ) ); - // Files.delete( lockMarker ); - return; - } - - - exitValue = process.waitFor(); - toc = System.currentTimeMillis(); - - try( BufferedWriter writer = Files.newBufferedWriter( reportFile, cs, StandardOpenOption.APPEND ) ) { - - obj = new JSONObject(); - obj.put( JsonReportEntry.LABEL_REALTIME, toc-tic ); - entry = invoc.createJsonReportEntry( tic, JsonReportEntry.KEY_INVOC_TIME, obj ); - - writer.write( entry.toString() ); - writer.write( '\n' ); - - try( BufferedReader reader = Files.newBufferedReader( stdOutFile, cs ) ) { - - buf = new StringBuffer(); - while( ( line = reader.readLine() ) != null ) - buf.append( line ).append( '\n' ); - - - stdOut = buf.toString(); - - if( !stdOut.isEmpty() ) { - entry = invoc.createJsonReportEntry( JsonReportEntry.KEY_INVOC_STDOUT, stdOut ); - writer.write( entry.toString() ); - writer.write( '\n' ); - } - } - - try( BufferedReader reader = Files.newBufferedReader( stdErrFile, cs ) ) { - - buf = new StringBuffer(); - while( ( line = reader.readLine() ) != null ) - buf.append( line ).append( '\n' ); - - stdErr = buf.toString(); - if( !stdErr.isEmpty() ) { - - entry = invoc.createJsonReportEntry( JsonReportEntry.KEY_INVOC_STDERR, stdErr ); - writer.write( entry.toString() ); - writer.write( '\n' ); - } - } - - if( exitValue == 0 ) - - Files.createFile( successMarker ); - - else { - - if( log.isInfoEnabled() ) - log.info( "Sending ticket failed message because exit value was non-zero." ); - - ticketSrc.sendMsg( new TicketFailedMsg( cre, ticket, null, script, stdOut, stdErr ) ); - // Files.delete( lockMarker ); - return; - - } - } - - - } - - // gather report - report = new HashSet<>(); - try( BufferedReader reader = Files.newBufferedReader( reportFile, cs ) ) { - - while( ( line = reader.readLine() ) != null ) { - - line = line.trim(); - - if( line.isEmpty() ) - continue; - - entry = new JsonReportEntry( line ); - - // If the report comes from the hard cache then the run id - // is different from the run id of this invocation. This is - // corrected here. - entry.setRunId( invoc.getRunId() ); - - report.add( entry ); - } - - } - - invoc.evalReport( report ); - - // create link in central data repository - for( String f : invoc.getStageOutList() ) { - - srcPath = location.resolve( f ); - destPath = centralRepo.resolve( f ); - - if( Files.exists( destPath ) ) - continue; - - if( log.isTraceEnabled() ) - log.trace( "Creating link from "+srcPath+" to "+destPath+"." ); - - Files.createSymbolicLink( destPath, srcPath ); - } - - - ticketSrc.sendMsg( new TicketFinishedMsg( cre, invoc.getTicket(), report ) ); - - if( log.isTraceEnabled() ) - log.trace( "Local thread ran through without exception." ); - - // Files.deleteIfExists( lockMarker ); - - } - catch( InterruptedException e ) { - - if( log.isTraceEnabled() ) - log.trace( "Local thread has been interrupted." ); - } - catch( Exception e ) { - - if( log.isTraceEnabled() ) - log.trace( "Something went wrong. Deleting success marker if present." ); - - if( successMarker != null ) - try { - Files.deleteIfExists( successMarker ); - } - catch( IOException e1 ) { - e1.printStackTrace(); - } - - if( log.isInfoEnabled() ) - log.info( "Sending ticket failed message because an exception was caught: "+e.getMessage() ); - - msg = new TicketFailedMsg( cre, ticket, e, script, stdOut, stdErr ); - - ticketSrc.sendMsg( msg ); - - } - finally { - - - if( process != null ) { - - if( log.isDebugEnabled() ) - log.debug( "Stopping local thread for ticket "+invoc.getTicketId()+"." ); - - process.destroy(); - } - } - } - - public static void deleteIfExists( Path f ) throws IOException { - - if( !Files.exists( f, LinkOption.NOFOLLOW_LINKS ) ) - return; - - if( Files.isDirectory( f ) ) - try( DirectoryStream stream = Files.newDirectoryStream( f ) ) { - for( Path p : stream ) - deleteIfExists( p ); - } - - Files.delete( f ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/TicketReadyMsg.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/TicketReadyMsg.java deleted file mode 100644 index 393a77f..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/cre/TicketReadyMsg.java +++ /dev/null @@ -1,74 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.cre; - -import java.util.UUID; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.actormodel.Message; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class TicketReadyMsg extends Message { - - private final UUID queryId; - private final Ticket ticket; - - public TicketReadyMsg( Actor sender, UUID queryId, Ticket ticket ) { - - super( sender ); - - if( ticket == null ) - throw new NullPointerException( "Ticket set must not be null." ); - - if( queryId == null ) - throw new NullPointerException( "Run ID must not be null." ); - - this.queryId = queryId; - this.ticket = ticket; - } - - public UUID getQueryId() { - return queryId; - } - - public Ticket getTicket() { - return ticket; - } - - @Override - public String toString() { - return "{ ticketReady, \""+queryId+"\", "+ticket.getTicketId()+", \""+ticket.toString().replace( '\n', ' ' )+"\" }"; - } - - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/BashInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/BashInvocation.java deleted file mode 100644 index 352cc87..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/BashInvocation.java +++ /dev/null @@ -1,451 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.invoc; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class BashInvocation extends Invocation { - - private static String bashAllFrom( String varname ) { - - if( varname == null ) - throw new NullPointerException( "Varname must not be null." ); - - if( varname.isEmpty() ) - throw new RuntimeException( "Varname must not be empty." ); - - return bashArrayGet( varname, "@" ); - } - - private static String bashArrayGet( String varName, String index ) { - - String v, i; - - if( varName == null ) - throw new NullPointerException( "Varname must not be null." ); - - v = varName.trim(); - - if( v.isEmpty() ) - throw new RuntimeException( "Varname must not be empty." ); - - if( v.startsWith( "$" ) ) - throw new RuntimeException( "Varname '"+varName+"' already dereferenced." ); - - if( v.contains( "\n" ) ) - throw new RuntimeException( - "Varname cannot be a multi-line expression." ); - - if( index == null ) - throw new NullPointerException( "Index must not be null." ); - - i = index.trim(); - - if( i.isEmpty() ) - throw new RuntimeException( "Index must not be empty." ); - - if( i.contains( "\n" ) ) - throw new RuntimeException( - "Index cannot be a multi-line expression." ); - - - - return "${"+v+"["+index+"]}"; - } - - private static String bashArraySize( String varName ) { - return "${#"+varName+"[@]}"; - } - - private static String bashIfNotEquals( String a, String b, String thenBlock ) { - - if( a == null ) - throw new NullPointerException( "Operand a must not be null." ); - - if( a.isEmpty() ) - throw new RuntimeException( "Operand a must not be empty." ); - - if( b == null ) - throw new NullPointerException( "Operand b must not be null." ); - - if( b.isEmpty() ) - throw new RuntimeException( "Operand b must not be empty." ); - - if( thenBlock == null ) - throw new NullPointerException( "Then block must not be null." ); - - if( thenBlock.isEmpty() ) - throw new RuntimeException( "Then block must not be empty." ); - - - String ret; - ret = "if [ \""+a+"\" -ne \""+b+"\" ]\nthen\n"+thenBlock+"\n"; - - ret += "fi\n"; - - return ret; - } - - private static final String BASH_SHEBANG = "#!/usr/bin/env bash\nset -eu -o pipefail\n"; - - - public BashInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - - if( name == null ) - throw new NullPointerException( "Function name must not be null." ); - - if( name.isEmpty() ) - throw new RuntimeException( "Function name must not be empty." ); - - if( argValue == null ) - throw new NullPointerException( "Parameter bindings must not be null." ); - - buf = new StringBuffer(); - - buf.append( '`' ).append( name ); - - for( String value : argValue ) - - if( value.trim().startsWith( "\"" ) && value.trim().endsWith( "\"" ) ) - buf.append( ' ' ).append( value ); - else - buf.append( " \"" ).append( value ).append( '"' ); - - - buf.append( '`' ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - - StringBuffer buf; - - if( name == null ) - throw new NullPointerException( "Function name must not be null." ); - - if( name.isEmpty() ) - throw new RuntimeException( "Function name must not be empty." ); - - if( argValue == null ) - throw new NullPointerException( "Parameter bindings must not be null." ); - - buf = new StringBuffer(); - - buf.append( name ); - - for( String value : argValue ) - - if( value.trim().startsWith( "\"" ) && value.trim().endsWith( "\"" ) ) - buf.append( ' ' ).append( value ); - else - buf.append( " \"" ).append( value ).append( '"' ); - - - buf.append( '\n' ); - - return buf.toString(); - } - - @Override - protected String clip(String varName) { - return varName+"=${"+varName+":1}\n"; - } - - @Override - protected String comment( String comment ) { - return "# "+comment.replace( "\n", "\n# " )+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return newList( to )+varDef( to, bashAllFrom( from ) ); - } - - private String defFunction( String funName, String outputName, String[] inputNameList, String body ) { - - int i; - - if( funName == null ) - throw new NullPointerException( "Function name must not be null." ); - - if( funName.isEmpty() ) - throw new RuntimeException( "Function name must not be empty." ); - - if( body == null ) - throw new NullPointerException( "Function body must not be null." ); - - if( body.isEmpty() ) - throw new RuntimeException( "Function body must not be empty." ); - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( funName ).append( "() {\n" ); - - i = 0; - for( String inputName : inputNameList ) - buf.append( - varDef( inputName, dereference( String.valueOf( ++i ) ) ) ); - - buf.append( body ); - - if( outputName != null ) - buf.append( "echo " ).append( dereference( outputName ) ); - - buf.append( "\n}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "echo -e \"{" ).append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"$1\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":$2}\" >> " ) - .append( REPORT_FILENAME ); - - return defFunction( FUN_LOG, null, new String[] { "key", "payload" }, buf.toString() ); - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "echo -e \"{" ).append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_FILE ).append( ":\\\"$1\\\"," ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"$2\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":$3}\" >> " ) - .append( REPORT_FILENAME ); - - return defFunction( FUN_LOGFILE, null, new String[] { "file", "key", "payload" }, buf.toString() ); - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - return defFunction( - FUN_NORMALIZE, - null, - new String[] { "channel", "f" }, - "echo "+getTicketId()+"_"+"${channel}_${f##*/}\n" ); - } - - @Override - protected String dereference(String varName) { - return "$"+varName; - } - - @Override - protected String fileSize(String filename) { - // return "`du -b -L "+filename+" | awk '{print $1}'`"; - // return "`stat -c %s "+filename+"`"; - return "`du -k -L "+filename+" | awk '{print $1}'`"; - } - - @Override - protected String forEach(String listName, String elementName, String body) { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "for " ).append( elementName ).append( " in "+bashAllFrom( listName )+"\ndo\n" ); - buf.append( body ); - buf.append( "\ndone\n" ); - - return buf.toString(); - } - - @Override - protected String getImport() { - return ""; - } - - @Override - protected String getShebang() { - return BASH_SHEBANG; - } - - @Override - protected String ifListIsNotEmpty( String listName, String body ) { - return bashIfNotEquals( bashArraySize( listName ), "0", body ); - } - - @Override - protected String ifNotFileExists( String filename, String body ) { - - String ret; - - if( filename == null ) - throw new NullPointerException( "Filename must not be null." ); - - if( filename.isEmpty() ) - throw new RuntimeException( "Filename must not be empty." ); - - if( body == null ) - throw new NullPointerException( "Then block must not be null." ); - - if( body.isEmpty() ) - throw new RuntimeException( "Then block must not be empty." ); - - - ret = "if [ ! -e "+filename+" ]\nthen\n"+body+"\n"; - - ret += "fi\n"; - - return ret; - } - - @Override - protected String join( String... elementList ) { - - StringBuffer buf; - - buf = new StringBuffer(); - for( String element : elementList ) - buf.append( element ); - return buf.toString(); - } - - @Override - protected String listAppend( String listName, String element ) { - return listName+"=("+bashAllFrom( listName )+" "+element+")\n"; - } - - @Override - protected String listToBraceCommaSeparatedString( String src, String dest, String open, String close ) { - return dest+"=`printf \",%s\" ${"+src+"[@]}`\n"+dest+"="+open+"${"+dest+":1}"+close+"\n"; - } - - @Override - protected String newList( String listName ) { - return listName+"=\"\"\n"; - } - - @Override - protected String quote( String content ) { - - String ret; - - ret = "\""+content.replace( "\"", "\\\"" )+"\""; - - return ret; - } - - @Override - protected String raise( String msg ) { - return "echo "+msg+" >&2\nexit -1"; - } - - @Override - protected String symlink( String src, String dest ) { - // return "mv -f "+src+" "+dest+"\n"; - return "ln -s "+src+" "+dest+"\n"; - } - - @Override - protected String varDef( String varname, CompoundExpr list ) throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( varname ).append( "=( " ); - for( String s : list.normalize() ) - buf.append( '"' ).append( s ).append( "\" " ); - buf.append( ")\n" ); - - return buf.toString(); - - } - - @Override - protected String varDef( String varname, String value ) { - return varname+"="+value+"\n"; - } - - @Override - protected String getLibPath() { - throw new UnsupportedOperationException( "NYI" ); - } - - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/BeanshellInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/BeanshellInvocation.java deleted file mode 100644 index 683c42a..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/BeanshellInvocation.java +++ /dev/null @@ -1,273 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.invoc; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class BeanshellInvocation extends Invocation { - - public BeanshellInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( name ).append( '(' ); - - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( arg ); - } - - buf.append( ")" ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - return callFunction( name, argValue )+";\n"; - } - - @Override - protected String clip( String varName ) { - return varName+"="+varName+".substring(1);\n"; - } - - @Override - protected String comment( String comment ) { - return "// "+comment+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return to+"=new ArrayList();\nto.addAll("+from+");\n"; - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "void "+FUN_LOG+"( String key, String value ) {\n" ); - - buf.append( " FileWriter writer = new FileWriter(\""+REPORT_FILENAME+"\",true);\n" ); - buf.append( " writer.write(\"{" ); - - buf.append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"\"+key+\"\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":\"+value+\"}\\n\"" ); - - buf.append( ");\n writer.close();\n" ); - - buf.append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "void "+FUN_LOGFILE+"( String file, String key, String value ) {\n" ); - - buf.append( " FileWriter writer = new FileWriter(\""+REPORT_FILENAME+"\",true);\n" ); - buf.append( " writer.write(\"{" ); - - buf.append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_FILE ).append( ":\\\"\"+file+\"\\\"," ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"\"+key+\"\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":\"+value+\"}\\n\"" ); - - buf.append( ");\n writer.close();\n" ); - - buf.append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "String "+FUN_NORMALIZE+"(int channel, String filename) {" ); - buf.append( " Path p = Paths.get(filename, new String[] {});\n"); - buf.append( " return \""+getTicketId()+"_\"+channel+\"_\"+p.getName(p.getNameCount()-1);\n" ); - buf.append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String dereference( String varName ) { - return varName; - } - - @Override - protected String fileSize( String filename ) { - return "Files.size(Paths.get("+filename+")).toString()"; - } - - @Override - protected String forEach( String listName, String elementName, String body ) { - return "for(String "+elementName+":"+listName+") {\n"+body+"\n}\n"; - } - - @Override - protected String getShebang() { - return "#!/usr/bin/env bsh"; - } - - @Override - protected String getLibPath() { - throw new UnsupportedOperationException( "NYI" ); - } - - @Override - protected String getImport() { - return "import java.nio.file.Paths;\nimport java.io.FileWriter;\n" - +"import java.nio.file.Files;\nimport java.nio.file.Path;\nimport java.nio.file.attribute.FileAttribute;\n"; - } - - @Override - protected String ifListIsNotEmpty(String listName, String body) { - return( "if(!"+listName+".isEmpty()) {\n"+body+"\n}\n" ); - } - - @Override - protected String ifNotFileExists(String fileName, String body) { - return( "if(!Files.exists(Paths.get("+fileName+"))) {\n"+body+"\n}\n" ); - } - - @Override - protected String declareString( String outputName ) { - return "String "+outputName+";\n"; - } - - @Override - protected String declareList( String paramName ) { - return "List "+paramName+";\n"; - } - - @Override - protected String join( String... elementList ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - comma = false; - for( String element : elementList ) { - - if( comma ) - buf.append( "+" ); - comma = true; - - buf.append( element ); - } - - return buf.toString(); - } - - @Override - protected String listAppend(String listName, String element) { - return listName+"="+listName+":+"+element+"\n"; - } - - @Override - protected String listToBraceCommaSeparatedString(String listName, String stringName, String open, String close) { - return stringName+"=\""+open+"\"+"+listName+".mkString(\",\")+\""+close+"\"\n"; - } - - @Override - protected String newList( String listName ) { - return "var "+listName+"=List()\n"; - } - - @Override - protected String quote( String content ) { - return "\""+content.replace( "\"", "\\\"" )+"\""; - } - - @Override - protected String raise( String msg ) { - return "throw new RuntimeException("+msg+")\n"; - } - - @Override - protected String symlink( String src, String dest ) { - return "Files.createSymbolicLink(Paths.get("+dest+", new String[] {}),Paths.get("+src+", new String[] {}), new FileAttribute[] {});\n"; - } - - @Override - protected String varDef( String varname, String value ) { - return varname+"="+value+";\n"; - } - - @Override - protected String varDef( String varname, CompoundExpr ce ) throws NotDerivableException { - - StringBuffer buf; - int i, n; - - buf = new StringBuffer(); - - buf.append( varname+"=new ArrayList();\n" ); - - n = ce.getNumAtom(); - for( i = 0; i < n; i++ ) - buf.append( varname+".add(\""+ce.getStringExprValue( i )+"\");\n" ); - - return buf.toString(); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/Invocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/Invocation.java deleted file mode 100644 index 281fc29..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/Invocation.java +++ /dev/null @@ -1,886 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.invoc; - -import java.nio.file.Path; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import org.json.JSONArray; -import org.json.JSONException; -import org.json.JSONObject; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.DataType; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Param; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ReduceVar; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Type; - -public abstract class Invocation { - - private static final Map libPathMap = new HashMap<>(); - - protected static final String FUN_LOG = "cflogmsg"; - protected static final String FUN_LOGFILE = "cflogfilemsg"; - protected static final String FUN_NORMALIZE = "cfnormalize"; - public static final String REPORT_FILENAME = "__report__.txt"; - public static final String SCRIPT_NAME = "cfscript"; - public static final String SUCCESS_FILENAME = "__success__"; - public static final String STDOUT_FILENAME = "__stdout__.txt"; - public static final String STDERR_FILENAME = "__stderr__.txt"; - public static final String LOCK_FILENAME = "__lock__"; - - - protected final String libPath; - private final Ticket ticket; - - public Invocation( Ticket ticket, String libPath ) { - - if( ticket == null ) - throw new NullPointerException( "Ticket must not be null." ); - - this.ticket = ticket; - this.libPath = libPath; - } - - public void evalReport( Set report ) throws JSONException { - - if( report == null ) - throw new NullPointerException( "Report entry set must not be null." ); - - for( JsonReportEntry entry : report ) - evalReport( entry ); - } - - public void evalReport( JsonReportEntry entry ) throws JSONException { - - JSONObject obj; - JSONArray array; - CompoundExpr ce; - int i, n; - - if( entry == null ) - throw new NullPointerException( "Report entry must not be null." ); - - - if( !entry.getKey().equals( JsonReportEntry.KEY_INVOC_OUTPUT ) ) - return; - - - obj = entry.getValueJsonObj(); - - - for( NameExpr nameExpr : ticket.getOutputList() ) { - - array = obj.getJSONArray( nameExpr.getId() ); - ce = new CompoundExpr(); - n = array.length(); - - for( i = 0; i < n; i++ ) - ce.addSingleExpr( new StringExpr( array.getString( i ) ) ); - - ticket.setValue( nameExpr, ce ); - } - - } - - @SuppressWarnings( "static-method" ) - public String[] getCmd() { - return new String[] { "./"+SCRIPT_NAME }; - } - - @SuppressWarnings( "static-method" ) - public Path getExecutablePath( Path location ) { - return location.resolve( SCRIPT_NAME ); - } - - public JsonReportEntry getExecutableLogEntry() { - return ticket.getExecutableLogEntry(); - } - - public JsonReportEntry getScriptLogEntry() throws NotBoundException, NotDerivableException { - return new JsonReportEntry( ticket, JsonReportEntry.KEY_INVOC_SCRIPT, toScript() ); - } - - public String getFunDef() throws NotDerivableException { - return defFunctionLog() - +defFunctionNormalize() - +defFunctionLogFile(); - } - - public String getLangLabel() { - return ticket.getLangLabel(); - } - - public Set getStageInList() throws NotDerivableException { - - CompoundExpr ce; - Set set; - - try { - - set = new HashSet<>(); - - for( NameExpr nameExpr : ticket.getNameSet() ) - - if( isParamStage( nameExpr.getId() ) ) { - - ce = ticket.getExpr( nameExpr ); - for( String s : ce.normalize() ) - set.add( s ); - } - - return set; - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - } - - /** Get the union of all files that have been created by this invocation. - * - * All the files created in the execution of the invocation are subsumed - * independent of the channel in which they appear. - * - * This method assumes that the method Invocation.evalReport() has been - * called with the report created during execution of this invocation. - * Otherwise, a NotBoundException will be rethrown as a RuntimeException. - * - * @return A set of strings containing relative file names. - * @throws NotDerivableException if normalization fails - */ - public Set getStageOutList() throws NotDerivableException { - - CompoundExpr ce; - Set set; - - try { - - set = new HashSet<>(); - - for( NameExpr nameExpr : ticket.getOutputList() ) - - if( isOutputStage( nameExpr.getId() ) ) { - - ce = ticket.getOutputValue( nameExpr ); - for( String s : ce.normalize() ) - set.add( s ); - } - - return set; - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - } - - public UUID getRunId() { - return ticket.getRunId(); - } - - public long getTaskId() { - return ticket.getLambdaId(); - } - - public String getTaskName() { - return ticket.getTaskName(); - } - - public Ticket getTicket() { - return ticket; - } - - public long getTicketId() { - return ticket.getTicketId(); - } - - public boolean hasTaskName() { - return ticket.hasTaskName(); - } - - public boolean hasLibPath() { - return libPath != null; - } - - public String toScript() throws NotBoundException, NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - // insert shebang - buf.append( getShebang() ).append( '\n' ); - - // import libraries - buf.append( comment( "import libraries" ) ); - buf.append( getImport() ).append( '\n' ); - - // modify library path - buf.append( comment( "modify library path" ) ); - if( hasLibPath() ) - buf.append( getLibPath() ); - - // define necessary functions - buf.append( comment( "define necessary functions" ) ); - buf.append( getFunDef() ).append( '\n' ); - - // bind single output variables to default values - buf.append( comment( "bind single output variables to default values" ) ); - for( String outputName : getSingleOutputNameSet() ) { - buf.append( declareString( outputName ) ); - buf.append( - varDef( - outputName, - quote( outputName ) ) ); - } - buf.append( '\n' ); - - // bind input parameters - buf.append( comment( "bind input parameters" ) ); - for( String paramName : getSingleParamNameSet() ) { - - if( paramName.equals( CfSemanticModelVisitor.LABEL_TASK ) ) - continue; - - buf.append( declareString( paramName ) ); - buf.append( varDef( paramName, quote( getResolveableBoundToSingleParam( paramName ) ) ) ); - } - for( String paramName : getReduceParamNameSet() ) { - - buf.append( declareList( paramName ) ); - buf.append( varDef( paramName, getReduceParam( paramName ) ) ); - } - buf.append( '\n' ); - - // report stage in file sizes and report error when something is missing - buf.append( comment( "report stage in file sizes and report error when something is missing" ) ); - buf.append( getStageInCollect() ).append( '\n' ); - - // insert body prefix - buf.append( comment( "insert body prefix" ) ); - buf.append( getBodyPrefix() ).append( '\n' ); - - - // insert function body - buf.append( comment( "insert function body" ) ); - buf.append( postProcess( ticket.getBody() ) ).append( '\n' ); - - // rename output files - buf.append( comment( "rename output files" ) ); - buf.append( getOutputRename() ).append( '\n' ); - - // collect output variables - buf.append( comment( "collect output variables" ) ); - buf.append( getOutputCollect() ).append( '\n' ); - - // collect stage out information - buf.append( comment( "collect stage out information" ) ); - buf.append( getStageOutCollect() ).append( '\n' ); - - return buf.toString(); - } - - @SuppressWarnings({ "static-method", "unused" }) - protected String declareList( String paramName ) { - return ""; - } - - @SuppressWarnings({ "static-method", "unused" }) - protected String declareString( String outputName ) { - return ""; - } - - @SuppressWarnings( "static-method" ) - protected String getImport() { - return ""; - } - - @SuppressWarnings("static-method") - protected String getBodyPrefix() { - return ""; - } - - protected abstract String callFunction( String name, String... argValue ); - protected abstract String callProcedure( String name, String... argValue ); - - /** Removes the first character of a string. - * - * @param varName The name of the variable that holds the input string. - * @return A statement in the foreign language. - */ - protected abstract String clip( String varName ); - - protected abstract String comment( String comment ); - protected abstract String copyArray( String from, String to ); - protected abstract String defFunctionLog() throws NotDerivableException; - protected abstract String defFunctionLogFile() throws NotDerivableException; - protected abstract String defFunctionNormalize() throws NotDerivableException; - protected abstract String dereference( String varName ); - protected abstract String fileSize( String filename ); - protected abstract String forEach( String listName, String elementName, String body ); - protected abstract String getShebang(); - protected abstract String getLibPath(); - protected abstract String ifListIsNotEmpty( String listName, String body ); - - protected abstract String ifNotFileExists( String fileName, String body ); - protected abstract String join( String ... elementList ); - protected abstract String listAppend( String listName, String element ); - protected abstract String listToBraceCommaSeparatedString( String listName, String stringName, String open, String close ); - protected abstract String newList( String listName ); - protected abstract String quote( String content ); - protected abstract String raise( String msg ); - protected abstract String symlink( String src, String dest ); - protected abstract String varDef( String varname, String value ); - protected abstract String varDef( String varname, CompoundExpr ce ) throws NotDerivableException; - - private boolean isOutputStage( String outputName ) { - - Prototype prototype; - Type type; - DataType dataType; - - prototype = ticket.getPrototype(); - - for( NameExpr nameExpr : prototype.getOutputList() ) - - if( nameExpr.getId().equals( outputName ) ) { - - if( !nameExpr.hasType() ) - return false; - - type = nameExpr.getType(); - if( !( type instanceof DataType ) ) - return false; - - dataType = ( DataType )type; - - return dataType.getId().equals( CfSemanticModelVisitor.LABEL_FILE ); - } - - throw new RuntimeException( "Output not found." ); - } - - private boolean isParamStage( String paramName ) { - - Prototype prototype; - Type type; - DataType dataType; - - prototype = ticket.getPrototype(); - - for( NameExpr nameExpr : prototype.getParamNameSet() ) - - if( nameExpr.getId().equals( paramName ) ) { - - if( !nameExpr.hasType() ) - return false; - - type = nameExpr.getType(); - if( !( type instanceof DataType ) ) - return false; - - dataType = ( DataType )type; - - return dataType.getId().equals( CfSemanticModelVisitor.LABEL_FILE ); - } - - throw new RuntimeException( "Output not found." ); - } - - private int getOutputChannel( String outputName ) { - - Prototype prototype; - int i, n; - NameExpr output; - - prototype = ticket.getPrototype(); - n = prototype.getNumOutput(); - - - for( i = 0; i < n; i++ ) { - - output = prototype.getOutput( i ); - - if( output.getId().equals( outputName ) ) - return i+1; - } - - throw new RuntimeException( "Output not found." ); - } - - protected String getOutputCollect() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( declareString( "CFSTR" ) ); - - buf.append( varDef( "CFSTR", quote( "" ) ) ); - for( String outputName : getSingleOutputNameSet() ) - - buf.append( - varDef( - "CFSTR", - join( - dereference( "CFSTR" ), - quote( "," ), - quote( outputName+":[\"" ), - dereference( outputName ), - quote( "\"]" ) ) ) ); - - - buf.append( declareString( "CFSTR1" ) ); - buf.append( declareString( "CFI" ) ); - for( String outputName : getReduceOutputNameSet() ) { - - buf - .append( varDef( "CFSTR1", quote( "" ) ) ) - .append( - forEach( - outputName, - "CFI", - varDef( - "CFSTR1", - join( - dereference( "CFSTR1" ), - quote( ",\"" ), - dereference( "CFI" ), - quote( "\"" ) ) ) ) ) - .append( clip( "CFSTR1" ) ) - .append( - varDef( - "CFSTR", - join( - dereference( "CFSTR" ), - quote( "," ), - quote( outputName+":[" ), - dereference( "CFSTR1" ), - quote( "]" ) ) ) ); - } - - buf - .append( clip( "CFSTR" ) ) - - .append( - varDef( - "CFSTR", - join( - quote( "{" ), - dereference( "CFSTR" ), - quote( "}" ) ) ) ) - - .append( - callProcedure( - FUN_LOG, - quote( JsonReportEntry.KEY_INVOC_OUTPUT ), - dereference( "CFSTR" ) ) ).append( '\n' ); - - - return buf.toString(); - - } - - protected String getOutputRename() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( declareString( "CFFILENAME" ) ); - - for( String outputName : getSingleOutputNameSet() ) - - if( isOutputStage( outputName ) ) { - - buf.append( - varDef( - "CFFILENAME", - callFunction( - FUN_NORMALIZE, - String.valueOf( getOutputChannel( outputName ) ), - dereference( outputName ) ) ) ); - - buf.append( - symlink( - dereference( outputName ), - dereference( "CFFILENAME" ) ) ); - - buf.append( - varDef( - outputName, - dereference( "CFFILENAME" ) ) ); - } - - for( String outputName : getReduceOutputNameSet() ) - - if( isOutputStage( outputName ) ) { - - buf.append( newList( "CFLIST" ) ); - - buf.append( - forEach( - outputName, - "CFFILENAME", - - varDef( - "CFNEWFILENAME", - callFunction( - FUN_NORMALIZE, - String.valueOf( - getOutputChannel( outputName ) ), - dereference( "CFFILENAME" ) ) ) - - +listAppend( "CFLIST", dereference( "CFNEWFILENAME" ) ) - - +symlink( - dereference( "CFFILENAME" ), - dereference( "CFNEWFILENAME" ) ) ) ); - - // buf.append( varDef( outputName, dereference( "__LIST" ) ) ); - buf.append( copyArray( "CFLIST", outputName ) ); - } - - return buf.toString(); - } - - private Set getReduceOutputNameSet() { - - List outputList; - Set reduceList; - - outputList = ticket.getOutputList(); - reduceList = new HashSet<>(); - - for( NameExpr nameExpr : outputList ) - - if( nameExpr instanceof ReduceVar ) - reduceList.add( nameExpr.getId() ); - - return reduceList; - } - - protected Set getReduceParamNameSet() { - - Set paramSet; - Set reduceList; - - paramSet = ticket.getParamSet(); - reduceList = new HashSet<>(); - - for( Param param : paramSet ) - - if( param instanceof ReduceVar ) - reduceList.add( ( ( ReduceVar )param ).getId() ); - - - return reduceList; - } - - protected CompoundExpr getReduceParam( String paramName ) throws NotBoundException { - return ticket.getExpr( paramName ); - } - - protected String getResolveableBoundToSingleParam( String paramName ) - throws NotBoundException, NotDerivableException { - return ticket.getExpr( paramName ).normalize().get( 0 ); - } - - protected Set getSingleOutputNameSet() { - - List outputList; - Set nonReduceList; - - outputList = ticket.getOutputList(); - nonReduceList = new HashSet<>(); - - for( NameExpr nameExpr : outputList ) - - if( !( nameExpr instanceof ReduceVar ) ) - nonReduceList.add( nameExpr.getId() ); - - return nonReduceList; - } - - protected Set getSingleParamNameSet() { - - Set paramSet; - Set nonReduceList; - - paramSet = ticket.getParamSet(); - nonReduceList = new HashSet<>(); - - for( Param param : paramSet ) - - if( !( param instanceof ReduceVar ) ) - for( NameExpr nameExpr : param.getNameExprSet() ) - nonReduceList.add( nameExpr.getId() ); - - return nonReduceList; - - } - - protected String getStageInCollect() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( declareString( "SIZE" ) ); - - - for( String inputName : getSingleParamNameSet() ) - if( isParamStage( inputName ) ) - buf - .append( ifNotFileExists( - dereference( inputName ), - raise( join( quote( "Stage in: A file " ), dereference( inputName ), - quote( " should be present but has not been found." ) ) ) ) ) - - .append( varDef( "SIZE", fileSize( dereference( inputName ) ) ) ) - - .append( - callProcedure( - FUN_LOGFILE, - dereference( inputName ), - quote( JsonReportEntry.KEY_FILE_SIZE_STAGEIN ), - dereference( "SIZE" ) ) ).append( '\n' ); - - - for( String inputName : getReduceParamNameSet() ) - if( isParamStage( inputName ) ) - - buf.append( - forEach( - inputName, - "CFI", - ifNotFileExists( - dereference( "CFI" ), - raise( - join( - quote( "Stage in: A file " ), - dereference( "CFI" ), - quote( " should be present but has not been found." ) ) ) ) - - - +varDef( "SIZE", fileSize( dereference( "CFI" ) ) ) - - +callFunction( - FUN_LOGFILE, - dereference( "CFI" ), - quote( JsonReportEntry.KEY_FILE_SIZE_STAGEIN ), - dereference( "SIZE" ) ) ) ); - - return buf.toString(); - } - - protected String getStageOutCollect() { - - StringBuffer buf; - - buf = new StringBuffer(); - - - for( String outputName : getSingleOutputNameSet() ) - if( isOutputStage( outputName ) ) - buf - .append( ifNotFileExists( - dereference( outputName ), - raise( join( quote( "Stage out: A file " ), dereference( outputName ), - quote( " should have been created but has not been found." ) ) ) ) ) - - .append( varDef( "SIZE", fileSize( dereference( outputName ) ) ) ) - - .append( - callProcedure( - FUN_LOGFILE, - dereference( outputName ), - quote( JsonReportEntry.KEY_FILE_SIZE_STAGEOUT ), - dereference( "SIZE" ) ) ).append( '\n' ); - - - for( String outputName : getReduceOutputNameSet() ) - if( isOutputStage( outputName ) ) - - buf.append( - forEach( - outputName, - "CFI", - ifNotFileExists( - dereference( "CFI" ), - raise( - join( - quote( "Stage out: A file " ), - dereference( "CFI" ), - quote( " should be present but has not been found." ) ) ) ) - - - +varDef( "SIZE", fileSize( dereference( "CFI" ) ) ) - - +callFunction( - FUN_LOGFILE, - dereference( "CFI" ), - quote( JsonReportEntry.KEY_FILE_SIZE_STAGEOUT ), - dereference( "SIZE" ) ) ) ); - - return buf.toString(); - } - - @SuppressWarnings("static-method") - protected String postProcess( String body ) { return body; } - - public static Invocation createInvocation( Ticket ticket ) { - - String label, libPath; - - label = ticket.getLangLabel(); - libPath = libPathMap.get( label ); - - switch( label ) { - - case ForeignLambdaExpr.LANGID_BASH : return new BashInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_R : return new RInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_PERL : return new PerlInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_MATLAB : return new MatlabInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_OCTAVE : return new OctaveInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_PYTHON : return new PythonInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_LISP : return new LispInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_PEGASUS : return new PegasusInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_JAVA : - case ForeignLambdaExpr.LANGID_BEANSHELL : return new BeanshellInvocation( ticket, libPath ); - case ForeignLambdaExpr.LANGID_SCALA : return new ScalaInvocation( ticket, libPath ); - default : throw new RuntimeException( "Language label '"+label+"' not recognized." ); - } - } - - public static void putLibPath( String langId, String libPath ) { - - if( langId == null ) - throw new NullPointerException( "Language id must not be null." ); - - libPathMap.put( langId, libPath ); - } - - - public JsonReportEntry createJsonReportEntry( String file, String key, String value ) { - return new JsonReportEntry( - getRunId(), - getTaskId(), - getTaskName(), - getLangLabel(), - getTicketId(), - file, - key, - value ); - } - - public JsonReportEntry createJsonReportEntry( long timestamp, String file, String key, String value ) { - return new JsonReportEntry( - timestamp, - getRunId(), - getTaskId(), - getTaskName(), - getLangLabel(), - getTicketId(), - file, - key, - value ); - } - - public JsonReportEntry createJsonReportEntry( long timestamp, String file, String key, JSONObject value ) { - return new JsonReportEntry( - timestamp, - getRunId(), - getTaskId(), - getTaskName(), - getLangLabel(), - getTicketId(), - file, - key, - value ); - } - - public JsonReportEntry createJsonReportEntry( String key, String value ) { - return new JsonReportEntry( - getRunId(), - getTaskId(), - getTaskName(), - getLangLabel(), - getTicketId(), - null, - key, - value ); - } - - public JsonReportEntry createJsonReportEntry( long timestamp, String key, String value ) { - return new JsonReportEntry( - timestamp, - getRunId(), - getTaskId(), - getTaskName(), - getLangLabel(), - getTicketId(), - null, - key, - value ); - } - - public JsonReportEntry createJsonReportEntry( long timestamp, String key, JSONObject value ) { - return new JsonReportEntry( - timestamp, - getRunId(), - getTaskId(), - getTaskName(), - getLangLabel(), - getTicketId(), - null, - key, - value ); - } - - - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/LispInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/LispInvocation.java deleted file mode 100644 index e13d5aa..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/LispInvocation.java +++ /dev/null @@ -1,159 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.invoc; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class LispInvocation extends Invocation { - - public LispInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction(String name, String... argValue) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String callProcedure(String name, String... argValue) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String clip(String varName) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String comment(String comment) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String copyArray(String from, String to) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String dereference(String varName) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String fileSize(String filename) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String forEach(String listName, String elementName, String body) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String getShebang() { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String ifListIsNotEmpty(String listName, String body) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String ifNotFileExists(String fileName, String body) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String join(String... elementList) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String listAppend(String listName, String element) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String listToBraceCommaSeparatedString(String listName, - String stringName, String open, String close) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String newList(String listName) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String quote(String content) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String raise(String msg) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String symlink(String src, String dest) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String varDef(String varname, String value) { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String varDef(String varname, CompoundExpr ce) - throws NotDerivableException { - // TODO Auto-generated method stub - return null; - } - - @Override - protected String getLibPath() { - // TODO Auto-generated method stub - return null; - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/MatlabInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/MatlabInvocation.java deleted file mode 100644 index 8075ecf..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/MatlabInvocation.java +++ /dev/null @@ -1,143 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.invoc; - -import java.nio.file.Path; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class MatlabInvocation extends OctaveInvocation { - - public MatlabInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String getShebang() { - return ""; - } - - @Override - public Path getExecutablePath( Path location ) { - return location.resolve( SCRIPT_NAME+".m" ); - } - - @Override - public String[] getCmd() { - - return new String[] { - "matlab", - "-nodisplay", - "-nosplash", - "-r", - SCRIPT_NAME }; - } - - @SuppressWarnings("static-method") - public String getScriptHead() { - return "function "+SCRIPT_NAME+"\ntry\n"; - } - - @SuppressWarnings("static-method") - public String getScriptFoot() { - return "catch ex\nex\nex.stack\nexit( -1 )\nend\nexit\nend\n"; - } - - @Override - public String toScript() throws NotBoundException, NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - // get script header - buf.append( comment( "script header" ) ); - buf.append( getScriptHead() ); - - // bind single output variables to default values - buf.append( comment( "bind single output variables to default values" ) ); - for( String outputName : getSingleOutputNameSet() ) - buf.append( - varDef( - outputName, - quote( outputName ) ) ); - buf.append( '\n' ); - - // bind input parameters - buf.append( comment( "bind input parameters" ) ); - for( String paramName : getSingleParamNameSet() ) { - - if( paramName.equals( CfSemanticModelVisitor.LABEL_TASK ) ) - continue; - - buf.append( varDef( paramName, quote( getResolveableBoundToSingleParam( paramName ) ) ) ); - } - for( String paramName : getReduceParamNameSet() ) - buf.append( varDef( paramName, getReduceParam( paramName ) ) ); - buf.append( '\n' ); - - // report stage in file sizes and report error when something is missing - buf.append( comment( "report stage in file sizes and report error when something is missing" ) ); - buf.append( getStageInCollect() ).append( '\n' ); - - // insert function body - buf.append( comment( "insert function body" ) ); - buf.append( getTicket().getBody() ).append( '\n' ); - - // rename output files - buf.append( comment( "rename output files" ) ); - buf.append( getOutputRename() ).append( '\n' ); - - // collect output variables - buf.append( comment( "collect output variables" ) ); - buf.append( getOutputCollect() ).append( '\n' ); - - // collect stage out information - buf.append( comment( "collect stage out information" ) ); - buf.append( getStageOutCollect() ).append( '\n' ); - - // script footer - buf.append( comment( "script footer" ) ); - buf.append( getScriptFoot() ); - - // define necessary functions - buf.append( comment( "define necessary functions" ) ); - buf.append( getFunDef() ).append( '\n' ); - - return buf.toString(); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/OctaveInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/OctaveInvocation.java deleted file mode 100644 index 36bffc6..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/OctaveInvocation.java +++ /dev/null @@ -1,346 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.invoc; - -import java.util.List; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class OctaveInvocation extends Invocation { - - public OctaveInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( name ).append( "( " ); - - comma = false; - for( String value : argValue ) { - - if( comma ) - buf.append( ", " ); - - comma = true; - - buf.append( value ); - } - - buf.append( " )" ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - return callFunction( name, argValue )+";\n"; - } - - @Override - protected String clip( String varName ) { - return varName+" = "+varName+"( 2:length( "+varName+" ) );\n"; - } - - @Override - protected String comment( String comment ) { - return "% "+comment+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return to+" = "+from+";\n"; - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "if( ~ischar( key ) ); error( 'Parameter ''key'' must be of type char.' ); end\n" ) - .append( "if( ~ischar( value ) ); error( 'Parameter ''value'' must be of type char.' ); end\n" ) - .append( "[fid msg] = fopen( '"+REPORT_FILENAME+"', 'a' );\n" ) - .append( "if( fid < 0 ); error( msg ); end\n" ) - .append( "fprintf( fid, '{" ) - .append( JsonReportEntry.ATT_TIMESTAMP+":"+System.currentTimeMillis()+"," ) - .append( JsonReportEntry.ATT_RUNID+":\""+getRunId()+"\"," ) - .append( JsonReportEntry.ATT_TASKID+":"+getTaskId()+"," ); - - if( this.hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME+":\""+getTaskName()+"\"," ); - - buf.append( JsonReportEntry.ATT_LANG+":\""+getLangLabel()+"\"," ) - .append( JsonReportEntry.ATT_INVOCID+":"+getTicketId()+"," ) - .append( JsonReportEntry.ATT_KEY+":\"%s\"," ) - .append( JsonReportEntry.ATT_VALUE+":%s}\\n', key, value )\n" ) - .append( "if( fclose( fid ) < 0 ); error( 'Could not close report file' ); end" ); - - return defFunction( FUN_LOG, null, new String[] { "key", "value" }, buf.toString() ); - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "if( ~ischar( key ) ); error( 'Parameter ''key'' must be of type char.' ); end\n" ) - .append( "if( ~ischar( value ) ); error( 'Parameter ''value'' must be of type char.' ); end\n" ) - .append( "[fid msg] = fopen( '"+REPORT_FILENAME+"', 'a' );\n" ) - .append( "if( fid < 0 ); error( msg ); end\n" ) - .append( "fprintf( fid, '{" ) - .append( JsonReportEntry.ATT_TIMESTAMP+":"+System.currentTimeMillis()+"," ) - .append( JsonReportEntry.ATT_RUNID+":\""+getRunId()+"\"," ) - .append( JsonReportEntry.ATT_TASKID+":"+getTaskId()+"," ); - - if( this.hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME+":\""+getTaskName()+"\"," ); - - buf.append( JsonReportEntry.ATT_LANG+":\""+getLangLabel()+"\"," ) - .append( JsonReportEntry.ATT_INVOCID+":"+getTicketId()+"," ) - .append( JsonReportEntry.ATT_FILE+":\"%s\"," ) - .append( JsonReportEntry.ATT_KEY+":\"%s\"," ) - .append( JsonReportEntry.ATT_VALUE+":%s}\\n', file, key, value )\n" ) - .append( "if( fclose( fid ) < 0 ); error( 'Could not close report file' ); end" ); - - return defFunction( FUN_LOGFILE, null, new String[] { "file", "key", "value" }, buf.toString() ); - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - - return defFunction( - FUN_NORMALIZE, - null, - new String[] { "channel", "f" }, - "sprintf( '"+getTicketId() - +"_%d_%s', channel, f )\n" ); - } - - @Override - protected String dereference( String varName ) { - return varName; - } - - @Override - protected String fileSize( String filename ) { - return "stat( "+filename+" ).size"; - } - - @Override - protected String forEach( String listName, String elementName, String body ) { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "for i = 1:length( " ).append( listName ).append( ")\n" ); - buf.append( varDef( elementName, listName+"{ i };\n" ) ); - buf.append( body ); - buf.append( "\nend\n" ); - - return buf.toString(); - } - - @Override - protected String getShebang() { - return "#!/usr/bin/octave -q\n"; - } - - @Override - protected String ifListIsNotEmpty( String listName, String body ) { - return "if ~isempty( "+listName+" )\n"+body+"\nend\n"; - } - - @Override - protected String ifNotFileExists( String fileName, String body ) { - return "if exist( "+fileName+", 'file' ) == 2\n"+body+"\nend\n"; - } - - @Override - protected String join( String... elementList ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "[" ); - - comma = false; - for( String element : elementList ) { - - if( comma ) - buf.append( " " ); - comma = true; - - buf.append( element ); - } - buf.append( "]" ); - - return buf.toString(); - } - - @Override - protected String listAppend( String listName, String element ) { - return listName+" = ["+listName+" {"+element+"}];"; - } - - @Override - protected String listToBraceCommaSeparatedString(String listName, - String stringName, String open, String close) { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( stringName ).append( " = '" ).append( open ); - buf.append( "';\n__COMMA = false;\n" ); - buf.append( - octaveForLoop( "i", "length( "+listName+" )", - "if( __COMMA ); ret = [ret ',']; end\n__COMMA = true;\n" - +"ret = [ret "+listName+"{ i }];\n" ) ); - buf.append( "ret = [ret '" ).append( close ).append( "'];" ); - - return buf.toString(); - } - - @Override - protected String newList( String listName ) { - return listName+" = {};\n"; - } - - @Override - protected String quote( String content ) { - return "'"+content+"'"; - } - - @Override - protected String raise( String msg ) { - return callProcedure( "error", msg ); - } - - @Override - protected String symlink(String src, String dest) { - return "system( sprintf( 'ln -s %s %s', "+src+", "+dest+" ) )"; - } - - @Override - protected String varDef( String varname, String value ) { - return varname+" = "+value+";\n"; - } - - @Override - protected String varDef( String varname, CompoundExpr ce ) - throws NotDerivableException { - - StringBuffer buf; - int i; - boolean comma; - List list; - - list = ce.normalize(); - - buf = new StringBuffer(); - - buf.append( varname ) - .append( " = {" ); - - comma = false; - for( i = 0; i < list.size(); i++ ) { - - if( comma ) - buf.append( ';' ); - - comma = true; - - buf.append( " '" ).append( list.get( i ) ).append( "'" ); - } - - buf.append( "};\n" ); - - return buf.toString(); - } - - private static String defFunction( String funName, String outputName, String[] inputNameList, String body ) { - - boolean comma; - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "function " ); - - if( outputName != null ) - buf.append( outputName ).append( " = " ); - - buf.append( funName ).append( "( " ); - - comma = false; - for( String inputName : inputNameList ) { - - if( comma ) - buf.append( ", " ); - - comma = true; - - buf.append( inputName ); - } - - buf.append( " )\n" ).append( body ).append( "\nend\n" ); - - return buf.toString(); - } - - private static String octaveForLoop( String runVar, String times, String body ) { - return "for "+runVar+" = 1:"+times+"\n"+body+"\nend\n"; - } - - @Override - protected String getLibPath() { - // TODO Auto-generated method stub - return null; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PegasusInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PegasusInvocation.java deleted file mode 100644 index 5c6c1b0..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PegasusInvocation.java +++ /dev/null @@ -1,22 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.invoc; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class PegasusInvocation extends BashInvocation { - - public PegasusInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - public String getFunDef() throws NotDerivableException { - return defFunctionLog() - +defFunctionLogFile(); - } - - @Override - public String getOutputRename() { - return ""; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PerlInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PerlInvocation.java deleted file mode 100644 index 87afc37..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PerlInvocation.java +++ /dev/null @@ -1,347 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.invoc; - -import java.util.List; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class PerlInvocation extends Invocation { - - public PerlInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "&" + name ).append( "( " ); - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( arg ); - } - - buf.append( " )" ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "&" + name ).append( "( " ); - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( arg ); - } - - buf.append( " );\n" ); - - return buf.toString(); - } - - @Override - protected String clip( String varName ) { - - return "$"+varName+" = substr(\"$"+varName+"\",1);"; - } - - @Override - protected String comment( String comment ) { - return "# "+comment+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return "@"+to+" = @"+from+";"; - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "sub " ).append( FUN_LOG ).append( " {\n" ) - .append( " my ( $key, $value ) = @_;\n" ) - - .append( " my $logfile = \"" ).append( REPORT_FILENAME ).append( "\";\n" ) - .append( " open my $fh, \">>\", $logfile or die \"Can't open $logfile for appending log: $!\\n\";\n" ) - .append( " flock($fh, 2) or die \"Can't obtain write lock on $logfile:$!\\n\";\n" ) - - - // .append( " print $fh \"$key : $value\\\n\";\n" ) - - .append( " print $fh \"{" ) - .append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"$key\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":$value}\";\n" ) - - - .append( " close $fh or die \"Can't close $logfile after appending: $!\\n\";\n" ) - .append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "sub " ).append( FUN_LOGFILE ).append( " {\n" ) - .append( " my ( $file, $key, $value ) = @_;\n" ) - - .append( " my $logfile = \"" ).append( REPORT_FILENAME ).append( "\";\n" ) - .append( " open my $fh, \">>\", $logfile or die \"Can't open $logfile for appending log: $!\\\n\";\n" ) - .append( " flock($fh, 2) or die \"Can't obtain write lock on $logfile:$!\\n\";\n" ) - - // .append( " print $fh \"$file : $key : $value\\\n\";\n" ) - - .append( " print $fh \"{" ) - .append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_FILE ).append( ":\\\"$file\\\"," ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"$key\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":$value}\";\n" ) - - .append( " close $fh or die \"Can't close $logfile after appending: $!\\n\";\n" ) - .append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - StringBuffer buf = new StringBuffer(); - - long id = getTicketId(); - buf.append( "sub " ).append( FUN_NORMALIZE ).append( " {\n" ) - .append( " my ( $channel, $f ) = @_;\n" ) - .append( " $f =~ m!([^/])+$!;\n" ) - .append(" my $fn = $1;\n") - .append(" die \"Can't determine filename from $f\\n\" unless $fn;\n") - .append(" return \"" ).append( id ).append( "_${channel}_$fn\";\n") - .append(" }\n" ); - - return buf.toString(); - } - - @Override - protected String dereference(String varName) { - return "$"+varName; - } - - @Override - protected String fileSize( String filename ) { - return "-s "+filename; - } - - @Override - protected String forEach(String listName, String elementName, String body) { - - StringBuffer buf; - - buf = new StringBuffer(); - - - buf.append( "for my $" ).append( elementName ).append( "(@" ) - .append( listName ).append( ") {\n" ) - .append( body ) - .append( "\n}\n" ); - - return buf.toString(); - } - - @Override - protected String getShebang() { - return "#!/usr/bin/env perl"; - } - - @Override - protected String ifListIsNotEmpty(String listName, String body) { - return "if (scalar "+listName+") {\n"+body+"\n}\n"; - - } - - @Override - protected String ifNotFileExists(String fileName, String body) { - return "unless (-e "+fileName+") {\n"+body+"\n}\n"; - } - - @Override - protected String join( String... elementList ) { - - StringBuffer buf; - - buf = new StringBuffer(); - buf.append("join('',("); - for (int i = 0; i < elementList.length; i++) { - buf.append( elementList[i] ); - if (i < elementList.length - 1) buf.append(","); - } - buf.append("))"); - return buf.toString(); - } - - @Override - protected String listAppend(String listName, String element) { - return "push @"+listName+", "+element+";\n"; - } - - @Override - protected String listToBraceCommaSeparatedString(String listName, - String stringName, String open, String close) { - return "$"+stringName + " = " + open + ".join(',', @"+listName+")."+ close +";\n"; - } - - @Override - protected String newList(String listName) { - return "@"+listName+" = ();\n"; - } - - @Override - protected String quote( String content ) { - - String ret; - - ret = "\""+content.replace( "\"", "\\\"" )+"\""; - - return ret; - } - - @Override - protected String raise( String msg ) { - return "die "+msg+";\n"; - } - - @Override - protected String symlink(String src, String dest) { - return "`ln -s "+src+" "+dest+"`;\n"; - } - - @Override - protected String varDef(String varname, String value) { - StringBuffer buf; - buf = new StringBuffer(); - - buf.append( "$"+ varname ).append( " = " ).append( value ).append(";\n"); - return buf.toString(); - } - - @Override - protected String varDef( String varname, CompoundExpr ce ) throws NotDerivableException { - StringBuffer buf; - int i; - boolean comma; - List list; - - list = ce.normalize(); - - buf = new StringBuffer(); - - comma = false; - buf.append( "@"+ varname ).append( " = ( " ); - for( i = 0; i < list.size(); i++ ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( "\"" ).append( list.get( i ) ).append( "\"" ); - } - - buf.append( " );\n" ); - - return buf.toString(); - } - - @Override - protected String getLibPath() { - // TODO Auto-generated method stub - return null; - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PythonInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PythonInvocation.java deleted file mode 100644 index 5538902..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/PythonInvocation.java +++ /dev/null @@ -1,265 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.invoc; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class PythonInvocation extends Invocation { - - private static String defFunction( String funname, String[] inputNameList, String body ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "def " ).append( funname ).append( "( " ); - - comma = false; - for( String inputName : inputNameList ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( inputName ); - } - - buf.append( " ):\n " ); - - buf.append( body.replace( "\n", "\n " ) ); - buf.append( "\n" ); - - return buf.toString(); - } - - public PythonInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( name ).append( '(' ); - - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ',' ); - - comma = true; - - buf.append( arg ); - } - - buf.append( ")" ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - return callFunction( name, argValue )+"\n"; - } - - @Override - protected String clip( String varName ) { - return varName+" = "+varName+"[1:]\n"; - } - - @Override - protected String comment( String comment ) { - return "# "+comment.replace( "\n", "\n# " )+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return to+" = copy.copy( "+from+" )\n"; - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - return defFunction( FUN_LOG, new String[] { "key", "value" }, - "f = file(\""+REPORT_FILENAME+"\",\"a\")\nf.write("+"\"{" - +JsonReportEntry.ATT_TIMESTAMP+":"+System.currentTimeMillis()+"," - +JsonReportEntry.ATT_RUNID+":\\\""+getRunId()+"\\\"," - +JsonReportEntry.ATT_TASKID+":"+getTaskId()+"," - +JsonReportEntry.ATT_TASKNAME+":\\\""+getTaskName()+"\\\"," - +JsonReportEntry.ATT_LANG+":\\\""+getLangLabel()+"\\\"," - +JsonReportEntry.ATT_INVOCID+":"+getTicketId()+"," - +JsonReportEntry.ATT_KEY+":\\\"%s\\\"," - +JsonReportEntry.ATT_VALUE+":%s}\\n\"%(key,value))\nf.close()\n" ); - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - return defFunction( FUN_LOGFILE, new String[] { "fx", "key", "value" }, - "f = file(\""+REPORT_FILENAME+"\",\"a\")\nf.write("+"\"{" - +JsonReportEntry.ATT_TIMESTAMP+":"+System.currentTimeMillis()+"," - +JsonReportEntry.ATT_RUNID+":\\\""+getRunId()+"\\\"," - +JsonReportEntry.ATT_TASKID+":"+getTaskId()+"," - +JsonReportEntry.ATT_TASKNAME+":\\\""+getTaskName()+"\\\"," - +JsonReportEntry.ATT_LANG+":\\\""+getLangLabel()+"\\\"," - +JsonReportEntry.ATT_INVOCID+":"+getTicketId()+"," - +JsonReportEntry.ATT_FILE+":\\\"%s\\\"," - +JsonReportEntry.ATT_KEY+":\\\"%s\\\"," - +JsonReportEntry.ATT_VALUE+":%s}\\n\"%(fx,key,value))\nf.close()\n" ); - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - return defFunction( - FUN_NORMALIZE, - new String[] { "channel", "f" }, - "return \""+getTicketId()+"_%d_%s\"%(channel,os.path.basename(f))" ); - } - - @Override - protected String dereference( String varName ) { - return varName; - } - - @Override - protected String fileSize( String filename ) { - return "os.stat("+filename+").st_size"; - } - - @Override - protected String forEach( String listName, String elementName, String body ) { - return - "for "+elementName+" in "+listName+":\n " - +body.replace( "\n", "\n " )+"\n"; - } - - @Override - protected String getBodyPrefix() { - return "if True:\n"; - } - - @Override - protected String getImport() { - return "import copy, os, sys\n"; - } - - @Override - protected String getShebang() { - return "#!/usr/bin/env python\n"; - } - - @Override - protected String ifListIsNotEmpty( String listName, String body ) { - return "if not("+listName+"):\n "+body.replace( "\n", "\n " )+"\n"; - } - - @Override - protected String ifNotFileExists( String fileName, String body ) { - return - "if not(os.path.exists("+fileName+")):\n " - +body.replace( "\n", "\n " )+"\n"; - } - - @Override - protected String join( String... elementList ) { - - boolean comma; - StringBuffer buf; - - buf = new StringBuffer(); - - comma = false; - for( String element : elementList ) { - - if( comma ) - buf.append( '+' ); - - comma = true; - - buf.append( element ); - } - - return buf.toString(); - } - - @Override - protected String listAppend( String listName, String element ) { - return listName+".append("+element+")\n"; - } - - @Override - protected String listToBraceCommaSeparatedString( String listName, - String stringName, String open, String close ) { - return - stringName+"='"+open+"'+','.join( i for i in "+listName - +" )+'"+close+"'\n"; - } - - @Override - protected String newList( String listName ) { - return listName+"=[]\n"; - } - - @Override - protected String quote( String content ) { - return "\""+content.replace( "\\", "\\\\" ).replace( "\"", "\\\"")+"\""; - } - - @Override - protected String raise( String msg ) { - return "raise Exception("+quote( msg )+")"; - } - - @Override - protected String symlink( String src, String dest ) { - return "os.symlink("+src+","+dest+")\n"; - } - - @Override - protected String varDef( String varname, CompoundExpr ce ) - throws NotDerivableException { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( varname ).append( "=[" ); - - comma = false; - for( String s : ce.normalize() ) { - - if( comma ) - buf.append( ',' ); - - comma = true; - - buf.append( quote( s ) ); - } - - buf.append( "]\n" ); - - return buf.toString(); - } - - @Override - protected String varDef( String varname, String value ) { - return varname+"="+value+"\n"; - } - - @Override - protected String getLibPath() { - return callProcedure( "sys.path.append", quote( libPath ) ); - } - - @Override - protected String postProcess( String body ) { - return body.replace( "\n", "\n " ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/RInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/RInvocation.java deleted file mode 100644 index 92fb0aa..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/RInvocation.java +++ /dev/null @@ -1,333 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.invoc; - -import java.util.List; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class RInvocation extends Invocation { - - protected RInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( defFunction( FUN_LOG, null, new String[] { "key", "value" }, - "if( mode( key ) != \"character\" )\n" - +"stop( \"Expected key to be of type 'character'.\" )\n" - +"if( mode( value ) != \"character\" )\n" - +"stop( \"Expected key to be of type 'value'.\" )\n" - +"write( paste( \"{" - +JsonReportEntry.ATT_TIMESTAMP+":"+System.currentTimeMillis()+"," - +JsonReportEntry.ATT_RUNID+":\\\""+getRunId()+"\\\"," - +JsonReportEntry.ATT_TASKID+":"+getTaskId()+"," - +JsonReportEntry.ATT_TASKNAME+":"+getTaskName()+"," - +JsonReportEntry.ATT_LANG+":"+getLangLabel()+"," - +JsonReportEntry.ATT_INVOCID+":"+getTicketId()+"," - +JsonReportEntry.ATT_KEY+":\\\"\", key, \"\\\"," - +JsonReportEntry.ATT_VALUE+":\", value, \"}\\n\", sep=\"\" ), " - +"file=\""+REPORT_FILENAME+"\", append=TRUE )" ) ); - - return buf.toString(); - } - - @Override - protected String getShebang() { - return "#!/usr/bin/env Rscript\n"; - } - - @Override - protected String varDef( String varname, CompoundExpr ce ) - throws NotDerivableException { - - StringBuffer buf; - int i; - boolean comma; - List list; - - list = ce.normalize(); - - buf = new StringBuffer(); - - comma = false; - buf.append( varname ).append( " <- c( " ); - for( i = 0; i < list.size(); i++ ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( "\"" ).append( list.get( i ) ).append( "\"" ); - } - - buf.append( ")\n" ); - - return buf.toString(); - } - - @Override - protected String varDef( String varname, String value ) { - return varname+" <- "+value+"\n"; - } - - /** - * @param outputName - */ - @SuppressWarnings( "static-method" ) - private String defFunction( String funName, String outputName, String[] inputNameList, String body ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( funName ).append( " <- function(" ); - - comma = false; - for( String arg : inputNameList ) { - - if( comma ) - buf.append( ',' ); - - comma = true; - - buf.append( ' ' ).append( arg ); - } - - buf.append( " ) {\n" ).append( body ).append( "\n}\n" ); - - return buf.toString(); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( name ).append( "( " ); - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( arg ); - } - - buf.append( " )" ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( name ).append( "( " ); - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( arg ); - } - - buf.append( " )\n" ); - - return buf.toString(); - } - - @Override - protected String newList( String listName ) { - return varDef( listName, "NULL" ); - } - - @Override - protected String listAppend( String listName, String element ) { - return listName+" <- append( "+listName+", "+element+" )\n"; - } - - @Override - protected String dereference( String varName ) { - return varName; - } - - @Override - protected String forEach( String listName, String elementName, String body ) { - return "for( "+elementName+" in "+listName+" ) {\n"+body+"\n}\n"; - } - - @Override - protected String ifNotFileExists( String fileName, String body ) { - return - "if( !" - +callFunction( "file.exists", dereference( fileName ) ) - +" ) {\n" - +body - +"\n}\n"; - } - - @Override - protected String raise( String msg ) { - return callProcedure( "stop", msg ); - } - - @Override - protected String join( String... elementList ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "paste( " ); - - comma = false; - for( String element : elementList ) { - - if( comma ) - buf.append( ", " ); - comma = true; - - buf.append( element ); - } - - buf.append( ", sep=\"\" )" ); - - return buf.toString(); - } - - @Override - protected String quote( String content ) { - return "\""+content.replace( "\"", "\\\"" )+"\""; - } - - @Override - protected String fileSize( String filename ) { - return "sprintf( \"%.0f\", file.info( "+filename+" )$size/1024 )"; - } - - @Override - protected String ifListIsNotEmpty( String listName, String body ) { - return "if( length( "+listName+" ) != 0 ) {\n"+body+"\n}\n"; - } - - @Override - protected String listToBraceCommaSeparatedString( String listName, - String stringName, String open, String close ) { - return stringName+" = paste( "+listName+", sep=\"\", collapse=\",\" )\n"; - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - return defFunction( - FUN_NORMALIZE, - null, - new String[] { "channel", "f" }, - "sprintf( \""+getTicketId() - +"_%d_%s\", channel, basename( f ) )\n" ); - } - - @Override - protected String symlink( String src, String dest ) { - return callProcedure( "file.symlink", src, dest )+"\n"; - } - - @Override - protected String comment( String comment ) { - return "# "+comment.replace( "\n", "\n# " )+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return to+"="+from+"\n"; - } - - @Override - protected String clip( String varName ) { - return varName+" <- substring( "+varName+", 2 )\n"; - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( defFunction( FUN_LOGFILE, null, new String[] { "file", "key", "value" }, - "if( mode( key ) != \"character\" )\n" - +"stop( \"Expected key to be of type 'character'.\" )\n" - +"if( mode( value ) != \"character\" )\n" - +"stop( \"Expected value to be of type 'character'.\" )\n" - +"write( paste( \"{" - +JsonReportEntry.ATT_TIMESTAMP+":"+System.currentTimeMillis()+"," - +JsonReportEntry.ATT_RUNID+":\\\""+getRunId()+"\\\"," - +JsonReportEntry.ATT_TASKID+":"+getTaskId()+"," - +JsonReportEntry.ATT_TASKNAME+":"+getTaskName()+"," - +JsonReportEntry.ATT_LANG+":"+getLangLabel()+"," - +JsonReportEntry.ATT_INVOCID+":"+getTicketId()+"," - +JsonReportEntry.ATT_FILE+":\\\"\", file, \"\\\"," - +JsonReportEntry.ATT_KEY+":\\\"\", key, \"\\\"," - +JsonReportEntry.ATT_VALUE+":\", value, \"}\\n\", sep=\"\" ), " - +"file=\""+REPORT_FILENAME+"\", append=TRUE )" ) ); - - return buf.toString(); - } - - @Override - protected String getLibPath() { - return callProcedure( ".libPaths", quote( libPath ) ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/ScalaInvocation.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/ScalaInvocation.java deleted file mode 100644 index a6198a6..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/invoc/ScalaInvocation.java +++ /dev/null @@ -1,281 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.invoc; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class ScalaInvocation extends Invocation { - - public ScalaInvocation( Ticket ticket, String libPath ) { - super( ticket, libPath ); - } - - @Override - protected String callFunction( String name, String... argValue ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( name ).append( '(' ); - - comma = false; - for( String arg : argValue ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( arg ); - } - - buf.append( ")" ); - - return buf.toString(); - } - - @Override - protected String callProcedure( String name, String... argValue ) { - return callFunction( name, argValue )+"\n"; - } - - @Override - protected String clip( String varName ) { - return varName+"="+varName+" substring(1)\n"; - } - - @Override - protected String comment( String comment ) { - return "// "+comment+"\n"; - } - - @Override - protected String copyArray( String from, String to ) { - return "val "+to+"="+from+"\n"; - } - - @Override - protected String defFunctionLog() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "def "+FUN_LOG+"( key:String, value:String ) : Unit = {\n" ); - - buf.append( " val writer = new FileWriter(\""+REPORT_FILENAME+"\",true);\n" ); - buf.append( " writer write(\"{" ); - - buf.append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"\"+key+\"\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":\"+value+\"}\\n\"" ); - - buf.append( ")\n writer close\n" ); - - buf.append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionLogFile() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "def "+FUN_LOGFILE+"( file:String, key:String, value:String ) : Unit = {\n" ); - - buf.append( " val writer = new FileWriter(\""+REPORT_FILENAME+"\",true);\n" ); - buf.append( " writer write(\"{" ); - - buf.append( JsonReportEntry.ATT_TIMESTAMP ) - .append( ':' ).append( System.currentTimeMillis() ).append( ',' ) - .append( JsonReportEntry.ATT_RUNID ).append( ":\\\"" ) - .append( getRunId() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_TASKID ).append( ':' ) - .append( getTaskId() ).append( ',' ); - - if( hasTaskName() ) - buf.append( JsonReportEntry.ATT_TASKNAME ).append( ":\\\"" ) - .append( getTaskName() ).append( "\\\"," ); - - buf.append( JsonReportEntry.ATT_LANG ).append( ":\\\"" ) - .append( getLangLabel() ).append( "\\\"," ) - .append( JsonReportEntry.ATT_INVOCID ).append( ':' ) - .append( getTicketId() ).append( ',' ) - .append( JsonReportEntry.ATT_FILE ).append( ":\\\"\"+file+\"\\\"," ) - .append( JsonReportEntry.ATT_KEY ).append( ":\\\"\"+key+\"\\\"," ) - .append( JsonReportEntry.ATT_VALUE ).append( ":\"+value+\"}\\n\"" ); - - buf.append( ")\n writer close\n" ); - - buf.append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String defFunctionNormalize() throws NotDerivableException { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "def "+FUN_NORMALIZE+"(channel:Int,filename:String) : String = {" ); - buf.append( "val p = Paths.get(filename)\n"); - buf.append( "return \""+getTicketId()+"_\"+channel+\"_\"+p.getName(p.getNameCount-1)\n" ); - buf.append( "}\n" ); - - return buf.toString(); - } - - @Override - protected String dereference( String varName ) { - return varName; - } - - @Override - protected String fileSize( String filename ) { - return "Files.size(Paths.get("+filename+")).toString"; - } - - @Override - protected String forEach( String listName, String elementName, String body ) { - return "for(String "+elementName+":"+listName+") {\n"+body+"\n}\n"; - } - - @Override - protected String getShebang() { - return "#!/bin/sh\nexec scala \"$0\" \"$@\"\n!#"; - } - - @Override - protected String getLibPath() { - throw new UnsupportedOperationException( "NYI" ); - } - - @Override - protected String getImport() { - return "import java.nio.file.Paths\nimport java.io.FileWriter\nimport java.nio.file.Files"; - } - - @Override - protected String ifListIsNotEmpty(String listName, String body) { - return( "if(!"+listName+".isEmpty) {\n"+body+"\n}\n" ); - } - - @Override - protected String ifNotFileExists(String fileName, String body) { - return( "if(!Files.exists(Paths.get("+fileName+"))) {\n"+body+"\n}\n" ); - } - - @Override - protected String declareString( String outputName ) { - return "var "+outputName+"=\"\"\n"; - } - - @Override - protected String declareList( String paramName ) { - return "var "+paramName+"=List()\n"; - } - - @Override - protected String join( String... elementList ) { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - comma = false; - for( String element : elementList ) { - - if( comma ) - buf.append( "+" ); - comma = true; - - buf.append( element ); - } - - return buf.toString(); - } - - @Override - protected String listAppend(String listName, String element) { - return listName+"="+listName+":+"+element+"\n"; - } - - @Override - protected String listToBraceCommaSeparatedString(String listName, String stringName, String open, String close) { - return stringName+"=\""+open+"\"+"+listName+".mkString(\",\")+\""+close+"\"\n"; - } - - @Override - protected String newList( String listName ) { - return "var "+listName+"=List()\n"; - } - - @Override - protected String quote( String content ) { - return "\""+content.replace( "\"", "\\\"" )+"\""; - } - - @Override - protected String raise( String msg ) { - return "throw new RuntimeException("+msg+")\n"; - } - - @Override - protected String symlink( String src, String dest ) { - return "Files.createSymbolicLink(Paths.get("+dest+"),Paths.get("+src+"))\n"; - } - - @Override - protected String varDef( String varname, String value ) { - return varname+"="+value+"\n"; - } - - @Override - protected String varDef( String varname, CompoundExpr ce ) throws NotDerivableException { - - StringBuffer buf; - boolean comma; - int i, n; - - buf = new StringBuffer(); - - buf.append( "val "+varname+"=List(" ); - - comma = false; - n = ce.getNumAtom(); - for( i = 0; i < n; i++ ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( ce.getStringExprValue( i ) ); - } - buf.append( ")\n" ); - - return buf.toString(); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/ChannelListener.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/ChannelListener.java deleted file mode 100644 index fa77652..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/ChannelListener.java +++ /dev/null @@ -1,266 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.preprocess; - -import java.util.BitSet; -import java.util.LinkedList; -import java.util.List; - -import org.antlr.v4.runtime.ANTLRErrorListener; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStream; -import org.antlr.v4.runtime.TokenStreamRewriter; -import org.antlr.v4.runtime.atn.ATNConfigSet; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.NotNull; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import de.huberlin.wbi.cuneiform.core.parser.CuneiformBaseListener; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformLexer; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.ApplyExprContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.CallExprContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.CompoundExprContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.ExprContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NameContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NilExprContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.SingleExprContext; - -public class ChannelListener extends CuneiformBaseListener implements ANTLRErrorListener { - - private final TokenStreamRewriter rewriter; - private final Log log; - - public ChannelListener( CommonTokenStream tokenStream ) { - - if( tokenStream == null ) - throw new NullPointerException( "Token stream must not be empty." ); - - rewriter = new TokenStreamRewriter( tokenStream ); - log = LogFactory.getLog( ChannelListener.class ); - } - - @Override - public void enterAssign( @NotNull CuneiformParser.AssignContext ctx ) { - - List unqual; - int i, n; - StringBuffer buf; - TokenStream stream; - NameContext name; - String nameString; - - if( rewriter == null ) - throw new NullPointerException( "Token stream has never been set." ); - - stream = rewriter.getTokenStream(); - unqual = getUnqualifiedApplicationList( ctx.expr() ); - - - n = ctx.name().size(); - - for( i = n-1; i > 0; i-- ) { - - buf = new StringBuffer(); - name = ctx.name( i ); - - buf.append( '\n' ); - buf.append( stream.getText( name.getStart(), name.getStop() ) ); - buf.append( " =" ); - - if( unqual.isEmpty() ) - buf.append( " nil" ); - else - for( SingleExprContext se : unqual ) - - buf.append( " [" ).append( i+1 ).append( "]" ) - .append( stream.getText( se.getStart(), se.getStop() ) ); - - - - buf.append( ';' ); - - rewriter.insertAfter( ctx.getStop(), buf.toString() ); - } - - name = ctx.name( 0 ); - nameString = stream.getText( name.getStart(), name.getStop() ); - - rewriter.replace( name.getStart(), ctx.EQUAL().getSymbol(), nameString+" =" ); - } - - @Override - public void enterCondExpr( @NotNull CuneiformParser.CondExprContext ctx ) { - - if( rewriter == null ) - throw new NullPointerException( "Token stream has never been set." ); - } - - @Override - public void enterScript( @NotNull CuneiformParser.ScriptContext ctx ) { - - if( rewriter == null ) - throw new NullPointerException( "Token stream not set." ); - } - - public String getRewrittenText() { - - if( rewriter == null ) - throw new NullPointerException( "Token stream not set." ); - - return rewriter.getText(); - } - - public static String process( String input ) { - - ANTLRInputStream instream; - CuneiformLexer lexer; - CommonTokenStream tokenStream; - CuneiformParser parser; - ParseTree tree; - ParseTreeWalker walker; - ChannelListener channelListener; - - walker = new ParseTreeWalker(); - - - // parse original content - instream = new ANTLRInputStream( input ); - - lexer = new CuneiformLexer( instream ); - lexer.removeErrorListeners(); - - tokenStream = new CommonTokenStream( lexer ); - - parser = new CuneiformParser( tokenStream ); - parser.removeErrorListeners(); - - channelListener = new ChannelListener( tokenStream ); - lexer.addErrorListener( channelListener ); - parser.addErrorListener( channelListener ); - - tree = parser.script(); - - walker.walk( channelListener, tree ); - - return channelListener.getRewrittenText(); - } - - public static List getUnqualifiedApplicationList( ExprContext expr ) { - - List unqualifiedList; - CompoundExprContext cec; - - if( expr == null ) - throw new NullPointerException( "Expression must not be null." ); - - unqualifiedList = new LinkedList<>(); - - if( expr instanceof NilExprContext ) - return unqualifiedList; - - if( expr instanceof CompoundExprContext ) { - - cec = ( CompoundExprContext )expr; - - for( SingleExprContext singleExpr : cec.singleExpr() ) - if( isUnqualifiedApplication( singleExpr ) ) - unqualifiedList.add( singleExpr ); - - return unqualifiedList; - } - - throw new RuntimeException( "Expression type not recognized." ); - } - - public static boolean isUnqualifiedApplication( SingleExprContext singleExpr ) { - - if( singleExpr instanceof ApplyExprContext ) - - if( ( ( ApplyExprContext )singleExpr ).channel() == null ) - return true; - - - if( singleExpr instanceof CallExprContext ) - - if( ( ( CallExprContext )singleExpr ).channel() == null ) - return true; - - return false; - } - - @Override - public void syntaxError( Recognizer recognizer, - Object offendingSymbol, int line, int charPositionInLine, - String msg, RecognitionException e ) { - throw new ParseException( line, charPositionInLine, ( ( Token )offendingSymbol ).getText(), msg ); - } - - @Override - public void reportAmbiguity( Parser arg0, DFA arg1, int arg2, int arg3, - boolean arg4, BitSet arg5, ATNConfigSet arg6 ) { - - if( log.isDebugEnabled() ) - log.debug( "Ambiguity detected." ); - - } - - @Override - public void reportAttemptingFullContext( Parser arg0, DFA arg1, int arg2, - int arg3, BitSet arg4, ATNConfigSet arg5 ) { - - if( log.isTraceEnabled() ) - log.trace( "Attempting full context." ); - - } - - @Override - public void reportContextSensitivity( Parser arg0, DFA arg1, int arg2, - int arg3, int arg4, ATNConfigSet arg5) { - - if( log.isTraceEnabled() ) - log.trace( "Context sensitivity detected." ); - } - - - - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/ParseException.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/ParseException.java deleted file mode 100644 index 94c6e4b..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/ParseException.java +++ /dev/null @@ -1,130 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.preprocess; - -public class ParseException extends RuntimeException { - - private static final long serialVersionUID = -3262992285049846787L; - - private Integer line; - private Integer charPositionInLine; - private String near; - - public ParseException( Integer line, Integer charPositionInLine, String near, String msg ) { - super( msg ); - setLine( line ); - setCharPositionInLine( charPositionInLine ); - setNear( near ); - } - - public int getLine() { - return line; - } - - public int getCharPositionInLine() { - return charPositionInLine; - } - - public String getNear() { - return near; - } - - public boolean hasLine() { - return line != null; - } - - public boolean hasCharPositionInLine() { - return charPositionInLine != null; - } - - public boolean hasNear() { - return near != null; - } - - public void setLine( Integer line ) { - - if( line == null ) { - this.line = null; - return; - } - - if( line < 1 ) - throw new RuntimeException( "Line must be a positive number. Received "+line+" instead." ); - this.line = line; - } - - public void setNear( String near ) { - - if( near == null ) { - this.near = null; - return; - } - - if( near.isEmpty() ) - throw new RuntimeException( "Near string must not be empty." ); - - this.near = near; - } - - public void setCharPositionInLine( Integer charPositionInLine ) { - - if( charPositionInLine == null ) { - this.charPositionInLine = null; - return; - } - - if( charPositionInLine < 0 ) - throw new RuntimeException( "Character position in line must be a positive number. Received "+charPositionInLine+" instead." ); - - this.charPositionInLine = charPositionInLine; - } - - @Override - public String toString() { - - StringBuffer buf; - - buf = new StringBuffer(); - - if( hasLine() ) - buf.append( "line " ).append( line ).append( ":" ) - .append( charPositionInLine ).append( ' ' ); - - if( hasNear() ) - buf.append( "near " ).append( near ).append( ": " ); - - buf.append( getMessage() ); - - return buf.toString(); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/PreListener.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/PreListener.java deleted file mode 100644 index b979503..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/PreListener.java +++ /dev/null @@ -1,329 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.preprocess; - -import java.util.BitSet; -import java.util.LinkedList; -import java.util.List; - -import org.antlr.v4.runtime.ANTLRErrorListener; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.TokenStreamRewriter; -import org.antlr.v4.runtime.atn.ATNConfigSet; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.NotNull; -import org.antlr.v4.runtime.tree.ParseTree; -import org.antlr.v4.runtime.tree.ParseTreeWalker; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import de.huberlin.wbi.cuneiform.core.parser.CuneiformBaseListener; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformLexer; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.CondExprContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.CorrelParamContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.PrototypeContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NameContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NameDataTypeContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NameDeepFnTypeContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NameInferredTypeContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NamePlainFnTypeContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.ParamContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.ReduceVarContext; - -/** Simplifies token stream by removing syntactic sugar. - * - * - Replaces deftask statements with assignments to lambda expressions - * - Replaces task calls with apply expressions - * - Resolves pipe notation to regular nested function calls - * - Removes comments - * - Explicitly references channels in applications - * - Adds explicit task parameter to all task prototypes - * - Quotes numbers - * - * @author jorgen - * - */ -public class PreListener extends CuneiformBaseListener implements ANTLRErrorListener { - - private final TokenStreamRewriter rewriter; - private final LinkedList exprStack; - private final Log log; - - public PreListener( CommonTokenStream tokenStream ) { - - if( tokenStream == null ) - throw new NullPointerException( "Token stream must not be empty." ); - - rewriter = new TokenStreamRewriter( tokenStream ); - exprStack = new LinkedList<>(); - log = LogFactory.getLog( PreListener.class ); - } - - public static String process( String input ) { - - ANTLRInputStream instream; - CuneiformLexer lexer; - CommonTokenStream tokenStream; - CuneiformParser parser; - ParseTree tree; - ParseTreeWalker walker; - PreListener preListener; - - - walker = new ParseTreeWalker(); - - // parse original content - instream = new ANTLRInputStream( input ); - - lexer = new CuneiformLexer( instream ); - lexer.removeErrorListeners(); - - tokenStream = new CommonTokenStream( lexer ); - - parser = new CuneiformParser( tokenStream ); - parser.removeErrorListeners(); - - preListener = new PreListener( tokenStream ); - lexer.addErrorListener( preListener ); - parser.addErrorListener( preListener ); - - - tree = parser.script(); - - walker.walk( preListener, tree ); - - return preListener.getRewrittenText(); - } - - @Override - public void enterForeignDefTask( @NotNull CuneiformParser.ForeignDefTaskContext ctx ) { - - Token deftask; - Token id; - Token bodyStop; - - deftask = ctx.DEFTASK().getSymbol(); - bodyStop = ctx.foreignBody().getStop(); - id = ctx.ID().getSymbol(); - - rewriter.insertAfter( bodyStop, ";" ); - rewriter.insertAfter( id, " = \\" ); - rewriter.replace( deftask, id, id.getText() ); - - } - - @Override - public void enterNativeDefTask( @NotNull CuneiformParser.NativeDefTaskContext ctx ) { - - Token deftask; - Token id; - Token bodyStop; - - deftask = ctx.DEFTASK().getSymbol(); - bodyStop = ctx.RBRACE().getSymbol(); - id = ctx.ID().getSymbol(); - - rewriter.insertAfter( bodyStop, ";" ); - rewriter.insertAfter( id, " = \\" ); - rewriter.replace( deftask, id, id.getText() ); - - } - - @Override - public void exitDanglingExpr( @NotNull CuneiformParser.DanglingExprContext ctx ) { - exprStack.push( rewriter.getText( ctx.expr().getSourceInterval() ) ); - rewriter.delete( ctx.getStart(), ctx.getStop() ); - } - - @Override - public void enterPrototype( @NotNull CuneiformParser.PrototypeContext ctx ) { - - - if( !( ctx.getParent() instanceof CondExprContext ) ) - if( !containsParam( ctx, "task" ) ) - rewriter.insertAfter( ctx.COLON().getSymbol(), " task " ); - } - - @Override - public void enterScript( @NotNull CuneiformParser.ScriptContext ctx ) { - - if( rewriter == null ) - throw new NullPointerException( "Token stream not set." ); - } - - @Override - public void exitFromStackExpr( @NotNull CuneiformParser.FromStackExprContext ctx ) { - - Token symbol; - - - symbol = ctx.FROMSTACK().getSymbol(); - - if( exprStack.isEmpty() ) - throw new ParseException( symbol.getLine(), symbol.getCharPositionInLine(), symbol.getText(), - "Pipe destination encountered where there is no source." ); - - - rewriter.replace( symbol, exprStack.pop() ); - } - - public String getRewrittenText() { - return rewriter.getText(); - } - - public static boolean containsParam( PrototypeContext prototype, String paramName ) { - - List nameList; - - nameList = getNameList( prototype ); - return nameList.contains( paramName ); - } - - public static List getNameList( PrototypeContext prototype ) { - - List nameList; - - nameList = new LinkedList<>(); - for( ParamContext param : prototype.param() ) - nameList.addAll( getNameList( param ) ); - - return nameList; - } - - public static List getNameList( ParamContext param ) { - - List list; - - list = new LinkedList<>(); - - if( param.name() != null ) { - list.add( getName( param.name() ) ); - return list; - } - - if( param.reduceVar() != null ) { - list.add( getName( param.reduceVar() ) ); - return list; - } - - if( param.correlParam() != null ) { - list.addAll( getNameList( param.correlParam() ) ); - return list; - } - - throw new RuntimeException( "Parameter type not recognized." ); - - } - - public static String getName( NameContext name ) { - - if( name == null ) - throw new NullPointerException( "Name must not be null." ); - - if( name instanceof NameInferredTypeContext ) - return ( ( NameInferredTypeContext )name ).ID().getText(); - - if( name instanceof NameDataTypeContext ) - return ( ( NameDataTypeContext )name ).ID( 0 ).getText(); - - if( name instanceof NamePlainFnTypeContext ) - return ( ( NamePlainFnTypeContext )name ).ID().getText(); - - if( name instanceof NameDeepFnTypeContext ) - return ( ( NameDeepFnTypeContext )name ).ID().getText(); - - throw new RuntimeException( "Name type "+name.getClass().getName()+" not recognized. In '"+name.getText()+"'" ); - } - - public static String getName( ReduceVarContext reduceVar ) { - return getName( reduceVar.name() ); - } - - public static List getNameList( CorrelParamContext correlVar ) { - - List list; - - list = new LinkedList<>(); - - for( NameContext nc : correlVar.name() ) - list.add( getName( nc ) ); - - return list; - } - - @Override - public void syntaxError( Recognizer recognizer, - Object offendingSymbol, int line, int charPositionInLine, - String msg, RecognitionException e ) { - - String near; - - near = null; - if( offendingSymbol != null ) - near = ( ( Token )offendingSymbol ).getText(); - - throw new ParseException( line, charPositionInLine, near, msg ); - } - - @Override - public void reportAmbiguity( Parser arg0, DFA arg1, int arg2, int arg3, - boolean arg4, BitSet arg5, ATNConfigSet arg6 ) { - - if( log.isDebugEnabled() ) - log.debug( "Ambiguity detected." ); - - } - - @Override - public void reportAttemptingFullContext( Parser arg0, DFA arg1, int arg2, - int arg3, BitSet arg4, ATNConfigSet arg5 ) { - - if( log.isTraceEnabled() ) - log.trace( "Attempting full context." ); - - } - - @Override - public void reportContextSensitivity( Parser arg0, DFA arg1, int arg2, - int arg3, int arg4, ATNConfigSet arg5) { - - if( log.isTraceEnabled() ) - log.trace( "Context sensitivity detected." ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/PrePhaseException.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/PrePhaseException.java deleted file mode 100644 index f537e9a..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/preprocess/PrePhaseException.java +++ /dev/null @@ -1,43 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.preprocess; - -public class PrePhaseException extends ParseException { - - private static final long serialVersionUID = 5468102633060593479L; - - public PrePhaseException( Integer row, Integer col, String near, String msg ) { - super( row, col, near, msg ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/BaseRepl.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/BaseRepl.java deleted file mode 100644 index 550f57d..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/BaseRepl.java +++ /dev/null @@ -1,441 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.repl; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.nio.charset.Charset; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import de.huberlin.wbi.cuneiform.core.parser.CuneiformLexer; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser; -import de.huberlin.wbi.cuneiform.core.preprocess.ChannelListener; -import de.huberlin.wbi.cuneiform.core.preprocess.ParseException; -import de.huberlin.wbi.cuneiform.core.preprocess.PreListener; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SingleExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; -import de.huberlin.wbi.cuneiform.core.ticketsrc.ReplTicketSrc; - -public abstract class BaseRepl { - - public static final int CTL_NIL = 0; - public static final int CTL_QUIT = 1; - public static final int CTL_STATE = 2; - public static final int CTL_QUERYSET = 4; - public static final int CTL_TICKETSET = 8; - - public static final String LABEL_VERSION = "2.0.4-RELEASE"; - public static final String LABEL_BUILD = "2016-04-11"; - - private final CfSemanticModelVisitor state; - private final Map runningMap; - private final ReplTicketSrc ticketSrc; - private final Log log; - private CompoundExpr ans; - private final Log statLog; - - - - public BaseRepl( ReplTicketSrc ticketSrc, Log statLog ) { - - if( ticketSrc == null ) - throw new NullPointerException( "Ticket source actor must not be null." ); - - this.ticketSrc = ticketSrc; - - runningMap = new HashMap<>(); - state = new CfSemanticModelVisitor(); - log = LogFactory.getLog( BaseRepl.class ); - this.statLog = statLog; - } - - public CompoundExpr getAns() { - return ans; - } - - public Set getRunningSet() { - return runningMap.keySet(); - } - - public Set getTicketSet() { - - Set set; - - set = new HashSet<>(); - - for( UUID queryId : runningMap.keySet() ) - set.addAll( ticketSrc.getTicketSet( queryId ) ); - - return set; - } - - public TopLevelContext getState() { - return state.getTopLevelContext(); - } - - public boolean hasAns() { - return ans != null; - } - - public int interpret( String input ) { - - String afterPre, afterChannel; - CuneiformLexer lexer; - CuneiformParser parser; - TopLevelContext tlc; - - if( input == null ) - throw new NullPointerException( "Input string must not be null." ); - - // pre-step - afterPre = PreListener.process( input ); - - // channel-step - afterChannel = ChannelListener.process( afterPre ); - - // create lexer - lexer = new CuneiformLexer( new ANTLRInputStream( afterChannel ) ); - lexer.removeErrorListeners(); - lexer.addErrorListener( state ); - - // create parser - parser = new CuneiformParser( new CommonTokenStream( lexer ) ); - parser.removeErrorListeners(); - parser.addErrorListener( state ); - - // visit parsed script - tlc = ( TopLevelContext )state.visit( parser.script() ); - - return interpret( tlc ); - } - - public int interpret( TopLevelContext tlc ) { - - int ctl; - DynamicNodeVisitor reducer; - UUID queryId; - - // remove control targets - ctl = fetchCtl( tlc ); - - if( !tlc.isTargetListEmpty() ) { - - reducer = new DynamicNodeVisitor( ticketSrc, this, tlc ); - queryId = reducer.getQueryId(); - runningMap.put( queryId, reducer ); - - try { - - queryStarted( queryId ); - - // trigger reduction - reducer.step(); - - } - catch( ParseException e ) { - queryFailed( queryId, null, e, null, null, null ); - } - } - - return ctl; - } - - public boolean isBusy() { - return !runningMap.isEmpty(); - } - - public boolean isRunning( UUID queryId ) { - - if( queryId == null ) - throw new NullPointerException( "Query id must not be null." ); - - return runningMap.containsKey( queryId ); - } - - public void queryFailed( UUID queryId, Long ticketId, Exception e, String script, String stdOut, String stdErr ) { - - String s; - - if( queryId == null ) - throw new NullPointerException( "Query id must not be null." ); - - runningMap.remove( queryId ); - - s = ""; - - if( e != null ) { - - s += " "+e.getClass().getName(); - - if( e.getMessage() != null ) - s += ": \""+e.getMessage()+"\""; - } - - /* if( stdOut != null ) - s += " Output channel: \""+stdOut.replace( '\n', ' ' )+"\""; - - if( stdErr != null ) - s += " Error channel: \""+stdErr.replace( '\n', ' ' )+"\""; */ - - if( log.isErrorEnabled() ) - if( ticketId == null ) - log.error( "Query "+queryId+" failed."+s ); - else - log.error( "Query "+queryId+" failed while executing ticket "+ticketId+"."+s ); - - queryFailedPost( queryId, ticketId, e, script, stdOut, stdErr ); - } - - public abstract void queryFailedPost( UUID queryId, Long ticketId, Exception e, String script, String stdOut, String stdErr ); - - public void queryFinished( UUID queryId, CompoundExpr result ) { - - if( queryId == null ) - throw new NullPointerException( "Query id must not be null." ); - - if( result == null ) - throw new NullPointerException( "Result compound expression must not be null." ); - - runningMap.remove( queryId ); - ans = result; - - if( log.isInfoEnabled() ) - log.info( "Query "+queryId+" finished: "+result ); - - queryFinishedPost( queryId, result ); - } - - public abstract void queryFinishedPost( UUID queryId, CompoundExpr result ); - - public void queryStarted( UUID runId ) { - - if( log.isInfoEnabled() ) - log.info( "Query "+runId+" started." ); - - queryStartedPost( runId ); - } - - public abstract void queryStartedPost( UUID runId ); - - public synchronized void ticketFinished( UUID queryId, long ticketId, Set reportEntrySet ) { - - DynamicNodeVisitor dnv; - - if( queryId == null ) - throw new NullPointerException( "Query ID must not be null." ); - - if( reportEntrySet == null ) - throw new NullPointerException( "Report entry set must not be null." ); - - dnv = runningMap.get( queryId ); - if( dnv == null ) - throw new NullPointerException( "Dynamic node visitor must not be null." ); - - if( log.isDebugEnabled() ) - log.debug( "Stepping expression in query "+queryId+"." ); - - dnv.step(); - - if( log.isDebugEnabled() ) - log.debug( "Ticket finished: "+ticketId+" part of query "+queryId ); - - flushStatLog( reportEntrySet ); - - } - - protected void flushStatLog( Set reportEntrySet ) { - - if( statLog == null ) - return; - - if( statLog.isDebugEnabled() ) - for( JsonReportEntry entry : reportEntrySet ) - statLog.debug( entry ); - } - - public static void run( BaseRepl repl ) throws IOException { - - StringBuffer buf; - String line; - QueryParseCtlLexer lexer; - int ctl; - - System.out.println( BaseRepl.getLogo() ); - - buf = new StringBuffer(); - System.out.print( "> " ); - - try( BufferedReader reader = new BufferedReader( new InputStreamReader( System.in, Charset.forName( "UTF-8" ) ) ) ) { - - while( ( line = reader.readLine() ) != null ) { - - buf.append( line ).append( '\n' ); - - lexer = new QueryParseCtlLexer( new ANTLRInputStream( buf.toString() ) ); - - - if( lexer.isReady() ) { - - try { - - ctl = repl.interpret( buf.toString() ); - - if( ( ctl & BaseRepl.CTL_STATE ) != 0 ) - System.out.print( repl.getState() ); - - if( ( ctl & BaseRepl.CTL_QUERYSET ) != 0 ) - System.out.print( repl.getRunningSet() ); - - if( ( ctl & BaseRepl.CTL_TICKETSET ) != 0 ) - System.out.print( repl.getTicketSet() ); - - if( ( ctl & BaseRepl.CTL_QUIT ) != 0 ) - break; - } - catch( ParseException e ) { - System.out.println( e ); - } - - buf = new StringBuffer(); - System.out.print( "> " ); - } - - } - - } - System.out.println( "Bye." ); - } - - public static String getLogo() { - - return " _\n" - +" _gg@@@L\n" - +" _g@@@@@@k\n" - +" g@@@@BBBA\n" - +" gB@@@P\n" - +" _@@@@B\n" - +" __@@@@@gg@@BBBB@@gggp\n" - +" __gg@@@@@@@@@@@@@@@@@@@@BF\n" - +" _gg@@@@@@@@@@@BBBBBBBBBBBB@BL\n" - +" _g@@@@@B@@BMB@@@F\n" - +" _@B@@@@@#P^ @@@@F\n" - +" _gB@@@B#F q@@@@L\n" - +" _@@@@@#\" q@@@BL\n" - +" g@@@BF q@@@BL\n" - +" _@@@@F q@@@BL\n" - +" 4@@@@L q@@@BL\n" - +" ^@@@@gg_____ggp q@@@BL\n" - +" MB@@@@@@@@@@@g q@@@BL\n" - +" ^WB@@@@@@@@BB q@@@BL\n" - +" \"MM##MP\"\" q@BB@L\n" - +" \"\"\"`\n" - +"\nCUNEIFORM - A Functional Workflow Language\nVersion " - +LABEL_VERSION+" build "+LABEL_BUILD+"\n\nJorgen Brandt Marc Bux Ulf Leser\n"; - } - - public static int fetchCtl( TopLevelContext tlc ) { - - int ctl; - List removeCandidateList; - CompoundExpr cp; - - if( tlc == null ) - throw new NullPointerException( "Top level context must not be null." ); - - ctl = 0; - removeCandidateList = new ArrayList<>(); - - - for( CompoundExpr ce : tlc.getTargetList() ) { - - cp = new CompoundExpr( ce ); - - for( SingleExpr se : cp.getSingleExprList() ) - if( se instanceof NameExpr ) { - - - if( ( ( NameExpr )se ).getId().equals( "state" ) ) { - ctl += CTL_STATE; - ce.remove( se ); - removeCandidateList.add( ce ); - continue; - } - - if( ( ( NameExpr )se ).getId().equals( "quit" ) ) { - ctl += CTL_QUIT; - ce.remove( se ); - removeCandidateList.add( ce ); - continue; - } - - if( ( ( NameExpr )se ).getId().equals( "queries" ) ) { - ctl += CTL_QUERYSET; - ce.remove( se ); - removeCandidateList.add( ce ); - continue; - } - - if( ( ( NameExpr )se ).getId().equals( "tickets" ) ) { - ctl += CTL_TICKETSET; - ce.remove( se ); - removeCandidateList.add( ce ); - continue; - } - } - } - - - for( CompoundExpr ce : removeCandidateList ) - if( ce.getNumSingleExpr() == 0 ) - tlc.removeTarget( ce ); - - return ctl; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/CmdlineRepl.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/CmdlineRepl.java deleted file mode 100644 index ec49352..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/CmdlineRepl.java +++ /dev/null @@ -1,21 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.repl; - -import java.util.UUID; - -import org.apache.commons.logging.Log; - -import de.huberlin.wbi.cuneiform.core.ticketsrc.ReplTicketSrc; - -public class CmdlineRepl extends InteractiveRepl { - - public CmdlineRepl( ReplTicketSrc ticketSrc, Log statLog ) { - super( ticketSrc, statLog ); - } - - @Override - public synchronized void queryFailedPost( UUID queryId, Long ticketId, Exception e, String script, String stdOut, String stdErr ) { - super.queryFailedPost( queryId, ticketId, e, script, stdOut, stdErr ); - System.exit( -1 ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/DynamicNodeVisitor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/DynamicNodeVisitor.java deleted file mode 100644 index c6e83c5..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/DynamicNodeVisitor.java +++ /dev/null @@ -1,810 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.repl; - -import java.util.LinkedList; -import java.util.UUID; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.BaseBlock; -import de.huberlin.wbi.cuneiform.core.semanticmodel.BaseNodeVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Block; -import de.huberlin.wbi.cuneiform.core.semanticmodel.EnumHelper; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CondExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CurryExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.LambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NativeLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SemanticModelException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SingleExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; -import de.huberlin.wbi.cuneiform.core.ticketsrc.NodeVisitorTicketSrc; - -public class DynamicNodeVisitor extends BaseNodeVisitor { - - private final NodeVisitorTicketSrc ticketSrc; - private final BaseRepl repl; - private final UUID queryId; - private BaseBlock currentBlock; - private final LinkedList blockStack; - private final Log log; - private final Log statLog; - - public DynamicNodeVisitor( NodeVisitorTicketSrc ticketSrc, BaseRepl repl, TopLevelContext tlc ) { - - if( ticketSrc == null ) - throw new NullPointerException( "Ticket source must not be null." ); - - if( repl == null ) - throw new NullPointerException( "REPL must not be null." ); - - this.repl = repl; - this.ticketSrc = ticketSrc; - - queryId = UUID.randomUUID(); - blockStack = new LinkedList<>(); - log = LogFactory.getLog( DynamicNodeVisitor.class ); - statLog = LogFactory.getLog( "statLogger" ); - - setTopLevelContext( tlc ); - } - - @Override - public CompoundExpr accept( NameExpr nameExpr ) throws HasFailedException, NotBoundException { - return currentBlock.getExpr( nameExpr ).visit( this ); - } - - @Override - public CompoundExpr accept( CondExpr condExpr ) throws HasFailedException, NotBoundException { - - CompoundExpr ifExpr, ifExpr1; - CondExpr cnd1; - - ifExpr = condExpr.getIfExpr(); - if( ifExpr == null ) - throw new NullPointerException( "Condition must not be null." ); - - // try to reduce task expression - ifExpr1 = ifExpr.visit( this ); - if( ifExpr1 == null ) - throw new NullPointerException( "Evaluation of condition must not result in null." ); - - try { - - if( ifExpr1.isNormal() ) { - - // the result could be determined - - if( ifExpr1.getNumAtom() == 0 ) - return condExpr.getElseExpr().visit( this ); - - return condExpr.getThenExpr().visit( this ); - - } - - // the result is still open - cnd1 = new CondExpr( ifExpr1, condExpr.getThenExpr(), condExpr.getElseExpr() ); - - return new CompoundExpr( cnd1 ); - } - catch( NotDerivableException e ) { - throw new RuntimeException( e ); - } - - } - - @Override - public CompoundExpr accept( ApplyExpr applyExpr ) throws HasFailedException, NotBoundException { - - ApplyExpr applyExpr1; - CompoundExpr taskExpr1, paramExpr, paramExpr1; - boolean rest; - int channel; - SingleExpr se; - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: "+applyExpr.toString().replace( '\n', ' ' ) ); - - // prepare reduced apply expression - channel = applyExpr.getChannel(); - rest = applyExpr.hasRest(); - applyExpr1 = new ApplyExpr( channel, rest ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Trying to reduce task expression." ); - - // try to reduce task expression - taskExpr1 = applyExpr.getTaskExpr().visit( this ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Replacing task expression with "+taskExpr1+"." ); - applyExpr1.setTaskExpr( taskExpr1 ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Trying to reduce parameter list." ); - - // try to reduce the parameter list - try { - for( NameExpr nameExpr : applyExpr.getNameSet() ) { - - paramExpr = applyExpr.getExpr( nameExpr ); - paramExpr1 = paramExpr.visit( this ); - - applyExpr1.putAssign( nameExpr, paramExpr1 ); - } - } - catch( NotBoundException e ) { - throw new RuntimeException( e ); - } - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Trying to push rest bindings." ); - - // try to push the rest - /* try { - - applyExpr1.attemptPushRest(); - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Push of rest bindings successful or none present." ); - - } - catch( NotDerivableException e ) { - - // if it does not work in this round, it may still work in the next - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Push of rest bindings not successful. It may still work later." ); - } */ - - if( taskExpr1.getNumSingleExpr() == 1 ) { - - // task expression is single expression - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Task expression is single expression." ); - - try { - - if( taskExpr1.getNumAtom() == 1 ) { - - // task expression is single value - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Task expression is single value." ); - - se = taskExpr1.getSingleExpr( 0 ); - - if( !( se instanceof LambdaExpr ) ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Singular task expression is not a lambda expression. Returning what we have so far: "+applyExpr1.toString().replace( '\n', ' ' ) ); - - return new CompoundExpr( applyExpr1 ); - } - - // combine parameters - return combineParam( applyExpr1 ); - - } - - // task expression is single expression but multiple value - return reducePotentiallyCorrelated( applyExpr1 ); - } - catch( NotDerivableException e ) { - - // cardinality cannot be derived yet - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Cardinality cannot be derived yet. Returning what we have so far: "+applyExpr.toString().replace( '\n', ' ' ) ); - - return new CompoundExpr( applyExpr1 ); - } - } - - // task expression is multiple - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor accept ApplyExpr: Task expression is multiple." ); - - return reducePotentiallyCorrelated( applyExpr1 ); - - } - - - - - @Override - public CompoundExpr accept( CompoundExpr ce ) throws HasFailedException, NotBoundException { - - CompoundExpr result, intermediate; - - if( ce == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - result = new CompoundExpr(); - - for( SingleExpr singleExpr : ce.getSingleExprList() ) { - - intermediate = singleExpr.visit( this ); - if( intermediate == null ) - throw new NullPointerException( "Evaluation of single expression must not result in null." ); - - result.addCompoundExpr( intermediate ); - - } - - return result; - } - - @Override - public CompoundExpr accept( CurryExpr curryExpr ) throws HasFailedException, NotBoundException { - - Prototype originalPrototype; - SingleExpr se; - LambdaExpr lambdaExpr; - NativeLambdaExpr nativeLambdaExpr; - Block originalBodyBlock; - Block curriedBodyBlock; - Prototype curriedPrototype; - NativeLambdaExpr curriedLambdaExpr; - - if( !curryExpr.hasTaskExpr() ) - throw new SemanticModelException( - curryExpr.toString(), - "Task parameter not bound." ); - - if( curryExpr.getTaskExpr().getNumSingleExpr() == 0 ) - throw new SemanticModelException( - curryExpr.toString(), - "Task expression must not be nil." ); - - if( curryExpr.getTaskExpr().getNumSingleExpr() > 1 ) - return new CompoundExpr( curryExpr ); - - se = curryExpr.getTaskExpr().visit( this ).getSingleExpr( 0 ); - - if( se instanceof NameExpr ) - return new CompoundExpr( curryExpr ); - - if( !( se instanceof LambdaExpr ) ) - throw new SemanticModelException( curryExpr.toString(), - se+" is not a lambda expression." ); - - - lambdaExpr = ( LambdaExpr )se; - - originalPrototype = lambdaExpr.getPrototype(); - - // the prototype of the curried lambda expression is derived from - // the original prototype - curriedPrototype = new Prototype( originalPrototype ); - - // from the prototype we remove all inputs that are bound by - // currying - for( NameExpr nameExpr : curryExpr.getNameSet() ) - curriedPrototype.removeParam( nameExpr ); - - // TODO: Foreign lambda expressions can of course also be curried - // TODO: Make sure that the remaining parameters and all output - // variables in the prototype are uncorrelated reduce - if( !( lambdaExpr instanceof NativeLambdaExpr ) ) - throw new UnsupportedOperationException( "NYI. Only native lambda expression can be curried." ); - - nativeLambdaExpr = ( NativeLambdaExpr )lambdaExpr; - - originalBodyBlock = nativeLambdaExpr.getBodyBlock(); - - // the body block of the curried lambda expression is derived from the - // body block of the original lambda expression - curriedBodyBlock = new Block(); - for( NameExpr ne : originalBodyBlock.getFullNameSet() ) - curriedBodyBlock.putAssign( ne, new CompoundExpr( originalBodyBlock.getExpr( ne ) ) ); - - // with the curried expression's binding block merged in - try { - for( NameExpr nameExpr : curryExpr.getNameSet() ) - curriedBodyBlock.putAssign( - nameExpr, curryExpr.getExpr( nameExpr ) ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - // from the curried prototype and body expression we form the - // resulting curried lambda expression - curriedLambdaExpr = new NativeLambdaExpr( curriedPrototype, curriedBodyBlock ); - - return new CompoundExpr( curriedLambdaExpr ); - - // reuse this commented block when in dynamic reducer - /* if( lambdaExpr instanceof ForeignLambdaExpr ) { - - applyExpr = new ApplyExpr( 1, false ); - - applyExpr.setTaskExpr( new CompoundExpr( lambdaExpr ) ); - - try { - - for( NameExpr nameExpr : originalPrototype.getParamNameExprSet() ) - - if( curryExpr.containsName( nameExpr ) ) - applyExpr.putAssign( nameExpr, curryExpr.getExpr( nameExpr ) ); - else - applyExpr.putAssign( nameExpr, new CompoundExpr( nameExpr ) ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - curriedBodyBlock = new Block(); - - n = originalPrototype.getNumOutput(); - - // replicate apply expression for each output name expression - for( i = 0; i < n; i++ ) { - - ne = originalPrototype.getOutput( i ); - - ae = applyExpr.clone(); - ae.setChannel( i+1 ); - - curriedBodyBlock.putAssign( ne, new CompoundExpr( ae ) ); - } - - curriedLambdaExpr = new NativeLambdaExpr( curriedPrototype, curriedBodyBlock ); - - return new CompoundExpr( curriedLambdaExpr ); - } */ - - } - - public UUID getQueryId() { - return queryId; - } - - public void setTopLevelContext( TopLevelContext tlc ) { - - if( tlc == null ) - throw new NullPointerException( "Top level context must not be null." ); - - currentBlock = tlc; - - } - - public void step() { - - CompoundExpr ce; - long tic, toc; - StringBuffer buf; - - try { - tic = System.currentTimeMillis(); - - ce = currentBlock.visit( this ); - toc = System.currentTimeMillis(); - - if( statLog.isDebugEnabled() ) - statLog.debug( new JsonReportEntry( ticketSrc.getRunId(), null, null, null, null, null, JsonReportEntry.KEY_REDUCTION_TIME, String.valueOf( toc-tic ) ) ); - - if( log.isDebugEnabled() ) - log.debug( "Checking if queue is clear ..." ); - - if( ticketSrc.isQueueClear( queryId ) ) { - - if( log.isDebugEnabled() ) - log.debug( "Queue is clear. Finishing query "+queryId+"." ); - - repl.queryFinished( queryId, ce ); - - return; - } - - if( log.isDebugEnabled() ) { - - buf = new StringBuffer(); - - buf.append( "Queue is not empty. " ).append( queryId ).append( " waits for [ " ); - for( Ticket t : ticketSrc.getTicketSet( queryId ) ) - buf.append( t.getTicketId() ).append( ' ' ); - buf.append( ']' ); - - log.debug( buf.toString() ); - } - - - } - catch( Exception e ) { - repl.queryFailed( queryId, null, e, null, null, null ); - } - } - - public CompoundExpr getCurrentExpr() { - - CompoundExpr result; - TopLevelContext tlc; - - if( !( currentBlock instanceof TopLevelContext ) ) - throw new RuntimeException( "Current block is not a top level context." ); - - tlc = ( TopLevelContext )currentBlock; - - result = new CompoundExpr(); - - for( CompoundExpr ce : tlc.getTargetList() ) - result.addCompoundExpr( ce ); - - return result; - - } - - private CompoundExpr combineParam( ApplyExpr applyExpr ) throws HasFailedException { - - EnumHelper helper; - int i, n; - CompoundExpr ce, ce0; - Block bindingBlock; - LambdaExpr lambda; - ApplyExpr singularApplyExpr; - QualifiedTicket qt; - - try { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: "+applyExpr.toString().replace( '\n', ' ') ); - - helper = new EnumHelper( applyExpr ); - n = helper.getCardinality(); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Creating CombiHelper with cardinality "+n ); - - ce = new CompoundExpr(); - - for( i = 0; i < n; i++ ) { - - bindingBlock = helper.getSingularBindingBlock( i ); // throws NotDerivableException - lambda = helper.getSingularLambdaExpr( i ); // throws NotDerivableException - - singularApplyExpr = new ApplyExpr( applyExpr.getChannel(), applyExpr.hasRest() ); - singularApplyExpr.setParamBindMap( bindingBlock.getParamBindMap() ); - singularApplyExpr.setTaskExpr( new CompoundExpr( lambda ) ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Enumerating singular application "+singularApplyExpr.toString().replace( '\n', ' ' ) ); - - if( lambda instanceof ForeignLambdaExpr ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Task expression is foreign." ); - - if( singularApplyExpr.isParamNormal() ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Application parameters are in normal form." ); - - qt = ticketSrc.requestTicket( repl, queryId, singularApplyExpr ); - if( qt == null ) - throw new NullPointerException( "Qualified ticket must not be null." ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Ticket requested. Appending qualified ticket: "+qt.toString().replace( '\n', ' ' ) ); - - ce.addSingleExpr( qt ); - continue; - } - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Application parameters are not in normal form. Appending what we have so far: "+singularApplyExpr.toString().replace( '\n', ' ' ) ); - - ce.addSingleExpr( singularApplyExpr ); - continue; - } - - - if( lambda instanceof NativeLambdaExpr ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Task expression is native." ); - - ce0 = reduceSingleNative( singularApplyExpr ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Appending result of single native reduction step: "+ce0.toString().replace( '\n', ' ' ) ); - - ce.addCompoundExpr( ce0 ); - continue; - } - - throw new RuntimeException( "Lambda expression type not recognized." ); - } - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Combination complete; returning: "+ce.toString().replace( '\n', ' ' ) ); - - return ce; - } - catch( NotDerivableException e ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor combineParam ApplyExpr: Some information could not be derived. Returning original: "+applyExpr.toString().replace( '\n', ' ' ) ); - - return new CompoundExpr( applyExpr ); - } - } - - private void popBlock() { - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor popping out of block." ); - - currentBlock = blockStack.pop(); - } - - private void pushIntoBlock( Block block ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor pushing into block." ); - - - blockStack.push( currentBlock ); - currentBlock = block; - } - - private CompoundExpr reducePotentiallyCorrelated( ApplyExpr applyExpr ) throws HasFailedException, NotBoundException { - - CompoundExpr taskExpr, ce, ce0; - SingleExpr se; - LambdaExpr lambda; - Prototype prototype; - ApplyExpr applyExpr1; - int i, n; - - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: "+applyExpr.toString().replace( '\n', ' ' ) ); - - - taskExpr = applyExpr.getTaskExpr(); - se = taskExpr.getSingleExpr( 0 ); - - if( !( se instanceof LambdaExpr ) ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Task expression is not a lambda expression. Returning original: "+applyExpr.toString().replace( '\n', ' ' ) ); - - return new CompoundExpr( applyExpr ); - } - - lambda = ( LambdaExpr )se; - prototype = lambda.getPrototype(); - - if( prototype.isTaskCorrelated() ) { - - // task parameter is correlated - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Prototype is task-correlated." ); - - // combine and create tickets - - return combineParam( applyExpr ); - } - - // enumerate task expression and create an application from each - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Prototype is not task-correlated.Enumerate task expressions and create application for each." ); - - try { - n = taskExpr.getNumAtom(); - } - catch( NotDerivableException e ) { - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Could not derive cardinality of task expression. Returning original: "+applyExpr.toString().replace( '\n', ' ' ) ); - - return new CompoundExpr( applyExpr ); - } - - ce = new CompoundExpr(); - - for( i = 0; i < n; i++ ) { - - applyExpr1 = new ApplyExpr( applyExpr ); - - // TODO: This won't work if the task expression was the result of an output-reduce task - ce0 = new CompoundExpr( taskExpr.getSingleExpr( i ) ); - - try { - if( ce0.getNumAtom() != 1 ) - throw new RuntimeException( "Enumeration resulted in non-singular expression: "+ce0 ); - } catch (NotDerivableException e) { - throw new RuntimeException( e ); - } - - applyExpr1.setTaskExpr( ce0 ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Appending "+applyExpr.toString().replace( '\n', ' ' ) ); - - ce.addSingleExpr( applyExpr1 ); - } - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Trying to reduce "+ce.toString().replace( '\n', ' ' ) ); - - ce0 = ce.visit( this ); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reducePotentiallyCorrelated ApplyExpr: Returning "+ce0.toString().replace( '\n', ' ' ) ); - - return ce0; - - } - - private CompoundExpr reduceSingleNative( ApplyExpr applyExpr ) throws HasFailedException { - - CompoundExpr taskResult; - SingleExpr se; - NativeLambdaExpr lambda; - NameExpr targetNameExpr; - int channel; - CompoundExpr targetCompoundExpr, ce; - ApplyExpr applyExpr1; - CompoundExpr lamList; - boolean push; - Block body, originalBody; - Prototype sign; - Block binding; - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reduceSingleNative ApplyExpr: "+applyExpr.toString().replace( '\n', ' ' ) ); - - - taskResult = applyExpr.getTaskExpr(); - try { - if( taskResult.getNumAtom() != 1 ) - throw new RuntimeException( "Expected application with singular task expression" ); - } - catch( NotDerivableException e2 ) { - throw new RuntimeException( "Cannot derive cardinality of task expression." ); - } - - channel = applyExpr.getChannel(); - push = applyExpr.hasRest(); - - // fetch that lambda expression - se = taskResult.getSingleExpr( 0 ); - - if( se == null ) - throw new NullPointerException( "Single expression must not be null." ); - - // continue only if that single lambda expression is native - if( !( se instanceof NativeLambdaExpr ) ) - throw new RuntimeException( "Single native lambda expression expected." ); - - - lambda = ( NativeLambdaExpr )se; - applyExpr.setParent( lambda.getBodyBlock() ); - - - try { - pushIntoBlock( applyExpr.getParamBlock() ); - } catch( NotDerivableException e ) { - throw new RuntimeException( e ); - } - - targetNameExpr = lambda.getPrototype().getOutput( channel-1 ); - - try { - - if( log.isDebugEnabled() ) - log.debug( "Visiting "+currentBlock.getExpr( targetNameExpr ) ); - - targetCompoundExpr = currentBlock.getExpr( targetNameExpr ).visit( this ); - if( targetCompoundExpr == null ) - throw new NullPointerException( "Evaluation of target expression must not result in null." ); - } - catch( NotBoundException e1 ) { - throw new SemanticModelException( applyExpr.toString(), e1.getMessage() ); - } - popBlock(); - - if( log.isTraceEnabled() ) - log.trace( "DynamicNodeVisitor reduceSingleNative ApplyExpr: Returning "+targetCompoundExpr.toString().replace( '\n', ' ' ) ); - - if( targetCompoundExpr.isNormal() ) - return targetCompoundExpr; - - - - sign = lambda.getPrototype(); - - - originalBody = lambda.getBodyBlock(); - - try { - body = new Block(); - for( NameExpr ne : originalBody.getFullNameSet() ) - body.putAssign( ne, new CompoundExpr( originalBody.getExpr( ne ) ) ); - } catch (NotBoundException e) { - throw new RuntimeException( e ); - } - - body.putAssign( targetNameExpr, targetCompoundExpr ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - binding = new Block(); - for( NameExpr ne : applyExpr.getParamBindMap().keySet() ) { - - ce = applyExpr.getParamBindMap().get( ne ); - - binding.putAssign( ne, new CompoundExpr( ce ) ); - } - - - applyExpr1 = new ApplyExpr( channel, push ); - applyExpr1.setTaskExpr( lamList ); - applyExpr1.setParamBindMap( binding.getParamBindMap() ); - - - return new CompoundExpr( applyExpr1 ); - - - } - - @Override - public CompoundExpr accept( TopLevelContext tlc ) throws HasFailedException, NotBoundException { - - CompoundExpr result; - - - result = new CompoundExpr(); - - for( CompoundExpr ce : tlc.getTargetList() ) - result.addCompoundExpr( ce.visit( this ) ); - - tlc.clearTargetList(); - tlc.addTarget( result ); - - return result; - } - - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/InteractiveRepl.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/InteractiveRepl.java deleted file mode 100644 index eb99ac7..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/InteractiveRepl.java +++ /dev/null @@ -1,114 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.repl; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.StringReader; -import java.util.UUID; - -import org.apache.commons.logging.Log; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.ticketsrc.ReplTicketSrc; - -public class InteractiveRepl extends BaseRepl { - - private static final int MAX_OUTPUT_LEN = 1000; - - public InteractiveRepl( ReplTicketSrc ticketSrc, Log statLog ) { - super( ticketSrc, statLog ); - } - - @Override - public synchronized void queryFinishedPost( UUID queryId, CompoundExpr result ) { - // nothing to do - } - - @Override - public synchronized void queryStartedPost( UUID runId ) { - // nothing to do - } - - @Override - public synchronized void queryFailedPost( UUID queryId, Long ticketId, Exception e, String script, String stdOut, String stdErr ) { - - String line; - int i; - - if( script != null ) - try( BufferedReader reader = new BufferedReader( new StringReader( script ) ) ) { - - System.out.println( "[script]" ); - - i = 0; - - while( ( line = reader.readLine() ) != null ) - System.out.println( String.format( "%3d %s", ++i, line ) ); - - } - catch( IOException e1 ) { - throw new RuntimeException( e1 ); - } - - if( stdOut != null ) { - System.out.println( "[out]" ); - - i = stdOut.length(); - if( i > MAX_OUTPUT_LEN ) - line = "...\n"+stdOut.substring( i-MAX_OUTPUT_LEN ); - else - line = stdOut; - - System.out.println( line ); - } - - if( stdErr != null ) { - System.out.println( "[err]" ); - - i = stdErr.length(); - if( i > MAX_OUTPUT_LEN ) - line = "...\n"+stdErr.substring( i-MAX_OUTPUT_LEN ); - else - line = stdErr; - - System.out.println( line ); - } - - if( e != null ) { - System.out.println( "[trace]" ); - e.printStackTrace(); - } - - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/QueryParseCtlLexer.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/QueryParseCtlLexer.java deleted file mode 100644 index 9c435b2..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/repl/QueryParseCtlLexer.java +++ /dev/null @@ -1,81 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.repl; - -import org.antlr.v4.runtime.CharStream; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; - -import de.huberlin.wbi.cuneiform.core.parser.ParseCtlLexer; - -public class QueryParseCtlLexer extends ParseCtlLexer { - - public QueryParseCtlLexer( CharStream input ) { - super( input ); - } - - public boolean isReady() { - - int depth; - Token t; - boolean ready; - - reset(); - - depth = 0; - ready = true; - while( ( t = nextToken() ).getType() != Recognizer.EOF ) { - - switch( t.getType() ) { - - case ParseCtlLexer.LXCOMMENT : - case ParseCtlLexer.LCCOMMENT : - case ParseCtlLexer.LBRACE : depth++; break; - - case ParseCtlLexer.RXCOMMENT : - case ParseCtlLexer.RCCOMMENT : - case ParseCtlLexer.RBRACE : depth--; if( depth == 0 ) ready = true; break; - - case ParseCtlLexer.RMMECB : if( depth == 0 ) ready = true; break; - case ParseCtlLexer.ANY : if( depth == 0 ) ready = false; break; - case ParseCtlLexer.SEMICOLON : if( depth == 0 ) ready = true; break; - default : break; // ignore - } - } - - if( depth != 0 ) - return false; - - return ready; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ApplyExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ApplyExpr.java deleted file mode 100644 index 62c5d98..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ApplyExpr.java +++ /dev/null @@ -1,324 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - - -public class ApplyExpr extends BaseBlock implements SingleExpr { - - private int channel; - private boolean rest; - private CompoundExpr taskExpr; - - public ApplyExpr( int channel, boolean inheritsExtra ) { - this( channel, inheritsExtra, null ); - } - - public ApplyExpr( int channel, boolean inheritsExtra, BaseBlock parent ) { - - super( parent ); - - setChannel( channel ); - setRest( inheritsExtra ); - } - - public ApplyExpr( ApplyExpr template ) { - - if( template.taskExpr == null ) - throw new IllegalArgumentException( "Template apply expression has no task expression: "+template+"." ); - - channel = template.channel; - rest = template.rest; - taskExpr = new CompoundExpr( template.taskExpr ); - - try { - for( NameExpr ne : template.getFullNameSet() ) - putAssign( ne, new CompoundExpr( template.getExpr( ne ) ) ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e ); - } - } - - public String getBlockString() { - return super.toString(); - } - - public int getChannel() { - return channel; - } - - public Prototype getPrototype() throws NotDerivableException { - - for( SingleExpr se : taskExpr.getSingleExprList() ) - if( se instanceof LambdaExpr ) - return ( ( LambdaExpr )se ).getPrototype(); - - throw new NotDerivableException( "Cannot derive prototype for "+taskExpr+"." ); - } - - public Block getParamBlock() throws NotDerivableException { - - Prototype prototype; - Block paramBlock; - - prototype = getPrototype(); - - paramBlock = new Block( getParent() ); - - try { - for( NameExpr nameExpr : getNameSet() ) { - - if( prototype.containsParam( nameExpr ) ) - paramBlock.putAssign( nameExpr, getExpr( nameExpr ) ); - } - } - catch( NotBoundException e ) { - // cannot happen because we only ask for legal symbols - throw new RuntimeException( e.getMessage() ); - } - - return paramBlock; - } - - public CompoundExpr getTaskExpr() { - - if( taskExpr == null ) - throw new NullPointerException( "Task expression never set." ); - - return taskExpr; - } - - @Override - public StringExpr getStringExprValue( int i ) throws NotDerivableException { - throw new NotDerivableException( "Cannot derive value in application." ); - } - - @Override - public int hashCode() { - - String s; - - s = toString(); - s = s.substring( s.indexOf( ']' ) ); - - return s.hashCode(); - } - - public boolean hasRest() { - return rest; - } - - public boolean hasTaskExpr() { - return taskExpr != null; - } - - public boolean isParamNormal() { - - CompoundExpr ce; - - try { - for( NameExpr nameExpr : getNameSet() ) { - - ce = getExpr( nameExpr ); - if( !ce.isNormal() ) - return false; - } - - return true; - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - } - - public void attemptPushRest() throws NotDerivableException { - - Block restBlock; - Prototype prototype; - NativeLambdaExpr nativeLambda; - Block bodyBlock; - - try { - prototype = getPrototype(); - - restBlock = new Block(); - - for( NameExpr name : getNameSet() ) - if( !prototype.containsParam( name ) ) - restBlock.putAssign( name, getExpr( name ) ); - - if( !taskExpr.isNormal() ) - throw new NotDerivableException( "Task expression not in not normal." ); - - for( SingleExpr se : taskExpr.getSingleExprList() ) { - - if( se instanceof NativeLambdaExpr ) { - - - nativeLambda = ( NativeLambdaExpr )se; - - bodyBlock = nativeLambda.getBodyBlock(); - for( NameExpr name : bodyBlock.getNameSet() ) - bodyBlock.getExpr( name ).pushRest( restBlock ); - - continue; - } - - if( se instanceof ForeignLambdaExpr ) - continue; - - throw new RuntimeException( - "Lambda expression expected." ); - - } - - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - } - - @Override - public void pushRest( BaseBlock restBlock ) { - - if( !rest ) - return; - - super.pushRest( restBlock ); - - rest = false; - } - - public void setChannel( int channel ) { - - if( channel < 1 ) - throw new SemanticModelException( - String.valueOf( channel ), - "Invalid channel "+channel+". Channel must be a positive integer." ); - - this.channel = channel; - } - - public void setRest( boolean inheritsExtra ) { - this.rest = inheritsExtra; - } - - public void setTaskExpr( CompoundExpr taskExpr ) { - - if( taskExpr == null ) - throw new NullPointerException( "Task expression must not be null." ); - - this.taskExpr = taskExpr; - } - - @Override - public String toString() { - - StringBuffer buf; - boolean comma; - - try { - - buf = new StringBuffer(); - - buf.append( '[' ).append( channel ).append( "]apply(" ); - - if( hasTaskExpr() ) - buf.append( " task: " ).append( taskExpr ); - - comma = false; - for( NameExpr name : getNameSet() ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( ' ' ).append( name.getId() ).append( ": " ).append( getExpr( name ) ); - } - - if( rest ) - buf.append( " ~" ); - - buf.append( " )" ); - - return buf.toString(); - } - catch( NotBoundException e ) { - throw new RuntimeException( e ); - } - } - - @Override - public int getNumAtom() throws NotDerivableException { - - Prototype prototype; - NameExpr name; - - prototype = getPrototype(); - name = prototype.getOutput( channel-1 ); - - if( name instanceof ReduceVar ) - throw new NotDerivableException( "Cannot derive cardinality of reduce output variable." ); - - return 1; - } - - @Override - public boolean equals( Object obj ) { - - ApplyExpr other; - String str1, str2; - - if( !( obj instanceof ApplyExpr ) ) - return false; - - other = ( ApplyExpr )obj; - - str1 = toString(); - str1 = str1.substring( str1.indexOf( ']' ) ); - - str2 = other.toString(); - str2 = str2.substring( str2.indexOf( ']' ) ); - - return str1.equals( str2 ); - } - - @Override - public T visit( NodeVisitor visitor ) throws HasFailedException, NotBoundException { - return visitor.accept( this ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/BaseBlock.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/BaseBlock.java deleted file mode 100644 index 8c7fde7..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/BaseBlock.java +++ /dev/null @@ -1,182 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.Set; - -public abstract class BaseBlock implements CfNode, Cloneable { - - private Map assignMap; - private BaseBlock parent; - - public BaseBlock() { - this( null ); - } - - public BaseBlock( BaseBlock parent ) { - assignMap = new HashMap<>(); - setParent( parent ); - } - - public void clear() { - assignMap.clear(); - } - - public boolean containsName( NameExpr nameExpr ) { - return assignMap.containsKey( nameExpr ); - } - - public CompoundExpr getExpr( NameExpr name ) throws NotBoundException { - - CompoundExpr ce; - - ce = assignMap.get( name ); - - if( ce != null ) - return ce; - - if( parent == null ) - throw new NotBoundException( "A name '"+name+"' is not bound in this block." ); - - return parent.getExpr( name ); - } - - public CompoundExpr getExpr( String name ) throws NotBoundException { - return getExpr( new NameExpr( name ) ); - } - - public Set getNameSet() { - return assignMap.keySet(); - } - - public Set getFullNameSet() { - - Set set; - - set = new HashSet<>(); - set.addAll( assignMap.keySet() ); - - if( parent != null ) - set.addAll( parent.getFullNameSet() ); - - return set; - } - - public Map getParamBindMap() { - return Collections.unmodifiableMap( assignMap ); - } - - public BaseBlock getParent() { - return parent; - } - - public boolean hasParent() { - return parent != null; - } - - public boolean isEmpty() { - return assignMap.isEmpty(); - } - - public void pushRest( BaseBlock restBlock ) { - - try { - - for( NameExpr name : restBlock.getNameSet() ) - if( !containsName( name ) ) - putAssign( name, restBlock.getExpr( name ) ); - - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - } - - public void putAssign( NameExpr key, CompoundExpr value ) { - - if( key == null ) - throw new NullPointerException( "Name expression must not be null." ); - - if( value == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - assignMap.put( key, value ); - } - - public void removeAssign( String name ) { - removeAssign( new NameExpr( name ) ); - } - - public void removeAssign( NameExpr nameExpr ) { - - if( assignMap.remove( nameExpr ) == null ) - throw new NullPointerException( "Cannot remove. Assignment does not exist." ); - } - - public void setParamBindMap( Map paramBindMap ) { - - if( assignMap == null ) - throw new NullPointerException( "Parameter binding map must not be null." ); - - assignMap = paramBindMap; - } - - public void setParent( BaseBlock parent ) { - this.parent = parent; - } - - @Override - public String toString() { - - StringBuffer buf; - List nameExprSet; - - buf = new StringBuffer(); - - nameExprSet = new LinkedList<>(); - - nameExprSet.addAll( assignMap.keySet() ); - Collections.sort( nameExprSet ); - - for( NameExpr nameExpr : nameExprSet ) - buf.append( nameExpr ).append( " = " ).append( assignMap.get( nameExpr ) ).append( ";\n" ); - - return buf.toString(); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/BaseNodeVisitor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/BaseNodeVisitor.java deleted file mode 100644 index 7c0ede2..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/BaseNodeVisitor.java +++ /dev/null @@ -1,94 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public abstract class BaseNodeVisitor implements NodeVisitor { - - @Override - public CompoundExpr accept( Block block ) { - throw new RuntimeException( "Block is not an expression." ); - } - - @Override - public CompoundExpr accept( ForeignLambdaExpr foreignLambdaExpr ) { - return new CompoundExpr( foreignLambdaExpr ); - } - - @Override - public CompoundExpr accept( NativeLambdaExpr nativeLambdaExpr ) { - return new CompoundExpr( nativeLambdaExpr ); - } - - @Override - public CompoundExpr accept( StringExpr stringExpr ) { - return new CompoundExpr( stringExpr ); - } - - @Override - public CompoundExpr accept( QualifiedTicket qualifiedTicket ) { - return new CompoundExpr( qualifiedTicket ); - } - - @Override - public CompoundExpr accept( LambdaType lambdaType ) { - throw new RuntimeException( "Lambda type is not an expression." ); - } - - @Override - public CompoundExpr accept( DrawParam drawParam ) { - throw new RuntimeException( "Draw parameter is not an expression." ); - } - - @Override - public CompoundExpr accept( DataType dataType ) { - throw new RuntimeException( "Data type is not an expression." ); - } - - @Override - public CompoundExpr accept( CorrelParam correlParam ) { - throw new RuntimeException( "Correlated parameter is not an expression." ); - } - - @Override - public CompoundExpr accept( Prototype prototype ) { - throw new RuntimeException( "Prototype is not an expression." ); - } - - @Override - public CompoundExpr accept( ReduceVar reduceVar ) { - throw new RuntimeException( "Reduce variable is not an expression." ); - } - - - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Block.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Block.java deleted file mode 100644 index ec27f1a..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Block.java +++ /dev/null @@ -1,48 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - -public class Block extends BaseBlock { - - public Block() {} - - public Block( BaseBlock parent ) { - super( parent ); - } - - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CfNode.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CfNode.java deleted file mode 100644 index 75940ef..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CfNode.java +++ /dev/null @@ -1,38 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - -public interface CfNode { - public T visit(NodeVisitor visitor) throws HasFailedException, NotBoundException; -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CfSemanticModelVisitor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CfSemanticModelVisitor.java deleted file mode 100644 index 00b93cf..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CfSemanticModelVisitor.java +++ /dev/null @@ -1,692 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.BitSet; -import java.util.LinkedList; - -import org.antlr.v4.runtime.ANTLRErrorListener; -import org.antlr.v4.runtime.ANTLRInputStream; -import org.antlr.v4.runtime.CommonTokenStream; -import org.antlr.v4.runtime.Parser; -import org.antlr.v4.runtime.RecognitionException; -import org.antlr.v4.runtime.Recognizer; -import org.antlr.v4.runtime.Token; -import org.antlr.v4.runtime.atn.ATNConfigSet; -import org.antlr.v4.runtime.dfa.DFA; -import org.antlr.v4.runtime.misc.NotNull; -import org.antlr.v4.runtime.tree.ParseTree; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import de.huberlin.wbi.cuneiform.core.parser.CuneiformBaseVisitor; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformLexer; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.NameContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.OutputContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.ParamBindContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.ParamContext; -import de.huberlin.wbi.cuneiform.core.parser.CuneiformParser.SingleExprContext; -import de.huberlin.wbi.cuneiform.core.preprocess.ParseException; - -public class CfSemanticModelVisitor extends CuneiformBaseVisitor implements ANTLRErrorListener { - - public static final String LABEL_TASK = "task"; - public static final String LABEL_FILE = "File"; - - private BaseBlock currentBlock; - private final LinkedList blockStack; - private final Log log; - - public CfSemanticModelVisitor() { - - currentBlock = new TopLevelContext(); - blockStack = new LinkedList<>(); - log = LogFactory.getLog( CfSemanticModelVisitor.class ); - } - - @Override - public CfNode visitApplyExpr( @NotNull CuneiformParser.ApplyExprContext ctx ) { - - ApplyExpr applyExpr; - int channel; - NameExpr name; - CfNode compoundExpr; - String id; - boolean rest; - - if( ctx.channel() == null ) - channel = 1; - else - channel = Integer.parseInt( ctx.channel().INT().getText() ); - - rest = ctx.TILDE() != null; - applyExpr = new ApplyExpr( channel, rest ); - - - for( ParamBindContext pbc : ctx.paramBind() ) { - - id = pbc.ID().getText(); - name = new NameExpr( id ); - compoundExpr = visit( pbc.expr() ); - - if( compoundExpr == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - if( !( compoundExpr instanceof CompoundExpr ) ) - throw new RuntimeException( "Compound expression expected." ); - - if( id.equals( LABEL_TASK ) ) - applyExpr.setTaskExpr( ( CompoundExpr )compoundExpr ); - else - applyExpr.putAssign( name, ( CompoundExpr )compoundExpr ); - - } - - if( !applyExpr.hasTaskExpr() ) - throw new SemanticModelException( - applyExpr.toString(), - "Task parameter not bound." ); - - return applyExpr; - } - - - @Override - public CfNode visitAssign( @NotNull CuneiformParser.AssignContext ctx ) { - - CfNode nameExpr, compoundExpr; - CompoundExpr ce; - SingleExpr se; - ForeignLambdaExpr lambda; - - if( ctx.name().size() != 1 ) - // sorted out in the channel-phase - throw new RuntimeException( - "Illegal assignment. Left hand side must have exactly one name element." ); - - nameExpr = visit( ctx.name( 0 ) ); - - if( nameExpr == null ) - throw new NullPointerException( "Name expression must not be null." ); - - if( !( nameExpr instanceof NameExpr ) ) - throw new RuntimeException( "Expected name expression." ); - - compoundExpr = visit( ctx.expr() ); - - if( compoundExpr == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - if( !( compoundExpr instanceof CompoundExpr ) ) - throw new RuntimeException( "Expected compound expression. Found "+compoundExpr.getClass().getName()+"." ); - - ce = ( CompoundExpr )compoundExpr; - - currentBlock.putAssign( ( NameExpr )nameExpr, ce ); - - // if we assigned a single foreign lambda expression, store name - if( ce.getNumSingleExpr() == 1 ) { - - se = ce.getSingleExpr( 0 ); - if( se instanceof ForeignLambdaExpr ) { - - lambda = ( ForeignLambdaExpr )se; - lambda.setOptionalTaskName( ( ( NameExpr )nameExpr ).getId() ); - } - - } - - return currentBlock; - } - - @Override - public CfNode visitBlock( @NotNull CuneiformParser.BlockContext ctx ) { - - Block block; - - block = new Block( currentBlock ); - - pushIntoBlock( block ); - visitChildren( ctx ); - popBlock(); - - return block; - } - - @Override - public CfNode visitCallExpr( @NotNull CuneiformParser.CallExprContext ctx ) { - - ApplyExpr applyExpr; - int channel; - NameExpr name; - CfNode compoundExpr; - String id; - boolean rest; - - if( ctx.channel() == null ) - channel = 1; - else - channel = Integer.parseInt( ctx.channel().INT().getText() ); - - rest = ctx.TILDE() != null; - applyExpr = new ApplyExpr( channel, rest ); - - id = ctx.ID().getText(); - applyExpr.setTaskExpr( new CompoundExpr( new NameExpr( id ) ) ); - - - for( ParamBindContext pbc : ctx.paramBind() ) { - - id = pbc.ID().getText(); - name = new NameExpr( id ); - compoundExpr = visit( pbc.expr() ); - - if( compoundExpr == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - if( !( compoundExpr instanceof CompoundExpr ) ) - throw new RuntimeException( "Compound expression expected." ); - - if( id.equals( LABEL_TASK ) ) - applyExpr.setTaskExpr( ( CompoundExpr )compoundExpr ); - else - applyExpr.putAssign( name, ( CompoundExpr )compoundExpr ); - - } - - if( !applyExpr.hasTaskExpr() ) - throw new SemanticModelException( - applyExpr.toString(), - "Task parameter not bound." ); - - return applyExpr; - } - - @Override - public CfNode visitIdExpr( @NotNull CuneiformParser.IdExprContext ctx ) { - - return new NameExpr( ctx.ID().getText() ); - } - - - @Override - public CfNode visitCompoundExpr( @NotNull CuneiformParser.CompoundExprContext ctx ) { - - CompoundExpr compoundExpr; - CfNode singleExpr; - - compoundExpr = new CompoundExpr(); - - for( SingleExprContext sec : ctx.singleExpr() ) { - - singleExpr = visit( sec ); - - if( singleExpr == null ) - throw new NullPointerException( "While evaluating compound expression '"+sec.getText()+"': Single expression must not be null." ); - - - if( !( singleExpr instanceof SingleExpr ) ) - throw new RuntimeException( "Single expresssion expected." ); - - compoundExpr.addSingleExpr( ( SingleExpr )singleExpr ); - } - - return compoundExpr; - } - - @Override - public CfNode visitCondExpr( @NotNull CuneiformParser.CondExprContext ctx ) { - - CfNode ifExpr; - CfNode thenBlock; - CfNode elseBlock; - - - ifExpr = visit( ctx.expr( 0 ) ); - - if( ifExpr == null ) - throw new NullPointerException( "If expression must not be null." ); - - if( !( ifExpr instanceof CompoundExpr ) ) - throw new RuntimeException( "Compound expression expected." ); - - - - thenBlock = visit( ctx.expr( 1 ) ); - - if( thenBlock == null ) - throw new NullPointerException( "Then block must not be null." ); - - if( !( thenBlock instanceof CompoundExpr ) ) - throw new RuntimeException( "Compound expression expected." ); - - elseBlock = visit( ctx.expr( 2 ) ); - - if( elseBlock == null ) - throw new NullPointerException( "Then block must not be null." ); - - if( !( elseBlock instanceof CompoundExpr ) ) - throw new RuntimeException( "Compound expression expected." ); - - return new CondExpr( ( CompoundExpr )ifExpr, ( CompoundExpr )thenBlock, ( CompoundExpr )elseBlock ); - } - - - @Override - public CfNode visitCorrelParam( @NotNull CuneiformParser.CorrelParamContext ctx ) { - - CorrelParam correlParam; - CfNode nameExpr; - - correlParam = new CorrelParam(); - - for( NameContext nc : ctx.name() ) { - - nameExpr = visit( nc ); - - if( nameExpr == null ) - throw new NullPointerException( "Name expression must not be null." ); - - if( !( nameExpr instanceof NameExpr ) ) - throw new RuntimeException( "Name expression expected." ); - - correlParam.addName( ( NameExpr )nameExpr ); - } - - return correlParam; - } - @Override - public CfNode visitCurryExpr( @NotNull CuneiformParser.CurryExprContext ctx ) { - - CurryExpr curryExpr; - String id; - NameExpr name; - CfNode compoundExpr; - - curryExpr = new CurryExpr(); - - for( ParamBindContext pbc : ctx.paramBind() ) { - - id = pbc.ID().getText(); - name = new NameExpr( id ); - compoundExpr = visit( pbc.expr() ); - - if( compoundExpr == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - if( !( compoundExpr instanceof CompoundExpr ) ) - throw new RuntimeException( "Compound expression expected." ); - - if( id.equals( LABEL_TASK ) ) - curryExpr.setTaskExpr( ( CompoundExpr )compoundExpr ); - else - curryExpr.putAssign( name, ( CompoundExpr )compoundExpr ); - - } - - if( !curryExpr.hasTaskExpr() ) - throw new SemanticModelException( - curryExpr.toString(), - "Task parameter not bound." ); - - return curryExpr; - } - - @Override - public CfNode visitDanglingExpr( @NotNull CuneiformParser.DanglingExprContext ctx ) { - // sorted out in the pre-phase - throw new RuntimeException( "Illegal dangling expression encountered." ); - } - - @Override - public CfNode visitForeignDefTask( @NotNull CuneiformParser.ForeignDefTaskContext ctx ) { - // sorted out in the pre-phase - throw new RuntimeException( "Illegal foreign task definition encountered." ); - } - - @Override - public CfNode visitForeignLambdaExpr( @NotNull CuneiformParser.ForeignLambdaExprContext ctx ) { - - CfNode prototype; - String langString; - String body; - - prototype = visit( ctx.prototype() ); - - if( prototype == null ) - throw new NullPointerException( "Prototype must not be null." ); - - if( !( prototype instanceof Prototype ) ) - throw new RuntimeException( "Prototype expected." ); - - langString = ctx.foreignBody().INLANG().getText(); - langString = langString.substring( 2 ).replace( ",", "" ).trim(); - - body = ctx.foreignBody().BODY().getText(); - body = body.substring( 2, body.length()-2 ); - - if( body.trim().isEmpty() ) - throw new SemanticModelException( - prototype+"in "+langString+" *{}*", - "Foreign body block must not be empty." ); - - for( NameExpr name : ( ( Prototype )prototype ).getOutputList() ) - if( !body.contains( name.getId() ) ) - throw new SemanticModelException( prototype+"in "+langString+" *{ "+body+" }*", "Output variable "+name.getId()+" is never bound." ); - - return new ForeignLambdaExpr( ( Prototype )prototype, langString, body ); - } - - - @Override public CfNode visitFromStackExpr( @NotNull CuneiformParser.FromStackExprContext ctx ) { - // sorted out in the pre-phase - throw new RuntimeException( "Illegal from-stack expression encountered." ); - } - - @Override - public CfNode visitImportFile( @NotNull CuneiformParser.ImportFileContext ctx ) { - throw new RuntimeException( "Illegal import statement encountered." ); - } - - @Override - public CfNode visitIntExpr( @NotNull CuneiformParser.IntExprContext ctx ) { - return new StringExpr( ctx.getText() ); - } - - - @Override - public CfNode visitNameDataType( @NotNull CuneiformParser.NameDataTypeContext ctx ) { - return new NameExpr( ctx.ID( 0 ).getText(), new DataType( ctx.ID( 1 ).getText() ) ); - } - - @Override - public CfNode visitNameDeepFnType( @NotNull CuneiformParser.NameDeepFnTypeContext ctx ) { - - CfNode prototype; - - prototype = visit( ctx.prototype() ); - - if( prototype == null ) - throw new NullPointerException( "Prototype must not be null." ); - - if( !( prototype instanceof Prototype ) ) - throw new RuntimeException( "Prototype expected." ); - - return new NameExpr( ctx.ID().getText(), ( Prototype )prototype ); - - } - - @Override - public CfNode visitNameInferredType( @NotNull CuneiformParser.NameInferredTypeContext ctx ) { - return new NameExpr( ctx.ID().getText(), null ); - } - - @Override - public CfNode visitNamePlainFnType( @NotNull CuneiformParser.NamePlainFnTypeContext ctx ) { - return new NameExpr( ctx.ID().getText(), new LambdaType() ); - } - - @Override - public CfNode visitNativeDefTask( @NotNull CuneiformParser.NativeDefTaskContext ctx ) { - // sorted out in the pre-phase - throw new RuntimeException( "Illegal native task definition exncountered." ); - } - - @Override - public CfNode visitNativeLambdaExpr( @NotNull CuneiformParser.NativeLambdaExprContext ctx ) { - - CfNode prototype; - CfNode block; - - prototype = visit( ctx.prototype() ); - - if( prototype == null ) - throw new NullPointerException( "Prototype must not be null." ); - - if( !( prototype instanceof Prototype ) ) - throw new RuntimeException( "Prototype expected." ); - - block = visit( ctx.block() ); - - if( block == null ) - throw new NullPointerException( "Symbol table must not be null." ); - - if( !( block instanceof Block ) ) - throw new RuntimeException( "Symbol table expected." ); - - return new NativeLambdaExpr( ( Prototype )prototype, ( Block )block ); - } - - - @Override - public CfNode visitNilExpr( @NotNull CuneiformParser.NilExprContext ctx ) { - return new CompoundExpr(); - } - - @Override - public CfNode visitPrototype( @NotNull CuneiformParser.PrototypeContext ctx ) { - - CfNode node; - Prototype prototype; - - prototype = new Prototype(); - - for( OutputContext oc : ctx.output() ) { - - node = visit( oc ); - - if( node == null ) - throw new NullPointerException( "Output must not be null." ); - - if( !( node instanceof NameExpr ) ) - throw new RuntimeException( "Output expected." ); - - prototype.addOutput( ( NameExpr )node ); - } - - for( ParamContext pc : ctx.param() ) { - - node = visit( pc ); - - if( node == null ) - throw new NullPointerException( "Parameter must not be null." ); - - if( !( node instanceof Param ) ) - throw new RuntimeException( "Parameter expected." ); - - prototype.addParam( ( Param )node ); - } - - return prototype; - } - - @Override - public CfNode visitReduceVar( @NotNull CuneiformParser.ReduceVarContext ctx ) { - - CfNode nameExpr; - - nameExpr = visit( ctx.name() ); - - if( nameExpr == null ) - throw new NullPointerException( "Name expression must not be null." ); - - if( !( nameExpr instanceof NameExpr ) ) - throw new RuntimeException( "Name expression expected." ); - - return ( ( NameExpr )nameExpr ).toReduceVar(); - } - - @Override - public CfNode visitScript( @NotNull CuneiformParser.ScriptContext ctx ) { - - getTopLevelContext().clearTargetList(); - - visitChildren( ctx ); - return currentBlock; - } - - @Override - public CfNode visitStringExpr( @NotNull CuneiformParser.StringExprContext ctx ) { - - String content; - - content = ctx.getText(); - - if( content.startsWith( "\"" ) ) - content = content.replace( "\\\"", "\"" ); - else - content = content.replace( "\\'", "'" ); - - content = content.substring( 1, content.length()-1 ); - content = content.replace( "\\\\", "\\" ); - - return new StringExpr( content ); - } - - - @Override - public CfNode visitTarget( @NotNull CuneiformParser.TargetContext ctx ) { - - CfNode node; - - node = visit( ctx.expr() ); - - if( node == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - if( !( node instanceof CompoundExpr ) ) - throw new RuntimeException( "Expected compound expression." ); - - if( currentBlock == null ) - throw new NullPointerException( "Current symbol table must not be null." ); - - if( !( currentBlock instanceof TopLevelContext ) ) - throw new RuntimeException( "Block expected." ); - - ( ( TopLevelContext )currentBlock ).addTarget( ( CompoundExpr )node ); - - return node; - } - - private void pushIntoBlock( Block block ) { - - blockStack.push( currentBlock ); - currentBlock = block; - } - - private void popBlock() { - currentBlock = blockStack.pop(); - } - - public TopLevelContext getTopLevelContext() { - - if( !( currentBlock instanceof TopLevelContext ) ) - throw new RuntimeException( "Not in top level context." ); - - return ( TopLevelContext )currentBlock; - } - - public static TopLevelContext process( String input ) { - - ANTLRInputStream instream; - CuneiformLexer lexer; - CommonTokenStream tokenStream; - CuneiformParser parser; - ParseTree tree; - CfNode node; - CfSemanticModelVisitor calculusVisitor; - - // parse original content - instream = new ANTLRInputStream( input ); - - lexer = new CuneiformLexer( instream ); - lexer.removeErrorListeners(); - - tokenStream = new CommonTokenStream( lexer ); - - parser = new CuneiformParser( tokenStream ); - parser.removeErrorListeners(); - - calculusVisitor = new CfSemanticModelVisitor(); - lexer.addErrorListener( calculusVisitor ); - parser.addErrorListener( calculusVisitor ); - - tree = parser.script(); - - node = calculusVisitor.visit( tree ); - - if( node == null ) - throw new NullPointerException( "Node must not be null." ); - - if( !( node instanceof TopLevelContext ) ) - throw new RuntimeException( "Top level context expected." ); - - return ( TopLevelContext )node; - } - - @Override - public void syntaxError(Recognizer recognizer, - Object offendingSymbol, int line, int charPositionInLine, - String msg, RecognitionException e) { - throw new ParseException( line, charPositionInLine, ( ( Token )offendingSymbol ).getText(), msg ); - - } - - @Override - public void reportAmbiguity( Parser arg0, DFA arg1, int arg2, int arg3, - boolean arg4, BitSet arg5, ATNConfigSet arg6 ) { - - if( log.isDebugEnabled() ) - log.debug( "Ambiguity detected." ); - - } - - @Override - public void reportAttemptingFullContext( Parser arg0, DFA arg1, int arg2, - int arg3, BitSet arg4, ATNConfigSet arg5 ) { - - if( log.isTraceEnabled() ) - log.trace( "Attempting full context." ); - - } - - @Override - public void reportContextSensitivity( Parser arg0, DFA arg1, int arg2, - int arg3, int arg4, ATNConfigSet arg5) { - - if( log.isTraceEnabled() ) - log.trace( "Context sensitivity detected." ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CompoundExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CompoundExpr.java deleted file mode 100644 index 0643219..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CompoundExpr.java +++ /dev/null @@ -1,327 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public class CompoundExpr implements CfNode { - - private List singleExprList; - - public CompoundExpr() { - singleExprList = new ArrayList<>(); - } - - public CompoundExpr( SingleExpr se ) { - this(); - addSingleExpr( se ); - } - - public CompoundExpr( CompoundExpr template ) { - - this(); - - if( template == null ) - throw new NullPointerException( "Template compound expression must not be null." ); - - for( SingleExpr se : template.singleExprList ) - singleExprList.add( copySingleExpr( se ) ); - } - - public void addCompoundExpr( CompoundExpr ce ) { - - if( ce == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - singleExprList.addAll( ce.getSingleExprList() ); - } - - public void addSingleExpr( SingleExpr singleExpr ) { - - if( singleExpr == null ) - throw new NullPointerException( "Single expression must not be null." ); - - singleExprList.add( singleExpr ); - } - - public int getNumAtom() throws NotDerivableException { - - int n; - - n = 0; - - for( SingleExpr se : singleExprList ) - n += se.getNumAtom(); - - return n; - } - - public int getNumSingleExpr() { - return singleExprList.size(); - } - - public SingleExpr getSingleExpr( int i ) { - return singleExprList.get( i ); - } - - public List getSingleExprList() { - return Collections.unmodifiableList( singleExprList ); - } - - public StringExpr getStringExprValue( int idx ) throws NotDerivableException { - - int i, j, k; - - if( idx < 0 ) - throw new IndexOutOfBoundsException( - "Queried entry "+idx+". Negative indexes are not allowed." ); - - if( idx >= getNumAtom() ) - throw new IndexOutOfBoundsException( - "Queried entry "+idx+" but the compound expression has only " - +getNumAtom()+" elements: "+this ); - - j = 0; - i = idx; - - while( true ) { - - k = i-getSingleExpr( j ).getNumAtom(); - - if( k < 0 ) - return getSingleExpr( j ).getStringExprValue( i ); - - j++; - i = k; - } - } - - public boolean isNormal() { - - QualifiedTicket qt; - - for( SingleExpr se : singleExprList ) { - - if( se instanceof StringExpr ) - continue; - - if( se instanceof LambdaExpr ) - continue; - - if( !( se instanceof QualifiedTicket ) ) - return false; - - qt = ( QualifiedTicket )se; - - try { - if( !qt.getOutputValue().isNormal() ) - return false; - } catch( NotDerivableException e ) { - return false; - } - } - - return true; - } - - public List normalize() throws NotDerivableException { - - List normalList; - QualifiedTicket qt; - - normalList = new ArrayList<>(); - - for( SingleExpr se : singleExprList ) { - - if( se instanceof StringExpr ) { - - normalList.add( ( ( StringExpr )se ).getContent() ); - continue; - } - - if( se instanceof QualifiedTicket ) { - - qt = ( QualifiedTicket )se; - normalList.addAll( qt.getOutputValue().normalize() ); - continue; - } - - throw new NotDerivableException( "Non-normalizable expression encountered." ); - } - - return normalList; - } - - public void pushRest( Block restBlock ) { - - ApplyExpr applyExpr; - - for( SingleExpr se : singleExprList ) - - if( se instanceof ApplyExpr ) { - - applyExpr = ( ApplyExpr )se; - applyExpr.pushRest( restBlock ); - } - } - - public void remove( SingleExpr se ) { - if( !singleExprList.remove( se ) ) - throw new RuntimeException( "Single expression is no member of this compound expression." ); - } - - @Override - public String toString() { - - StringBuffer buf; - boolean comma; - - try { - - if( this.getNumAtom() == 0 ) - return "nil"; - - buf = new StringBuffer(); - - comma = false; - for( SingleExpr singleExpr : singleExprList ) { - - if( singleExpr.getNumAtom() == 0 ) - continue; - - if( comma ) - buf.append( ' ' ); - - comma = true; - - buf.append( singleExpr ); - } - - return buf.toString(); - } - catch( NotDerivableException e ) { - - if( singleExprList.isEmpty() ) - return "nil"; - - buf = new StringBuffer(); - - comma = false; - for( SingleExpr singleExpr : singleExprList ) { - - if( comma ) - buf.append( ' ' ); - - comma = true; - - buf.append( singleExpr ); - } - - return buf.toString(); - } - } - - @Override - public boolean equals( Object other ) { - - CompoundExpr ce; - int i, n; - - if( other == null ) - return false; - - if( !( other instanceof CompoundExpr ) ) - return false; - - ce = ( CompoundExpr )other; - - if( getNumSingleExpr() != ce.getNumSingleExpr() ) - return false; - - n = ce.getNumSingleExpr(); - for( i = 0; i < n; i++ ) - if( !ce.getSingleExpr( i ).equals( getSingleExpr( i ) ) ) - return false; - - - return true; - } - - @Override - public T visit(NodeVisitor visitor) throws HasFailedException, NotBoundException { - return visitor.accept(this); - } - - @Override - public int hashCode() { - - if( getNumSingleExpr() == 0 ) - return 0; - - return getSingleExpr( 0 ).hashCode(); - } - - private static SingleExpr copySingleExpr( SingleExpr se ) { - - if( se == null ) - throw new IllegalArgumentException( "Single expression to be copied must not be null." ); - - if( se instanceof StringExpr ) - return se; - - if( se instanceof ApplyExpr ) - return new ApplyExpr( ( ApplyExpr )se ); - - if( se instanceof NativeLambdaExpr ) - return new NativeLambdaExpr( ( NativeLambdaExpr )se ); - - if( se instanceof NameExpr ) - return se; - - if( se instanceof ForeignLambdaExpr ) - return new ForeignLambdaExpr( ( ForeignLambdaExpr )se ); - - if( se instanceof CondExpr ) - return new CondExpr( ( CondExpr )se ); - - if( se instanceof QualifiedTicket ) - return se; - - if( se instanceof CurryExpr ) - return new CurryExpr( ( CurryExpr )se ); - - throw new UnsupportedOperationException( "Copy operation unsupported for expression of type "+se.getClass() ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CondExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CondExpr.java deleted file mode 100644 index 2346162..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CondExpr.java +++ /dev/null @@ -1,111 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - -public class CondExpr implements SingleExpr { - - private final CompoundExpr ifExpr; - private final CompoundExpr thenExpr; - private final CompoundExpr elseExpr; - - public CondExpr( CompoundExpr ifExpr, CompoundExpr thenBlock, CompoundExpr elseBlock ) { - - if( thenBlock == null ) - throw new IllegalArgumentException( "Then expression must not be null." ); - - if( elseBlock == null ) - throw new IllegalArgumentException( "Else expression must not be null." ); - - if( ifExpr == null ) - throw new IllegalArgumentException( "Condition expression must not be null." ); - - this.ifExpr = ifExpr; - this.elseExpr = elseBlock; - this.thenExpr = thenBlock; - - } - - public CondExpr(CondExpr template ) { - - if( template == null ) - throw new IllegalArgumentException( "Template conditional expression must not be null." ); - - ifExpr = new CompoundExpr( template.ifExpr ); - thenExpr = new CompoundExpr( template.thenExpr ); - elseExpr = new CompoundExpr( template.elseExpr ); - } - - - public CompoundExpr getElseExpr() { - return elseExpr; - } - - public CompoundExpr getIfExpr() { - return ifExpr; - } - - public CompoundExpr getThenExpr() { - return thenExpr; - } - - @Override - public String toString() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( "if " ); - buf.append( ifExpr ); - buf.append( " then " ).append( thenExpr ); - buf.append( " else " ).append( elseExpr ).append( " end" ); - - return buf.toString(); - } - - @Override - public int getNumAtom() throws NotDerivableException { - throw new NotDerivableException( "Cannot derive cardinality of reduce output variable." ); - } - - @Override - public T visit( NodeVisitor visitor ) throws HasFailedException, NotBoundException { - return visitor.accept( this ); - } - - @Override - public StringExpr getStringExprValue( int i ) throws NotDerivableException { - throw new NotDerivableException( "Cannot derive value for conditional expression." ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CorrelParam.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CorrelParam.java deleted file mode 100644 index 88092f7..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CorrelParam.java +++ /dev/null @@ -1,135 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -public class CorrelParam implements Param { - - private final Set nameExprSet; - - public CorrelParam() { - nameExprSet = new HashSet<>(); - } - - public CorrelParam( CorrelParam template ) { - - this(); - nameExprSet.addAll( template.nameExprSet ); - } - - public void addName( NameExpr nameExpr ) { - - if( nameExpr == null ) - throw new NullPointerException( "Name expression must not be null." ); - - nameExprSet.add( nameExpr ); - } - - public boolean contains( NameExpr nameExpr ) { - return nameExprSet.contains( nameExpr ); - } - - public boolean contains( String id ) { - - for( NameExpr ne : nameExprSet ) - if( ne.getId().equals( id ) ) - return true; - - return false; - } - - @Override - public int getNumParam() { - return nameExprSet.size(); - } - - public NameExpr getLastNameExpr() { - - for( NameExpr nameExpr : nameExprSet ) - return nameExpr; - - throw new RuntimeException( "Name expression set must not be empty." ); - } - - @Override - public Set getNameExprSet() { - - // the parser already prevents that situation - if( nameExprSet.isEmpty() ) - throw new RuntimeException( "Empty correlated parameter list." ); - - if( nameExprSet.size() == 1 ) - throw new RuntimeException( "Correlated parameter list must have at least 2 entries." ); - - return Collections.unmodifiableSet( nameExprSet ); - } - - public void remove( NameExpr nameExpr ) { - if( !nameExprSet.remove( nameExpr ) ) - throw new RuntimeException( "Name expression '"+nameExpr+"' not member of correlated parameter." ); - } - - @Override - public String toString() { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( '[' ); - - comma = false; - for( NameExpr name : nameExprSet ) { - - if( comma ) - buf.append( ' ' ); - comma = true; - - buf.append( name ); - } - - buf.append( ']' ); - - return buf.toString(); - } - - - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CurryExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CurryExpr.java deleted file mode 100644 index 23ca610..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CurryExpr.java +++ /dev/null @@ -1,119 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - -public class CurryExpr extends BaseBlock implements SingleExpr { - - private CompoundExpr taskExpr; - - public CurryExpr() { - // do nothing - } - - public CurryExpr( CurryExpr se ) { - - if( se == null ) - throw new NullPointerException( "Task expression must not be null." ); - - taskExpr = new CompoundExpr( se.getTaskExpr() ); - } - - @Override - public int getNumAtom() throws NotDerivableException { - return 1; - } - - public CompoundExpr getTaskExpr() { - - if( taskExpr == null ) - throw new NullPointerException( "Task expression never set." ); - - return taskExpr; - } - - public boolean hasTaskExpr() { - return taskExpr != null; - } - - - public void setTaskExpr( CompoundExpr taskExpr ) { - - if( taskExpr == null ) - throw new NullPointerException( "Task expression must not be null." ); - - this.taskExpr = taskExpr; - } - - @Override - public String toString() { - - StringBuffer buf; - boolean comma; - - buf = new StringBuffer(); - - buf.append( "curry( task: " ).append( taskExpr ).append( ',' ); - - try { - - comma = false; - for( NameExpr nameExpr : getNameSet() ) { - - if( comma ) - buf.append( ',' ); - comma = true; - - buf.append( ' ' ).append( nameExpr.getId() ).append( ": " ).append( getExpr( nameExpr ) ); - } - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - - buf.append( " )" ); - - return buf.toString(); - } - - @Override - public T visit(NodeVisitor visitor) throws HasFailedException, NotBoundException { - return visitor.accept( this ); - } - - @Override - public StringExpr getStringExprValue(int i) throws NotDerivableException { - throw new NotDerivableException( "Cannot derive value of curry expression." ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DataType.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DataType.java deleted file mode 100644 index 77634a1..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DataType.java +++ /dev/null @@ -1,65 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class DataType implements Type { - - public static final String LABEL_FILE = "File"; - - private final String id; - - public DataType( String id ) { - - if( id == null ) - throw new NullPointerException( "Id string must not be null." ); - - if( id.isEmpty() ) - throw new RuntimeException( "Id string must not be empty." ); - - this.id = id; - } - - public String getId() { - return id; - } - - @Override - public String toString() { - return "( "+id+" )"; - } - - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DrawHelper.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DrawHelper.java deleted file mode 100644 index 2914e8f..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DrawHelper.java +++ /dev/null @@ -1,66 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class DrawHelper { - - private final CompoundExpr taskExpr; - private final BaseBlock bindingBlock; - private final Prototype prototype; - - public DrawHelper( ApplyExpr applyExpr ) { - this( applyExpr.getTaskExpr(), applyExpr ); - } - - public DrawHelper( CompoundExpr taskExpr, BaseBlock bindingBlock ) { - - if( taskExpr == null ) - throw new NullPointerException( "Task expression must not be null." ); - - if( bindingBlock == null ) - throw new NullPointerException( "Binding block must not be null." ); - - this.taskExpr = taskExpr; - this.prototype = ( ( LambdaExpr )taskExpr.getSingleExpr( 0 ) ).getPrototype(); - this.bindingBlock = bindingBlock; - } - - public CompoundExpr reduceComb() { - - CompoundExpr ce; - - ce = new CompoundExpr(); - // TODO - return ce; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DrawParam.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DrawParam.java deleted file mode 100644 index d33b13e..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/DrawParam.java +++ /dev/null @@ -1,55 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Set; - -public class DrawParam implements Param { - - @Override - public int getNumParam() { - // TODO - throw new UnsupportedOperationException( "NYI" ); - } - - @Override - public Set getNameExprSet() { - // TODO - throw new UnsupportedOperationException( "NYI" ); - } - - @Override - public T visit(NodeVisitor visitor) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/EnumHelper.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/EnumHelper.java deleted file mode 100644 index 7652c9a..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/EnumHelper.java +++ /dev/null @@ -1,235 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Set; - -public class EnumHelper { - - private final Prototype prototype; - private final BaseBlock bindingBlock; - private final CompoundExpr taskExpr; - private final Param[] paramArray; - - public EnumHelper( ApplyExpr applyExpr ) { - this( applyExpr.getTaskExpr(), applyExpr ); - } - - public EnumHelper( CompoundExpr taskExpr, BaseBlock bindingBlock ) { - - Set paramSet; - int i, n; - SingleExpr se; - - if( bindingBlock == null ) - throw new IllegalArgumentException( "Binding block must not be null." ); - - if( taskExpr == null ) - throw new IllegalArgumentException( "Task expression must not be null." ); - - if( taskExpr.getNumSingleExpr() < 1 ) - throw new IllegalArgumentException( "Task expression should not be nil." ); - - se = taskExpr.getSingleExpr( 0 ); - if( !( se instanceof LambdaExpr ) ) - throw new IllegalArgumentException( "Task expression must hold lambda expression." ); - - this.taskExpr = taskExpr; - this.prototype = ( ( LambdaExpr )se ).getPrototype(); - this.bindingBlock = bindingBlock; - - // check if parameter bindings are consistent - for( NameExpr paramName : prototype.getParamNameSet() ) { - - if( paramName.getId().equals( CfSemanticModelVisitor.LABEL_TASK ) ) - continue; - - if( !bindingBlock.containsName( paramName ) ) - throw new SemanticModelException( - bindingBlock.toString(), - "The parameter '"+paramName - +"' is unbound in task application." ); - } - - paramSet = prototype.getParamSet(); - - n = paramSet.size(); - - // impose order on parameter set - paramArray = new Param[ n ]; - i = 0; - for( Param param : paramSet ) - paramArray[ i++ ] = param; - - } - - public int getCardinality() throws NotDerivableException { - - int i; - int c; - - c = 1; - for( i = 0; i < paramArray.length; i++ ) - c *= getParamCardinality( i ); - - return c; - } - - public LambdaExpr getSingularLambdaExpr( int idx ) throws NotDerivableException { - - int k; - Param param; - int c, i, j; - NameExpr taskNameExpr; - SingleExpr se; - - i = idx; - taskNameExpr = new NameExpr( CfSemanticModelVisitor.LABEL_TASK ); - - - for( k = 0; k < paramArray.length; k++ ) { - - param = paramArray[ k ]; - - if( param instanceof ReduceVar ) - continue; - - c = getParamCardinality( k ); - j = i%c; - i /= c; - - if( !param.getNameExprSet().contains( taskNameExpr ) ) - continue; - - se = taskExpr.getSingleExpr( j ); - if( !( se instanceof LambdaExpr ) ) - throw new RuntimeException( "Expected lambda expression." ); - - return ( LambdaExpr )se; - } - - throw new RuntimeException( "Task parameter could not be addressed." ); - } - - - public Block getSingularBindingBlock( int idx ) throws NotDerivableException { - - int i, j, k, c; - Block block; - Param param; - CompoundExpr ce; - - block = new Block(); - i = idx; - - try { - for( k = 0; k < paramArray.length; k++ ) { - - param = paramArray[ k ]; - - if( param instanceof ReduceVar ) { - - block.putAssign( - ( ReduceVar )param, - bindingBlock.getExpr( ( ReduceVar )param ) ); - - continue; - } - - c = getParamCardinality( k ); - j = i%c; - i /= c; - - for( NameExpr name : param.getNameExprSet() ) { - - if( name.getId().equals( CfSemanticModelVisitor.LABEL_TASK ) ) - continue; - - ce = bindingBlock.getExpr( name ); - ce = new CompoundExpr( ce.getStringExprValue( j ) ); - - if( ce.getNumAtom() != 1 ) - throw new RuntimeException( "Enumeration resulted in non-singular expression: "+ce ); - - block.putAssign( name, ce ); - } - } - - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - return block; - } - - private int getParamCardinality( int i ) throws NotDerivableException { - - Param param; - int n; - CompoundExpr ce; - - - try { - - param = paramArray[ i ]; - - if( param instanceof ReduceVar ) - return 1; - - for( NameExpr name : param.getNameExprSet() ) { - - - if( name.getId().equals( CfSemanticModelVisitor.LABEL_TASK ) ) - return taskExpr.getNumAtom(); - - ce = bindingBlock.getExpr( name ); - - n = ce.getNumAtom(); - - return n; - } - } - catch( NotBoundException e ) { - - System.err.println( "[task]" ); - System.err.println( taskExpr ); - System.err.println( "[binding]" ); - System.err.println( bindingBlock ); - System.err.println( "[end]" ); - throw new RuntimeException( e.getMessage() ); - } - - throw new RuntimeException( "Parameter must at least contain one parameter name." ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ForeignLambdaExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ForeignLambdaExpr.java deleted file mode 100644 index 13edcc3..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ForeignLambdaExpr.java +++ /dev/null @@ -1,163 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class ForeignLambdaExpr extends LambdaExpr { - - public static final String LANGID_BASH = "bash"; - public static final String LANGID_LISP = "lisp"; - public static final String LANGID_OCTAVE = "octave"; - public static final String LANGID_MATLAB = "matlab"; - public static final String LANGID_R = "r"; - public static final String LANGID_PYTHON = "python"; - public static final String LANGID_PERL = "perl"; - public static final String LANGID_JAVA = "java"; - public static final String LANGID_SCALA = "scala"; - public static final String LANGID_ERLANG = "erlang"; - public static final String LANGID_HASKELL = "haskell"; - public static final String LANGID_PEGASUS = "pegasus"; - public static final String LANGID_BEANSHELL = "beanshell"; - - private static final String[] LABEL_LANG = { - LANGID_BASH, LANGID_LISP, LANGID_OCTAVE, LANGID_MATLAB, LANGID_R, - LANGID_PYTHON, LANGID_PERL, LANGID_JAVA, LANGID_SCALA, LANGID_ERLANG, - LANGID_HASKELL, LANGID_PEGASUS, LANGID_BEANSHELL - }; - - private final int lang; - private final String body; - private String name; - - public ForeignLambdaExpr( Prototype prototype, String langLabel, String body ) { - this( prototype, labelToInt( langLabel ), body ); - } - - public ForeignLambdaExpr( Prototype prototype, int lang, String body ) { - - super( prototype ); - - if( body == null ) - throw new NullPointerException( "Body must not be null." ); - - if( body.isEmpty() ) - throw new RuntimeException( "Body must not be empty." ); - - if( lang < 0 || lang >= LABEL_LANG.length ) - throw new RuntimeException( "Language id "+lang+" not recognized. Must be a number in [0,"+( LABEL_LANG.length-1 )+"]." ); - - this.body = body; - this.lang = lang; - } - - public ForeignLambdaExpr(ForeignLambdaExpr template ) { - - super( template ); - - if( template == null ) - throw new IllegalArgumentException( "Template foreign lambda expression must not be null." ); - - lang = template.lang; - body = template.body; - name = template.name; - } - - public String getBody() { - return body; - } - - public int getLang() { - return lang; - } - - public String getLangLabel() { - return LABEL_LANG[ lang ]; - } - - public long getLambdaId() { - - long h; - - h = HashHelper.add( getPrototype().getPrototypeId(), lang ); - h = HashHelper.add( h, body ); - - return h; - } - - public String getTaskName() { - return name; - } - - public boolean hasTaskName() { - return name != null; - } - - public void setOptionalTaskName( String name ) { - - if( name == null ) - throw new NullPointerException( "Task name must not be null." ); - - if( name.isEmpty() ) - throw new RuntimeException( "Task name must not be empty." ); - - this.name = name; - } - - @Override - public String toString() { - return super.toString()+"in "+intToLabel( lang )+" *{"+body+"}*"; - } - - public static String intToLabel( int i ) { - return LABEL_LANG[ i ]; - } - - public static int labelToInt( String label ) { - - int i, n; - String l; - - l = label.toLowerCase(); - - n = LABEL_LANG.length; - for( i = 0; i < n; i++ ) - if( LABEL_LANG[ i ].equals( l ) ) - return i; - - throw new SemanticModelException( label, "Language '"+label+"' not recognized." ); - } - - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HasFailedException.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HasFailedException.java deleted file mode 100644 index 9fd863d..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HasFailedException.java +++ /dev/null @@ -1,43 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class HasFailedException extends Exception { - - private static final long serialVersionUID = -430246437126890792L; - - public HasFailedException( String msg ) { - super( msg ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HashHelper.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HashHelper.java deleted file mode 100644 index d019044..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HashHelper.java +++ /dev/null @@ -1,58 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class HashHelper { - - private static final long PRIME = 99194853094755497L; // large prime number below 9223372036854775807 - - private static int abs( int x ) { - - if( x < 0 ) - return -x; - - return x; - } - - public static long add( long a, Object b ) { - - long hash; - - hash = a<<2; - hash = hash%PRIME; - hash = ( hash+( abs( b.hashCode() ) ) )%PRIME; - - return hash; - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/JsonReportEntry.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/JsonReportEntry.java deleted file mode 100644 index 3dd7e43..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/JsonReportEntry.java +++ /dev/null @@ -1,465 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.UUID; - -import org.json.JSONException; -import org.json.JSONObject; - -public class JsonReportEntry { - - public static final String KEY_INVOC_TIME = "invoc-time"; - public static final String KEY_FILE_SIZE_STAGEIN = "file-size-stagein"; - public static final String KEY_FILE_SIZE_STAGEOUT = "file-size-stageout"; - public static final String KEY_INVOC_OUTPUT = "invoc-output"; - public static final String KEY_INVOC_STDOUT = "invoc-stdout"; - public static final String KEY_INVOC_STDERR = "invoc-stderr"; - public static final String KEY_INVOC_USER = "invoc-user"; - public static final String KEY_INVOC_EXEC = "invoc-exec"; - public static final String KEY_INVOC_SCRIPT = "invoc-script"; - public static final String KEY_REDUCTION_TIME = "reduction-time"; - - public static final String ATT_INVOCID = "invocId"; - public static final String ATT_KEY = "key"; - public static final String ATT_RUNID = "runId"; - public static final String ATT_VALUE = "value"; - public static final String ATT_TASKNAME = "taskname"; - public static final String ATT_TASKID = "taskId"; - public static final String ATT_TIMESTAMP = "timestamp"; - public static final String ATT_LANG = "lang"; - public static final String ATT_FILE = "file"; - - public static final String LABEL_REALTIME = "realTime"; - - private UUID runId; - private Long invocId; - private String key; - private String value; - private String taskname; - private Long taskId; - private long timestamp; - private String lang; - private String file; - - public JsonReportEntry( - long timestamp, UUID runId, Long taskId, String taskname, - String lang, Long invocId, String file, String key, String rawValue ) { - - setTimestamp( timestamp ); - setRunId( runId ); - setTaskId( taskId ); - setTaskname( taskname ); - setInvocId( invocId ); - setLang( lang ); - setFile( file ); - setKey( key ); - setValueFromRawString( rawValue ); - } - - public JsonReportEntry( - UUID runId, Long taskId, String taskname, - String lang, Long invocId, String file, String key, String rawValue ) { - - this( System.currentTimeMillis(), runId, taskId, taskname, lang, invocId, file, key, rawValue ); - } - - public JsonReportEntry( - UUID runId, Long taskId, String taskname, - String lang, Long invocId, String key, String rawValue ) { - - this( System.currentTimeMillis(), runId, taskId, taskname, lang, invocId, null, key, rawValue ); - } - - public JsonReportEntry( Ticket ticket, String key, String rawValue ) { - this( - ticket.getRunId(), - ticket.getLambdaId(), - ticket.getTaskName(), - ticket.getLangLabel(), - ticket.getTicketId(), - key, - rawValue ); - } - - public JsonReportEntry( - long timestamp, UUID runId, Long taskId, String taskname, - String lang, Long invocId, String file, String key, JSONObject obj ) { - - setTimestamp( timestamp ); - setRunId( runId ); - setTaskId( taskId ); - setTaskname( taskname ); - setInvocId( invocId ); - setLang( lang ); - setFile( file ); - setKey( key ); - setValueFromJsonObj( obj ); - } - - public JsonReportEntry( - UUID runId, Long taskId, String taskname, - String lang, Long invocId, String file, String key, JSONObject obj ) { - this( System.currentTimeMillis(), runId, taskId, taskname, lang, invocId, file, key, obj ); - } - - public JsonReportEntry( String raw ) throws JSONException { - - JSONObject obj, valueObj; - String valueString; - - try { - obj = new JSONObject( raw.replace( "\0", "" ) ); - - - setTimestamp( obj.getLong( ATT_TIMESTAMP ) ); - setRunId( UUID.fromString( obj.getString( ATT_RUNID ) ) ); - - if( obj.has( ATT_TASKID ) ) - if( !obj.isNull( ATT_TASKID ) ) - setTaskId( obj.getLong( ATT_TASKID ) ); - - if( obj.has( ATT_TASKNAME ) ) - if( !obj.isNull( ATT_TASKNAME ) ) - setTaskname( obj.getString( ATT_TASKNAME ) ); - - if( obj.has( ATT_LANG ) ) - if( !obj.isNull( ATT_LANG ) ) - setLang( obj.getString( ATT_LANG ) ); - - if( obj.has( ATT_INVOCID ) ) - if( !obj.isNull( ATT_INVOCID ) ) - setInvocId( obj.getLong( ATT_INVOCID ) ); - - if( obj.has( ATT_FILE ) ) - if( !obj.isNull( ATT_FILE ) ) - setFile( obj.getString( ATT_FILE ) ); - - setKey( obj.getString( ATT_KEY ) ); - - try { - valueObj = obj.getJSONObject( ATT_VALUE ); - setValueFromJsonObj( valueObj ); - } - catch( JSONException e ) { - valueString = obj.getString( ATT_VALUE ); - setValueFromRawString( valueString ); - } - } - catch( JSONException e ) { - System.err.println( "[raw]" ); - System.err.println( raw ); - throw e; - } - } - - @Override - public boolean equals( Object obj ) { - - JsonReportEntry other; - String s1, s2; - - if( obj == null ) - return false; - - if( !( obj instanceof JsonReportEntry ) ) - return false; - - other = ( JsonReportEntry )obj; - - s1 = toString(); - s2 = other.toString(); - - return s1.equals( s2 ); - } - - public String getFile() { - return file; - } - - public Long getInvocId() { - return invocId; - } - - public String getKey() { - return key.replace( "\\n", "\n" ); - } - - public String getLang() { - return lang; - } - - public UUID getRunId() { - return runId; - } - - public JSONObject getValueJsonObj() throws JSONException { - - if( !isValueJson() ) - throw new RuntimeException( "Value is not a JSON object, but a string." ); - - return new JSONObject( value ); - } - - public String getValueRawString() { - - String s; - - if( !isValueString() ) - throw new RuntimeException( "Value is not a string, but a JSON object." ); - - s = value.substring( 1, value.length()-1 ); - s = s.replace( "\\n", "\n" ); - s = s.replace( "\\\"", "\"" ); - s = s.replace( "\\\\", "\\" ); - - return s; - } - - public String getTaskName() { - return taskname; - } - - public Long getTaskId() { - return taskId; - } - - public long getTimestamp() { - return timestamp; - } - - public boolean isValueString() { - - if( value.startsWith( "\"" ) ) - return true; - - return false; - } - - public boolean hasFile() { - return file != null; - } - - public boolean hasInvocId() { - return invocId != null; - } - - public boolean hasLang() { - return lang != null; - } - - public boolean hasTaskId() { - return taskId != null; - } - - public boolean hasTaskname() { - return taskname != null; - } - - public boolean isKeyInvocStdErr() { - return key.equals( KEY_INVOC_STDERR ); - } - - public boolean isKeyInvocStdOut() { - return key.equals( KEY_INVOC_STDOUT ); - } - - public boolean isKeyInvocTime() { - return key.equals( KEY_INVOC_TIME ); - } - - public boolean isKeyInvocOutput() { - return key.equals( KEY_INVOC_OUTPUT ); - } - - public boolean isKeyInvocExec() { - return key.equals( KEY_INVOC_EXEC ); - } - - public boolean isKeyFileSizeStageIn() { - return key.equals( KEY_FILE_SIZE_STAGEIN ); - } - - public boolean isKeyFileSizeStageOut() { - return key.equals( KEY_FILE_SIZE_STAGEOUT ); - } - - public boolean isValueJson() { - return !isValueString(); - } - - public void setFile( String file ) { - - if( file == null ) { - this.file = null; - return; - } - - if( file.isEmpty() ) - throw new RuntimeException( "File name must not be empty." ); - - this.file = file; - } - - public void setInvocId( Long invocId ) { - this.invocId = invocId; - } - - public void setKey( String key ) { - - if( key == null ) - throw new NullPointerException( "Key must not be null." ); - - if( key.isEmpty() ) - throw new RuntimeException( "Key must not be empty." ); - - if( key.contains( "\n" ) ) - throw new RuntimeException( "Key must not contain newline character." ); - - if( key.contains( "\"" ) ) - throw new RuntimeException( "Key must not contain double quote character." ); - - if( key.contains( "\\" ) ) - throw new RuntimeException( "Key must not contain backslash character." ); - - this.key = key; - } - - public void setRunId( UUID runId ) { - - if( runId == null ) - throw new NullPointerException( "DAG id must not be null." ); - - this.runId = runId; - } - - public void setValueFromRawString( String value ) { - - if( value == null ) - throw new NullPointerException( "Payload must not be null." ); - - if( value.isEmpty() ) - throw new RuntimeException( "Payload must not be empty." ); - - this.value = "\""+value.replace( "\\", "\\\\" ).replace( "\n", "\\n" ).replace( "\"", "\\\"" )+"\""; - } - - public void setValueFromJsonObj( JSONObject obj ) { - - if( obj == null ) - throw new NullPointerException( "Value JSON object must not be null." ); - - value = obj.toString(); - } - - public void setLang( String lang ) { - - if( lang == null ) { - this.lang = null; - return; - } - - if( lang.isEmpty() ) - throw new RuntimeException( "Language string must not be empty." ); - - this.lang = lang; - } - - public void setTaskId( Long taskId ) { - this.taskId = taskId; - } - - public void setTaskname( String taskname ) { - - if( taskname == null ) { - this.taskname = null; - return; - } - - if( taskname.isEmpty() ) - throw new RuntimeException( "Taskname must not be empty." ); - - if( taskname.contains( "\n" ) ) - throw new RuntimeException( "Taskname id must not contain newline character." ); - - if( taskname.contains( "\"" ) ) - throw new RuntimeException( "Taskname id must not contain double quote character." ); - - if( taskname.contains( "\\" ) ) - throw new RuntimeException( "Taskname must not contain backslash character." ); - - this.taskname = taskname; - } - - public void setTimestamp( long timestamp ) { - this.timestamp = timestamp; - } - - @Override - public String toString() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( '{' ); - buf.append( ATT_TIMESTAMP ).append( ':' ).append( timestamp ).append( ',' ); - buf.append( ATT_RUNID ).append( ":\"" ).append( runId ).append( "\"," ); - - if( hasTaskId() ) - buf.append( ATT_TASKID ).append( ':' ).append( taskId ).append( ',' ); - - if( hasTaskname() ) - buf.append( ATT_TASKNAME ).append( ':' ).append( "\"" ).append( taskname ).append( "\"," ); - - if( hasLang() ) - buf.append( ATT_LANG ).append( ':' ).append( "\"" ).append( lang ).append( "\"," ); - - if( hasInvocId() ) - buf.append( ATT_INVOCID ).append( ':' ).append( invocId ).append( ',' ); - - if( hasFile() ) - buf.append( ATT_FILE ).append( ":\"" ).append( file ).append( "\"," ); - - buf.append( ATT_KEY ).append( ":\"" ).append( key ).append( "\"," ); - buf.append( ATT_VALUE ).append( ':' ).append( value ); - buf.append( '}' ); - - return buf.toString(); - } - - @Override - public int hashCode() { - return toString().hashCode(); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/LambdaExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/LambdaExpr.java deleted file mode 100644 index cf83248..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/LambdaExpr.java +++ /dev/null @@ -1,78 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public abstract class LambdaExpr implements SingleExpr { - - private final Prototype prototype; - - public LambdaExpr( Prototype prototype ) { - - if( prototype == null ) - throw new NullPointerException( "Prototype must not be null." ); - - this.prototype = prototype; - } - - public LambdaExpr( LambdaExpr template ) { - - if( template == null ) - throw new IllegalArgumentException( "Template lambda expression must not be null." ); - - if( template.prototype == null ) - throw new IllegalArgumentException( "Template lambda expression must have non-null prototype." ); - - prototype = new Prototype( template.prototype ); - } - - @Override - public int getNumAtom() throws NotDerivableException { - return 1; - } - - public Prototype getPrototype() { - return prototype; - } - - @Override - public StringExpr getStringExprValue( int i ) throws NotDerivableException { - throw new RuntimeException( "Trying to convert lambda expression value to string expression value." ); - } - - - @Override - public String toString() { - return "\\"+prototype; - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/LambdaType.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/LambdaType.java deleted file mode 100644 index e51539b..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/LambdaType.java +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class LambdaType implements Type, Cloneable { - - @Override - public String toString() { - return "(:)"; - } - - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NameExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NameExpr.java deleted file mode 100644 index f2ebf66..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NameExpr.java +++ /dev/null @@ -1,142 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.HashSet; -import java.util.Set; - - -public class NameExpr implements SingleExpr, Param, Comparable { - - private final String id; - private final Type type; - - public NameExpr( String id ) { - this( id, null ); - } - - public NameExpr( String id, Type type ) { - - if( id == null ) - throw new NullPointerException( "Id string must not be null." ); - - if( id.isEmpty() ) - throw new RuntimeException( "Id string must not be empty." ); - - this.id = id; - this.type = type; - } - - @Override - public boolean equals( Object obj ) { - - NameExpr other; - - if( !( obj instanceof NameExpr ) ) - return false; - - other = ( NameExpr )obj; - return id.equals( other.getId() ); - } - - public String getId() { - return id; - } - - @Override - public int getNumParam() { - return 1; - } - - public Type getType() { - return type; - } - - @Override - public int hashCode() { - return id.hashCode(); - } - - public boolean hasType() { - return type != null; - } - - public ReduceVar toReduceVar() { - return new ReduceVar( id, type ); - } - - @Override - public String toString() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( id ); - - if( type != null ) - buf.append( type ); - - return buf.toString(); - } - - @Override - public Set getNameExprSet() { - - Set nameSet; - - nameSet = new HashSet<>(); - nameSet.add( this ); - - return nameSet; - } - - @Override - public int getNumAtom() throws NotDerivableException { - throw new NotDerivableException( "Name expression cannot hold size information." ); - } - - @Override - public int compareTo( NameExpr arg0 ) { - return arg0.getId().compareTo( id ); - } - @Override - public T visit( NodeVisitor visitor ) throws HasFailedException, NotBoundException { - return visitor.accept( this ); - } - - @Override - public StringExpr getStringExprValue( int i ) throws NotDerivableException { - throw new NotDerivableException( "Cannot derive value from name expression." ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NativeLambdaExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NativeLambdaExpr.java deleted file mode 100644 index a964654..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NativeLambdaExpr.java +++ /dev/null @@ -1,84 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class NativeLambdaExpr extends LambdaExpr { - - private final Block bodyBlock; - - public NativeLambdaExpr( Prototype prototype, Block bodyBlock ) { - - super( prototype ); - - if( bodyBlock == null ) - throw new NullPointerException( "Body block must not be null." ); - - this.bodyBlock = bodyBlock; - } - - public NativeLambdaExpr( NativeLambdaExpr template ) { - - super( template ); - - CompoundExpr value, copiedValue; - - bodyBlock = new Block( template.bodyBlock.getParent() ); - - try { - for( NameExpr ne : template.bodyBlock.getNameSet() ) { - - value = template.getBodyBlock().getExpr( ne ); - copiedValue = new CompoundExpr( value ); - - bodyBlock.putAssign( ne, copiedValue ); - } - - } catch (NotBoundException e) { - throw new RuntimeException( e ); - } - - } - - public Block getBodyBlock() { - return bodyBlock; - } - - @Override - public String toString() { - return super.toString()+" {\n"+bodyBlock+"}"; - } - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NodeVisitor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NodeVisitor.java deleted file mode 100644 index 0f1994d..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NodeVisitor.java +++ /dev/null @@ -1,61 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - -/** - * @author jorgen - * - */ -public interface NodeVisitor { - - public T accept( StringExpr stringExpr ); - public T accept( QualifiedTicket qualifiedTicket ); - public T accept( NativeLambdaExpr nativeLambdaExpr ); - public T accept( NameExpr nameExpr ) throws HasFailedException, NotBoundException; - public T accept( LambdaType lambdaType ); - public T accept( DrawParam drawParam ); - public T accept( DataType dataType ); - public T accept( CondExpr condExpr ) throws HasFailedException, NotBoundException; - public T accept( Block block ); - public T accept( ApplyExpr applyExpr ) throws HasFailedException, NotBoundException; - public T accept( CompoundExpr compoundExpr ) throws HasFailedException, NotBoundException; - public T accept( CorrelParam correlParam ); - public T accept( ForeignLambdaExpr foreignLambdaExpr ); - public T accept( CurryExpr curryExpr ) throws HasFailedException, NotBoundException; - public T accept( Prototype prototype ); - public T accept( ReduceVar reduceVar ); - public T accept(TopLevelContext topLevelContext) throws HasFailedException, NotBoundException; - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NotBoundException.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NotBoundException.java deleted file mode 100644 index 2af52cd..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NotBoundException.java +++ /dev/null @@ -1,42 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class NotBoundException extends Exception { - - private static final long serialVersionUID = -9001488787382738222L; - - public NotBoundException( String msg ) { - super( msg ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NotDerivableException.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NotDerivableException.java deleted file mode 100644 index c45f6ce..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NotDerivableException.java +++ /dev/null @@ -1,47 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class NotDerivableException extends Exception { - - private static final long serialVersionUID = 7155475670128340863L; - - public NotDerivableException( String msg ) { - super( msg ); - } - - public NotDerivableException( Exception e ) { - super( e ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Param.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Param.java deleted file mode 100644 index 3ad748c..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Param.java +++ /dev/null @@ -1,41 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Set; - -public interface Param extends CfNode { - - public int getNumParam(); - public Set getNameExprSet(); -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Prototype.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Prototype.java deleted file mode 100644 index 77e9386..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Prototype.java +++ /dev/null @@ -1,262 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -public class Prototype extends LambdaType { - - private final List outputList; - private final Set paramSet; - - public Prototype() { - outputList = new LinkedList<>(); - paramSet = new HashSet<>(); - } - - public Prototype( Prototype template ) { - - this(); - - if( template == null ) - throw new IllegalArgumentException( "Template prototype must not be null." ); - - if( template.outputList == null ) - throw new IllegalArgumentException( "Template prototype must not have null output list." ); - - if( template.paramSet == null ) - throw new IllegalArgumentException( "Template prototype must not have null parameter set." ); - - for( NameExpr output : template.outputList ) - outputList.add( output ); - - for( Param param : template.paramSet ) - paramSet.add( copyParam( param ) ); - } - - public void addOutput( NameExpr output ) { - - if( output == null ) - throw new NullPointerException( "Output must not be null." ); - - outputList.add( output ); - } - - public void addOutput( List ol ) { - outputList.addAll( ol ); - } - - public void addParam( Param param ) { - - if( param == null ) - throw new NullPointerException( "Output must not be null." ); - - paramSet.add( param ); - } - - public void addParam( Set ps ) { - paramSet.addAll( ps ); - } - - public NameExpr getOutput( int i ) { - return outputList.get( i ); - } - - public List getOutputList() { - - if( outputList.isEmpty() ) - // parser already checks that - throw new RuntimeException( "Empty output list." ); - - return Collections.unmodifiableList( outputList ); - } - - public Set getParamSet() { - return Collections.unmodifiableSet( paramSet ); - } - - public Set getParamNameSet() { - - Set nameSet; - - nameSet = new HashSet<>(); - - for( Param param : paramSet ) - nameSet.addAll( param.getNameExprSet() ); - - return nameSet; - } - - public Set getNonReduceParamNameSet() { - - Set set; - - set = new HashSet<>(); - - for( Param param : paramSet ) - if( !( param instanceof ReduceVar ) ) - set.addAll( param.getNameExprSet() ); - - return set; - } - - public boolean isTaskCorrelated() { - - for( Param param : paramSet ) - if( param instanceof CorrelParam ) - if( ( ( CorrelParam )param ).contains( - CfSemanticModelVisitor.LABEL_TASK ) ) - - return true; - - return false; - } - - public boolean containsParam( String paramId ) { - return containsParam( new NameExpr( paramId ) ); - } - - public boolean containsParam( NameExpr paramNameExpr ) { - return getParamNameSet().contains( paramNameExpr ); - } - - public int getNumOutput() { - return outputList.size(); - } - - public int getNumParam() { - - int n; - - n = 0; - for( Param param : paramSet ) - n += param.getNumParam(); - - return n; - } - - public long getPrototypeId() { - - long h; - - h = 0; - - for( NameExpr output : outputList ) - h = HashHelper.add( h, output ); - - for( NameExpr param : getParamNameSet() ) - h = HashHelper.add( h, param ); - - return h; - } - - public void removeParam( String name ) { - removeParam( new NameExpr( name ) ); - } - - public void removeParam( NameExpr nameExpr ) { - - CorrelParam correlParam; - int n; - - for( Param param : paramSet ) - if( param instanceof NameExpr ) { - if( nameExpr.equals( param ) ) { - paramSet.remove( nameExpr ); - return; - } - } - else if( param instanceof CorrelParam ) { - - correlParam = ( CorrelParam )param; - - if( correlParam.contains( nameExpr ) ) { - - correlParam.remove( nameExpr ); - - n = correlParam.getNumParam(); - - if( n > 1 ) - return; - - if( n == 1 ) { - - paramSet.remove( correlParam ); - paramSet.add( correlParam.getLastNameExpr() ); - - return; - } - - throw new RuntimeException( - "An empty correlated parameter set should never occur." ); - } - } - - throw new RuntimeException( "Name expression '"+nameExpr+"' not member of parameter set." ); - } - - @Override - public String toString() { - - StringBuffer buf; - - buf = new StringBuffer( "( " ); - - for( NameExpr output : outputList ) - buf.append( output ).append( ' ' ); - - buf.append( ": " ); - - for( Param param : paramSet ) - buf.append( param ).append( ' ' ); - - buf.append( ')' ); - - return buf.toString(); - } - - private static Param copyParam( Param template ) { - - if( template instanceof NameExpr ) - return template; - - if( template instanceof CorrelParam ) - return new CorrelParam( ( CorrelParam )template ); - - throw new UnsupportedOperationException( "Copy operation not supported for parameter of type "+template.getClass() ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/QualifiedTicket.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/QualifiedTicket.java deleted file mode 100644 index a8014e7..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/QualifiedTicket.java +++ /dev/null @@ -1,106 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Set; - -public class QualifiedTicket implements SingleExpr { - - private final Ticket ticket; - private final int channel; - - public QualifiedTicket( Ticket ticket, int channel ) { - - if( channel < 1 ) - throw new RuntimeException( "Channel must at least be 1." ); - - if( ticket == null ) - throw new NullPointerException( "Ticket must not be null." ); - - this.ticket = ticket; - this.channel = channel; - } - - public int getChannel() { - return channel; - } - - public Ticket getTicket() { - return ticket; - } - - public Set getParamNameSet() { - return ticket.getParamNameSet(); - } - - public CompoundExpr getParamValue( NameExpr name ) throws NotBoundException { - return ticket.getExpr( name ); - } - - public CompoundExpr getOutputValue() throws NotDerivableException { - try { - return ticket.getOutputValue( channel ); - } catch( NotBoundException e ) { - throw new NotDerivableException( e ); - } - } - - @Override - public int getNumAtom() throws NotDerivableException { - return ticket.getNumAtom( channel ); - } - - - @Override - public String toString() { - - try { - return getOutputValue().toString(); - } - catch( NotDerivableException e ) { - return "["+channel+"]"+ticket; - } - - - } - - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } - - @Override - public StringExpr getStringExprValue( int i ) throws NotDerivableException { - return getOutputValue().getStringExprValue( i ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ReduceVar.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ReduceVar.java deleted file mode 100644 index 6665ad4..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ReduceVar.java +++ /dev/null @@ -1,46 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - - -public class ReduceVar extends NameExpr { - - public ReduceVar( String id, Type type ) { - super( id, type ); - } - - @Override - public String toString() { - return "<"+super.toString()+">"; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/SemanticModelException.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/SemanticModelException.java deleted file mode 100644 index 81be366..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/SemanticModelException.java +++ /dev/null @@ -1,52 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import de.huberlin.wbi.cuneiform.core.preprocess.ParseException; - - -public class SemanticModelException extends ParseException { - - private static final long serialVersionUID = -826451567187918855L; - - public SemanticModelException( String near, - String msg ) { - super( null, null, near, msg ); - } - - public SemanticModelException( CfNode node, - String msg ) { - this( node.toString(), msg ); - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/SingleExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/SingleExpr.java deleted file mode 100644 index 108944f..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/SingleExpr.java +++ /dev/null @@ -1,39 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public interface SingleExpr extends CfNode { - - public int getNumAtom()throws NotDerivableException; - public StringExpr getStringExprValue( int i )throws NotDerivableException; -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/StringExpr.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/StringExpr.java deleted file mode 100644 index a58a232..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/StringExpr.java +++ /dev/null @@ -1,97 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class StringExpr implements SingleExpr { - - private final String value; - - public StringExpr( String content ) { - - if( content == null ) - throw new NullPointerException( "Content string must not be null." ); - - this.value = content; - } - - @Override - public boolean equals( Object obj ) { - - StringExpr other; - - if( !( obj instanceof StringExpr ) ) - return false; - - other = ( StringExpr )obj; - - return other.getContent().equals( value ); - - } - - public String getContent() { - return value; - } - - @Override - public StringExpr getStringExprValue( int i ) { - - if( i != 0 ) - throw new IndexOutOfBoundsException( - "String expression has only one value with index 0. Queried index "+i ); - - return this; - } - - @Override - public int getNumAtom() throws NotDerivableException { - return 1; - } - - @Override - public String toString() { - return "'"+value+"'"; - } - - @Override - public int hashCode() { - return value.hashCode(); - } - - /* (non-Javadoc) - * @see de.huberlin.cuneiform.core.semanticmodel.Node#visit(de.huberlin.cuneiform.core.semanticmodel.NodeVisitor) - */ - @Override - public T visit( NodeVisitor visitor ) { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Ticket.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Ticket.java deleted file mode 100644 index f408ab4..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Ticket.java +++ /dev/null @@ -1,247 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.Collections; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; -import java.util.UUID; - -import org.json.JSONException; -import org.json.JSONObject; - - -public class Ticket extends Block { - - private final ForeignLambdaExpr lambdaExpr; - private final Block bodyBlock; - private final UUID runId; - - public Ticket( ForeignLambdaExpr lambdaExpr, BaseBlock bindingBlock, UUID runId ) { - - if( bindingBlock == null ) - throw new NullPointerException( "Binding block must not be null." ); - - if( runId == null ) - throw new NullPointerException( "Run ID must not be null." ); - - if( lambdaExpr == null ) - throw new NullPointerException( - "Foreign lambda expression must not be null." ); - - setParamBindMap( bindingBlock.getParamBindMap() ); - - this.lambdaExpr = lambdaExpr; - this.runId = runId; - - bodyBlock = new Block(); - - if( !isReady() ) - throw new RuntimeException( "Attempted to create non-ready ticket." ); - } - - - public int getNumAtom( int channel ) throws NotDerivableException { - - NameExpr outputVar; - - try { - - outputVar = getPrototype().getOutput( channel-1 ); - - if( outputVar instanceof ReduceVar ) - return bodyBlock.getExpr( getPrototype().getOutput( channel-1 ) ).getNumAtom(); - - return 1; - } - catch( NotBoundException e ) { - throw new NotDerivableException( e.getMessage() ); - } - } - - public String getBody() { - return lambdaExpr.getBody(); - } - - public JsonReportEntry getExecutableLogEntry() { - - JsonReportEntry entry; - JSONObject obj; - - - try { - - obj = new JSONObject(); - obj.put( "lambda", lambdaExpr ); - obj.put( "bind", getParamBindMap() ); - - entry = new JsonReportEntry( - runId, lambdaExpr.getLambdaId(), lambdaExpr.getTaskName(), - lambdaExpr.getLangLabel(), getTicketId(), null, - JsonReportEntry.KEY_INVOC_EXEC, obj ); - - return entry; - - } - catch( JSONException e ) { - throw new RuntimeException( e ); - } - } - - public long getLambdaId() { - return lambdaExpr.getLambdaId(); - } - - public String getLangLabel() { - return lambdaExpr.getLangLabel(); - } - - public List getOutputList() { - return lambdaExpr.getPrototype().getOutputList(); - } - - public CompoundExpr getOutputValue( int channel ) throws NotBoundException { - return bodyBlock.getExpr( lambdaExpr.getPrototype().getOutput( channel-1 ) ); - } - - public CompoundExpr getOutputValue( NameExpr name ) throws NotBoundException { - return bodyBlock.getExpr( name ); - } - - public Set getParamSet() { - return lambdaExpr.getPrototype().getParamSet(); - } - - public Set getParamNameSet() { - return lambdaExpr.getPrototype().getParamNameSet(); - } - - public Prototype getPrototype() { - return lambdaExpr.getPrototype(); - } - - public UUID getRunId() { - return runId; - } - - public String getTaskName() { - return lambdaExpr.getTaskName(); - } - - public long getTicketId() { - - long h; - - h = toString().hashCode(); - if( h < 0 ) - h = -h; - - return h; - - } - - public boolean isNormal() { - - try { - - for( NameExpr name : getNameSet() ) - if( !getExpr( name ).isNormal() ) - return false; - - return true; - } - catch( NotBoundException e ) { - throw new RuntimeException( e ); - } - } - - public boolean isReady() { - - try { - for( NameExpr name : getNameSet() ) - if( !getExpr( name ).isNormal() ) - return false; - - return true; - } - catch( NotBoundException e ) { - throw new RuntimeException( e ); - } - } - - public boolean hasTaskName() { - return lambdaExpr.hasTaskName(); - } - - public boolean isEvaluated() { - return !bodyBlock.isEmpty(); - } - - public void setValue( NameExpr outputNameExpr, CompoundExpr value ) { - bodyBlock.putAssign( outputNameExpr, value ); - } - - @Override - public String toString() { - - StringBuffer buf; - List list; - - try { - - buf = new StringBuffer(); - - buf.append( "ticket(" ); - - buf.append( " task: " ).append( lambdaExpr ); - - list = new LinkedList<>(); - for( NameExpr name : getNameSet() ) - list.add( name.getId() ); - - Collections.sort( list ); - - for( String name : list ) - buf.append( ' ' ).append( name ).append( ": " ).append( getExpr( name ) ); - - buf.append( " )" ); - - return buf.toString(); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/TopLevelContext.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/TopLevelContext.java deleted file mode 100644 index f577df0..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/TopLevelContext.java +++ /dev/null @@ -1,119 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -public class TopLevelContext extends BaseBlock { - - private List targetList; - - public TopLevelContext() { - super( null ); - targetList = new ArrayList<>(); - } - - public TopLevelContext( TopLevelContext template ) { - - this(); - - try { - for( CompoundExpr ce : template.targetList ) - targetList.add( new CompoundExpr( ce ) ); - - for( NameExpr ne : template.getFullNameSet() ) - putAssign( ne, new CompoundExpr( template.getExpr( ne ) ) ); - } catch (NotBoundException e) { - throw new RuntimeException( e ); - } - } - - - public void addTarget( CompoundExpr target ) { - targetList.add( target ); - } - - public void clearTargetList() { - targetList.clear(); - } - - public String getBlockString() { - return super.toString(); - } - - public List getTargetList() { - return Collections.unmodifiableList( targetList ); - } - - public CompoundExpr getTarget( int i ) { - return targetList.get( i ); - } - - public int getTargetListSize() { - return targetList.size(); - } - - public boolean isTargetListEmpty() { - return targetList.isEmpty(); - } - - public boolean removeTarget( CompoundExpr target ) { - return targetList.remove( target ); - } - - public void setTarget( int i, CompoundExpr ce ) { - targetList.set( i, ce ); - } - - @Override - public String toString() { - - StringBuffer buf; - - buf = new StringBuffer(); - - buf.append( super.toString() ); - - for( CompoundExpr target : targetList ) - buf.append( target ).append( ";\n" ); - - return buf.toString(); - } - - @Override - public T visit( NodeVisitor visitor ) throws HasFailedException, NotBoundException { - return visitor.accept( this ); - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Type.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Type.java deleted file mode 100644 index ccba641..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/semanticmodel/Type.java +++ /dev/null @@ -1,37 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public interface Type extends CfNode { - // marker interface -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/staticreduction/DotNodeVisitor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/staticreduction/DotNodeVisitor.java deleted file mode 100644 index d9cb6f9..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/staticreduction/DotNodeVisitor.java +++ /dev/null @@ -1,406 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.staticreduction; - -import java.util.HashMap; -import java.util.Map; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Block; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CondExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CorrelParam; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CurryExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.DataType; -import de.huberlin.wbi.cuneiform.core.semanticmodel.DrawParam; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.LambdaType; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NativeLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfNode; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NodeVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ReduceVar; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SingleExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; - -public class DotNodeVisitor implements NodeVisitor { - - private static final int MAX_LEN = 20; - private static final String LABEL_FOREIGN_LAMBDA_EXPR = "\\\\ {}"; - private static final String LABEL_NATIVE_LAMBDA_EXPR = "\\\\ *{}*"; - private static final String LABEL_TASK = "task"; - private static final String LABEL_APPLY = "apply"; - private static final String LABEL_CURRY = "curry"; - - private final StringBuffer buf; - private final Map exprMap; - private int nextNodeId; - - - public DotNodeVisitor() { - - buf = new StringBuffer(); - nextNodeId = 0; - exprMap = new HashMap<>(); - } - - @Override - public String toString() { - return "digraph G {\n"+buf.toString()+"}\n"; - } - - @Override - public String accept(StringExpr stringExpr) { - - String nodeId; - String value; - - if( exprMap.containsKey( stringExpr ) ) - return exprMap.get( stringExpr ); - - nodeId = popNodeId(); - value = stringExpr.getContent(); - - if( value.length() > MAX_LEN ) - value = ".."; - - buf.append( nodeId ).append( " [shape=none,label=\"'"+value+"'\"];\n" ); - - exprMap.put( stringExpr, nodeId ); - - return nodeId; - } - - - - @Override - public String accept(QualifiedTicket qualifiedTicket) { - throw new RuntimeException( "Illegal qualified ticket." ); - } - - - - @Override - public String accept(NativeLambdaExpr nativeLambdaExpr) { - - String nodeId; - - if( exprMap.containsKey( nativeLambdaExpr ) ) - return exprMap.get( nativeLambdaExpr ); - - nodeId = popNodeId(); - - buf.append( nodeId ).append( " [shape=none,label=\""+LABEL_NATIVE_LAMBDA_EXPR+"\"];\n" ); - - exprMap.put( nativeLambdaExpr, nodeId ); - - return nodeId; - } - - - - @Override - public String accept( NameExpr nameExpr ) { - - String nodeId; - String value; - - if( exprMap.containsKey( nameExpr ) ) - return exprMap.get( nameExpr ); - - nodeId = popNodeId(); - value = nameExpr.getId(); - - if( value.length() > MAX_LEN ) - value = ".."; - - buf.append( nodeId ).append( " [shape=none,label=\""+value+"\"];\n" ); - - exprMap.put( nameExpr, nodeId ); - - return nodeId; - } - - - - @Override - public String accept( LambdaType lambdaType ) { - throw new RuntimeException( "Lambda type is not an expression." ); - } - - - - @Override - public String accept(DrawParam drawParam) { - throw new RuntimeException( "Draw parameter is not an expression." ); - } - - - - @Override - public String accept( DataType dataType ) { - throw new RuntimeException( "Data type is not an expression." ); - } - - - - @Override - public String accept(CondExpr condExpr) throws HasFailedException, NotBoundException { - - String nodeId; - String parentId; - - if( exprMap.containsKey( condExpr ) ) - return exprMap.get( condExpr ); - - nodeId = popNodeId(); - - buf.append( nodeId ).append( " [label=\"cond\",shape=diamond];\n" ); - - parentId = condExpr.getIfExpr().visit( this ); - buf.append( parentId ).append( " -> " ).append( nodeId ).append( " [label=\"if\"];\n" ); - - parentId = condExpr.getThenExpr().visit( this ); - buf.append( parentId ).append( " -> " ).append( nodeId ).append( " [label=\"then\"];\n" ); - - parentId = condExpr.getElseExpr().visit( this ); - buf.append( parentId ).append( " -> " ).append( nodeId ).append( " [label=\"else\"];\n" ); - - exprMap.put( condExpr, nodeId ); - - return nodeId; - } - - - - @Override - public String accept( Block block ) { - throw new RuntimeException( "Block is not an expression." ); - } - - - - @Override - public String accept(ApplyExpr applyExpr) throws HasFailedException, NotBoundException { - - String taskNodeId, refNodeId; - SingleExpr se; - String taskName; - - if( exprMap.containsKey( applyExpr ) ) - return exprMap.get( applyExpr ); - - taskNodeId = popNodeId(); - - if( applyExpr.getTaskExpr().getNumSingleExpr() == 0 ) - throw new RuntimeException( "Task expression must not be nil." ); - - se = applyExpr.getTaskExpr().getSingleExpr( 0 ); - - if( applyExpr.getTaskExpr().getNumSingleExpr() != 1 || se instanceof CurryExpr ) { - - refNodeId = applyExpr.getTaskExpr().visit( this ); - - buf.append( taskNodeId ).append( " [shape=box,label=\""+LABEL_APPLY+"\"];\n" ); - buf.append( refNodeId ).append( " -> " ).append( taskNodeId ).append( " [label=\""+LABEL_TASK+"\"];\n" ); - } - else { - - if( se instanceof NativeLambdaExpr ) - taskName = LABEL_NATIVE_LAMBDA_EXPR; - else if( se instanceof ForeignLambdaExpr ) - taskName = LABEL_FOREIGN_LAMBDA_EXPR; - else if( se instanceof NameExpr ) - taskName = ( ( NameExpr)se ).getId(); - else - throw new RuntimeException( "Task type not recognized." ); - - buf.append( taskNodeId ).append( " [shape=box,label=\""+taskName+"\"];\n" ); - } - - for( NameExpr nameExpr : applyExpr.getNameSet() ) - - try { - - refNodeId = applyExpr.getExpr( nameExpr ).visit( this ); - - buf.append( refNodeId ).append( " -> " ).append( taskNodeId ).append( ";\n" ); - } - catch( NotBoundException e ) { - // cannot happen, because we only ask for what is there - throw new RuntimeException( e.getMessage() ); - } - - - exprMap.put( applyExpr, taskNodeId ); - - return taskNodeId; - } - - - - @Override - public String accept( CompoundExpr compoundExpr ) throws HasFailedException, NotBoundException { - - String nodeId; - int n; - - if( exprMap.containsKey( compoundExpr ) ) - return exprMap.get( compoundExpr ); - - - - n = compoundExpr.getNumSingleExpr(); - - if( n == 0 ) { - - nodeId = popNodeId(); - - buf.append( nodeId+" [shape=none,label=\"nil\"];\n" ); - - exprMap.put( compoundExpr, nodeId ); - - return nodeId; - } - - if( n > 1 ) { - - nodeId = popNodeId(); - - buf.append( nodeId ).append( " [shape=point];\n" ); - for( SingleExpr se : compoundExpr.getSingleExprList() ) - buf.append( se.visit( this ) ).append( " -> " ).append( nodeId ).append( ";\n" ); - - exprMap.put( compoundExpr, nodeId ); - - return nodeId; - } - - - return compoundExpr.getSingleExpr( 0 ).visit( this ); - } - - - - @Override - public String accept( CorrelParam correlParam ) { - throw new RuntimeException( "Correlated parameter is not an expression." ); - } - - - - @Override - public String accept(ForeignLambdaExpr foreignLambdaExpr) { - - String nodeId; - - if( exprMap.containsKey( foreignLambdaExpr ) ) - return exprMap.get( foreignLambdaExpr ); - - nodeId = popNodeId(); - - buf.append( nodeId ).append( " [shape=none,label=\""+LABEL_FOREIGN_LAMBDA_EXPR+"\"];\n" ); - - exprMap.put( foreignLambdaExpr, nodeId ); - - return nodeId; - } - - - - @Override - public String accept(CurryExpr curryExpr) throws HasFailedException, NotBoundException { - - String curryNodeId, refNodeId; - - if( exprMap.containsKey( curryExpr ) ) - return exprMap.get( curryExpr ); - - curryNodeId = popNodeId(); - - refNodeId = curryExpr.getTaskExpr().visit( this ); - - buf.append( curryNodeId ).append( " [shape=box,label=\""+LABEL_CURRY+"\"];\n" ); - buf.append( refNodeId ).append( " -> " ).append( curryNodeId ).append( " [label=\""+LABEL_TASK+"\"];\n" ); - - for( NameExpr nameExpr : curryExpr.getNameSet() ) - - try { - - refNodeId = curryExpr.getExpr( nameExpr ).visit( this ); - - buf.append( refNodeId ).append( " -> " ).append( curryNodeId ).append( ";\n" ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - - exprMap.put( curryExpr, curryNodeId ); - - return curryNodeId; - } - - - - @Override - public String accept( Prototype prototype ) { - throw new RuntimeException( "Prototype is not an expression." ); - } - - - - @Override - public String accept( ReduceVar reduceVar ) { - throw new RuntimeException( "Reduce variable is not an expression." ); - } - - private String popNodeId() { - return "node"+( nextNodeId++ ); - } - - @Override - public String accept( TopLevelContext tlc ) throws HasFailedException, NotBoundException { - - for( CompoundExpr ce : tlc.getTargetList() ) - ce.visit( this ); - - return null; - } - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/staticreduction/StaticNodeVisitor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/staticreduction/StaticNodeVisitor.java deleted file mode 100644 index c6cf4b6..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/staticreduction/StaticNodeVisitor.java +++ /dev/null @@ -1,413 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - - -package de.huberlin.wbi.cuneiform.core.staticreduction; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashSet; -import java.util.LinkedList; -import java.util.List; -import java.util.Set; - -import de.huberlin.wbi.cuneiform.core.preprocess.ChannelListener; -import de.huberlin.wbi.cuneiform.core.preprocess.PreListener; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.BaseBlock; -import de.huberlin.wbi.cuneiform.core.semanticmodel.BaseNodeVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Block; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CondExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CurryExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.LambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NativeLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SemanticModelException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SingleExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; - -/** - * @author jorgen - * - */ -public class StaticNodeVisitor extends BaseNodeVisitor { - - private final LinkedList blockStack; - private BaseBlock currentBlock; - private final Set freeVarSet; - - public StaticNodeVisitor( BaseBlock currentBlock ) { - blockStack = new LinkedList<>(); - this.currentBlock = currentBlock; - freeVarSet = new HashSet<>(); - } - - @Override - public CompoundExpr accept( ApplyExpr applyExpr ) throws HasFailedException, NotBoundException { - SingleExpr se; - NativeLambdaExpr lambda; - NameExpr targetNameExpr; - CompoundExpr targetCompoundExpr; - int channel; - CompoundExpr taskResult, taskOriginal; - ApplyExpr applyExpr1; - boolean rest; - - - // try to reduce the task expression - taskOriginal = applyExpr.getTaskExpr(); - taskResult = taskOriginal.visit( this ); - - channel = applyExpr.getChannel(); - rest = applyExpr.hasRest(); - - applyExpr1 = new ApplyExpr( channel, rest ); - applyExpr1.setTaskExpr( taskResult ); - - // try to reduce the parameter list - try { - for( NameExpr nameExpr : applyExpr.getNameSet() ) { - CompoundExpr expr = applyExpr.getExpr( nameExpr ); - applyExpr1.putAssign( nameExpr, expr.visit( this )); - } - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - // continue only if one single lambda expression is applied - if( taskResult.getNumSingleExpr() != 1 ) - return new CompoundExpr( applyExpr1 ); - - // fetch that lambda expression - se = taskResult.getSingleExpr( 0 ); - - if( se == null ) - throw new NullPointerException( "Expression must not be null." ); - - // continue only if that single lambda expression is native - if( se instanceof NativeLambdaExpr ) { - - lambda = ( NativeLambdaExpr )se; - applyExpr1.setParent( lambda.getBodyBlock() ); - - try { - pushIntoBlock( applyExpr1.getParamBlock() ); - } catch( NotDerivableException e2 ) { - // can't happen we already know, we have a native lambda expression in front of us - throw new RuntimeException( e2.getMessage() ); - } - - targetNameExpr = lambda.getPrototype().getOutput( channel-1 ); - - try { - targetCompoundExpr = currentBlock.getExpr( targetNameExpr ); - } - catch( NotBoundException e1 ) { - throw new SemanticModelException( applyExpr.toString(), e1.getMessage() ); - } - - try { - CompoundExpr expr = currentBlock.getExpr( targetNameExpr ); - targetCompoundExpr = expr.visit( this ); - } - catch( NotBoundException e ) { - // we rely on the parser to sort out whether the target is bound - } - - popBlock(); - - return targetCompoundExpr; - } - - - return new CompoundExpr( applyExpr1 ); - } - - @Override - public CompoundExpr accept(CompoundExpr ce) throws HasFailedException, NotBoundException { - CompoundExpr result, intermediate; - - if( ce == null ) - throw new NullPointerException( "Compound expression must not be null." ); - - result = new CompoundExpr(); - - for( SingleExpr singleExpr : ce.getSingleExprList() ) { - intermediate = singleExpr.visit( this ); - result.addCompoundExpr( intermediate ); - } - - return result; - } - - @Override - public CompoundExpr accept(CondExpr condExpr) throws HasFailedException { - return new CompoundExpr( condExpr ); - } - - @Override - public CompoundExpr accept( CurryExpr curryExpr ) throws HasFailedException, NotBoundException { - - Prototype originalPrototype; - SingleExpr se; - LambdaExpr lambdaExpr; - NativeLambdaExpr nativeLambdaExpr; - Block originalBodyBlock; - Block curriedBodyBlock; - Prototype curriedPrototype; - NativeLambdaExpr curriedLambdaExpr; - - - if( !curryExpr.hasTaskExpr() ) - throw new SemanticModelException( - curryExpr.toString(), - "Task parameter not bound." ); - - if( curryExpr.getTaskExpr().getNumSingleExpr() == 0 ) - throw new SemanticModelException( - curryExpr.toString(), - "Task expression must not be nil." ); - - if( curryExpr.getTaskExpr().getNumSingleExpr() > 1 ) - return new CompoundExpr( curryExpr ); - - CompoundExpr taskExpr = curryExpr.getTaskExpr(); - se = taskExpr.visit( this ).getSingleExpr( 0 ); - - - if( se instanceof NameExpr ) - return new CompoundExpr( curryExpr ); - - if( !( se instanceof LambdaExpr ) ) - throw new SemanticModelException( curryExpr.toString(), se+" is not a lambda expression." ); - - lambdaExpr = ( LambdaExpr )se; - - originalPrototype = lambdaExpr.getPrototype(); - - // the prototype of the curried lambda expression is derived from - // the original prototype - curriedPrototype = new Prototype( originalPrototype ); - - // from the prototype we remove all inputs that are bound by - // currying - for( NameExpr nameExpr : curryExpr.getNameSet() ) - curriedPrototype.removeParam( nameExpr ); - - if( !( lambdaExpr instanceof NativeLambdaExpr ) ) - throw new RuntimeException( "Lambda expression type not recognized." ); - - nativeLambdaExpr = ( NativeLambdaExpr )lambdaExpr; - - originalBodyBlock = nativeLambdaExpr.getBodyBlock(); - - // the body block of the curried lambda expression is derived from the - // body block of the original lambda expression - curriedBodyBlock = new Block(); - for( NameExpr ne : originalBodyBlock.getFullNameSet() ) - curriedBodyBlock.putAssign( ne, new CompoundExpr( originalBodyBlock.getExpr( ne ) ) ); - - // with the curried expression's binding block merged in - try { - for( NameExpr nameExpr : curryExpr.getNameSet() ) - curriedBodyBlock.putAssign( - nameExpr, curryExpr.getExpr( nameExpr ) ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - // from the curried prototype and body expression we form the - // resulting curried lambda expression - curriedLambdaExpr = new NativeLambdaExpr( curriedPrototype, curriedBodyBlock ); - - return new CompoundExpr( curriedLambdaExpr ); - - - // reuse this commented block when in dynamic reducer - /* if( lambdaExpr instanceof ForeignLambdaExpr ) { - - applyExpr = new ApplyExpr( 1, false ); - - applyExpr.setTaskExpr( new CompoundExpr( lambdaExpr ) ); - - try { - - for( NameExpr nameExpr : originalPrototype.getParamNameExprSet() ) - - if( curryExpr.containsName( nameExpr ) ) - applyExpr.putAssign( nameExpr, curryExpr.getExpr( nameExpr ) ); - else - applyExpr.putAssign( nameExpr, new CompoundExpr( nameExpr ) ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - - curriedBodyBlock = new Block(); - - n = originalPrototype.getNumOutput(); - - // replicate apply expression for each output name expression - for( i = 0; i < n; i++ ) { - - ne = originalPrototype.getOutput( i ); - - ae = applyExpr.clone(); - ae.setChannel( i+1 ); - - curriedBodyBlock.putAssign( ne, new CompoundExpr( ae ) ); - } - - curriedLambdaExpr = new NativeLambdaExpr( curriedPrototype, curriedBodyBlock ); - - return new CompoundExpr( curriedLambdaExpr ); - } */ - } - - @Override - public CompoundExpr accept( NameExpr nameExpr ) throws HasFailedException { - - CompoundExpr result; - SingleExpr se; - CompoundExpr expr; - - try { - - expr = currentBlock.getExpr( nameExpr ); - result = expr.visit( this ); - - if( result.getNumSingleExpr() == 1 ) { - - se = result.getSingleExpr( 0 ); - if( se instanceof ForeignLambdaExpr ) - return new CompoundExpr( nameExpr ); - } - - return result; - } - catch( NotBoundException e ) { - freeVarSet.add( nameExpr ); - return new CompoundExpr( nameExpr ); - } - } - - @Override - public CompoundExpr accept( TopLevelContext tlc ) throws HasFailedException, NotBoundException { - - CompoundExpr result; - - freeVarSet.clear(); - - result = new CompoundExpr(); - - for( CompoundExpr ce : tlc.getTargetList() ) - result.addCompoundExpr( ce.visit( this ) ); - - return result; - } - - public Set getFreeVarSet() { - return Collections.unmodifiableSet( freeVarSet ); - } - - private void popBlock() { - currentBlock = blockStack.pop(); - } - - private void pushIntoBlock( Block block ) { - blockStack.push( currentBlock ); - currentBlock = block; - } - - public static TopLevelContext createTlc( String src ) { - - String afterPre, afterChannel; - TopLevelContext tlc; - - afterPre = PreListener.process( src ); - afterChannel = ChannelListener.process( afterPre ); - tlc = CfSemanticModelVisitor.process( afterChannel ); - - return tlc; - } - - public static List getFreeVarNameList( TopLevelContext tlc )throws HasFailedException, NotBoundException { - - StaticNodeVisitor staticVisitor; - List nameList; - - nameList = new ArrayList<>(); - - staticVisitor = new StaticNodeVisitor( tlc ); - - tlc.visit( staticVisitor ); - - for( NameExpr ne : staticVisitor.getFreeVarSet() ) - nameList.add( ne.getId() ); - - return Collections.unmodifiableList( nameList ); - } - - public static List getTargetVarNameList( TopLevelContext tlc ) { - - List nameList; - CompoundExpr ce; - - nameList = new ArrayList<>(); - - try { - for( NameExpr ne : tlc.getNameSet() ) { - - ce = tlc.getExpr( ne ); - if( ce.getNumSingleExpr() == 1 ) - if( ce.getSingleExpr( 0 ) instanceof LambdaExpr ) - continue; - - nameList.add( ne.getId() ); - } - return nameList; - } - catch ( NotBoundException e ) { - throw new RuntimeException( e ); - } - - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/NodeVisitorTicketSrc.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/NodeVisitorTicketSrc.java deleted file mode 100644 index 699e8a6..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/NodeVisitorTicketSrc.java +++ /dev/null @@ -1,51 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.ticketsrc; - -import java.util.Set; -import java.util.UUID; - -import de.huberlin.wbi.cuneiform.core.repl.BaseRepl; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public interface NodeVisitorTicketSrc { - - public UUID getRunId(); - public boolean isQueueClear( UUID queryId )throws HasFailedException; - public QualifiedTicket requestTicket( BaseRepl repl, UUID queryId, ApplyExpr applyExpr )throws HasFailedException; - public Set getTicketSet( UUID queryId ); - -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/ReplTicketSrc.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/ReplTicketSrc.java deleted file mode 100644 index 6755df4..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/ReplTicketSrc.java +++ /dev/null @@ -1,37 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.ticketsrc; - -public interface ReplTicketSrc extends NodeVisitorTicketSrc { - // marker interface -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketFailedMsg.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketFailedMsg.java deleted file mode 100644 index 9933234..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketFailedMsg.java +++ /dev/null @@ -1,151 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.ticketsrc; - -import de.huberlin.wbi.cuneiform.core.actormodel.Message; -import de.huberlin.wbi.cuneiform.core.cre.BaseCreActor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class TicketFailedMsg extends Message { - - private final String script; - private final String stdErr; - private final String stdOut; - private final Ticket ticket; - private final Exception e; - - public TicketFailedMsg( - BaseCreActor sender, - Ticket ticket, - Exception e, - String script, - String stdOut, - String stdErr ) { - - super( sender ); - - if( script == null ) - this.script = null; - else - if( script.isEmpty() ) - this.script = null; - else - this.script = script; - - if( ticket == null ) - throw new NullPointerException( "Ticket must not be null." ); - - if( stdOut == null ) - this.stdOut = null; - else - if( stdOut.isEmpty() ) - this.stdOut = null; - else - this.stdOut = stdOut; - - if( stdErr == null ) - this.stdErr = null; - else - if( stdErr.isEmpty() ) - this.stdErr = null; - else - this.stdErr = stdErr; - - this.ticket = ticket; - this.e = e; - } - - public String getStdErr() { - return stdErr; - } - - public String getStdOut() { - return stdOut; - } - - public Exception getException() { - return e; - } - - public String getScript() { - return script; - } - - public Ticket getTicket() { - return ticket; - } - - public long getTicketId() { - return ticket.getTicketId(); - } - - public boolean hasScript() { - return script != null; - } - - public boolean hasStdErr() { - return stdErr != null; - } - - public boolean hasStdOut() { - return stdOut != null; - } - - public boolean hasException() { - return e != null; - } - - @Override - public String toString() { - - String s; - - s = ""; - - if( e != null ) { - s += " "+e.getClass().getName(); - - if( e.getMessage() != null ) - s += ": \""+e.getMessage()+"\""; - } - - if( stdOut != null ) - s += " Output channel: \""+stdOut.replace( '\n', ' ' )+"\""; - - if( stdErr != null ) - s += " Error channel: \""+stdErr.replace( '\n', ' ' )+"\""; - - - return "{ ticketFailed, "+ticket.getTicketId()+","+s+" }"; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketFinishedMsg.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketFinishedMsg.java deleted file mode 100644 index 8337d00..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketFinishedMsg.java +++ /dev/null @@ -1,94 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.ticketsrc; - - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Set; - -import de.huberlin.wbi.cuneiform.core.actormodel.Message; -import de.huberlin.wbi.cuneiform.core.cre.BaseCreActor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class TicketFinishedMsg extends Message { - - private final Ticket ticket; - private final Set reportEntrySet; - - public TicketFinishedMsg( BaseCreActor sender, Ticket ticket, Collection report ) { - - super( sender ); - - if( ticket == null ) - throw new NullPointerException( "Ticket must not be null." ); - - this.ticket = ticket; - - reportEntrySet = new HashSet<>(); - - addReport( report ); - } - - public void addReport( Collection report ) { - - if( report == null ) - throw new NullPointerException( "JSON report entry collection must not be null." ); - - for( JsonReportEntry entry : report ) - addReport( entry ); - } - - public void addReport( JsonReportEntry entry ) { - - if( entry == null ) - throw new NullPointerException( "JSON report entry must not be null." ); - - reportEntrySet.add( entry ); - } - - public Set getReportEntrySet() { - return Collections.unmodifiableSet( reportEntrySet ); - } - - public Ticket getTicket() { - return ticket; - } - - @Override - public String toString() { - return "{ ticketFinished, "+ticket.getTicketId()+" }"; - } -} diff --git a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketSrcActor.java b/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketSrcActor.java deleted file mode 100644 index c1da58c..0000000 --- a/cuneiform-core/src/main/java/de/huberlin/wbi/cuneiform/core/ticketsrc/TicketSrcActor.java +++ /dev/null @@ -1,334 +0,0 @@ -/******************************************************************************* - * In the Hi-WAY project we propose a novel approach of executing scientific - * workflows processing Big Data, as found in NGS applications, on distributed - * computational infrastructures. The Hi-WAY software stack comprises the func- - * tional workflow language Cuneiform as well as the Hi-WAY ApplicationMaster - * for Apache Hadoop 2.x (YARN). - * - * List of Contributors: - * - * Jörgen Brandt (HU Berlin) - * Marc Bux (HU Berlin) - * Ulf Leser (HU Berlin) - * - * Jörgen Brandt is funded by the European Commission through the BiobankCloud - * project. Marc Bux is funded by the Deutsche Forschungsgemeinschaft through - * research training group SOAMED (GRK 1651). - * - * Copyright 2014 Humboldt-Universität zu Berlin - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - ******************************************************************************/ - -package de.huberlin.wbi.cuneiform.core.ticketsrc; - -import java.util.HashMap; -import java.util.HashSet; -import java.util.Map; -import java.util.Set; -import java.util.UUID; - -import de.huberlin.wbi.cuneiform.core.actormodel.Actor; -import de.huberlin.wbi.cuneiform.core.actormodel.Message; -import de.huberlin.wbi.cuneiform.core.cre.BaseCreActor; -import de.huberlin.wbi.cuneiform.core.cre.TicketReadyMsg; -import de.huberlin.wbi.cuneiform.core.repl.BaseRepl; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ReduceVar; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CfSemanticModelVisitor; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Ticket; - -public class TicketSrcActor extends Actor implements ReplTicketSrc { - - private final Set replSet; - private final Map> queryTicketMap; - private final Map> ticketQueryMap; - private final Map cacheMap; - private final UUID runId; - private final BaseCreActor cre; - private final Set failedQuerySet; - - - public TicketSrcActor( BaseCreActor cre ) { - if( cre == null ) - throw new NullPointerException( "CRE actor must not be null." ); - - this.cre = cre; - - queryTicketMap = new HashMap<>(); - ticketQueryMap = new HashMap<>(); - cacheMap = new HashMap<>(); - replSet = new HashSet<>(); - runId = UUID.randomUUID(); - failedQuerySet = new HashSet<>(); - - if( log.isDebugEnabled() ) - log.debug( "New TicketSrcActor's UUID: "+runId ); - } - - @Override - public UUID getRunId() { - return runId; - } - - @Override - public Set getTicketSet( UUID queryId ) { - return queryTicketMap.get( queryId ); - } - - @Override - public boolean isQueueClear( UUID queryId ) { - - Set set; - - if( queryId == null ) - throw new NullPointerException( "Run ID must not be null." ); - - set = queryTicketMap.get( queryId ); - - if( set == null ) - return true; - - return set.isEmpty(); - } - - @Override - protected void processMsg( Message msg ) { - - TicketFinishedMsg ticketFinishedMsg; - TicketFailedMsg ticketFailedMsg; - Ticket ticket; - String script, stdOut, stdErr; - long ticketId; - Exception e; - Set querySet; - - - if( msg instanceof TicketFinishedMsg ) { - - ticketFinishedMsg = ( TicketFinishedMsg )msg; - ticket = ticketFinishedMsg.getTicket(); - ticketId = ticket.getTicketId(); - querySet = ticketQueryMap.get( ticket ); - - clearTicket( ticket ); - - if( querySet != null ) - for( UUID queryId : querySet ) - for( BaseRepl repl : replSet ) - if( repl.isRunning( queryId ) ) - repl.ticketFinished( queryId, ticketId, ticketFinishedMsg.getReportEntrySet() ); - - - - return; - } - - if( msg instanceof TicketFailedMsg ) { - - ticketFailedMsg = ( TicketFailedMsg )msg; - - ticket = ticketFailedMsg.getTicket(); - ticketId = ticketFailedMsg.getTicketId(); - script = ticketFailedMsg.getScript(); - stdOut = ticketFailedMsg.getStdOut(); - stdErr = ticketFailedMsg.getStdErr(); - e = ticketFailedMsg.getException(); - querySet = ticketQueryMap.get( ticket ); - - clearTicket( ticket ); - cacheMap.remove( ticketId ); - - if( querySet != null ) { - - for( UUID queryId : querySet ) - for( BaseRepl repl : replSet ) - if( repl.isRunning( queryId ) ) - repl.queryFailed( queryId, ticketId, e, script, stdOut, stdErr ); - - for( UUID queryId : querySet ) { - clearQuery( queryId ); - failedQuerySet.add( queryId ); - } - } - - - return; - } - - throw new RuntimeException( "Message type "+msg.getClass()+" not recognized." ); - } - - @Override - public synchronized QualifiedTicket requestTicket( BaseRepl repl, UUID queryId, ApplyExpr applyExpr ) throws HasFailedException { - - CompoundExpr ce; - int channel; - ForeignLambdaExpr lambda; - Ticket ticket; - long ticketId; - QualifiedTicket qt; - Prototype prototype; - - if( queryId == null ) - throw new NullPointerException( "Query ID must not be null." ); - - if( applyExpr == null ) - throw new NullPointerException( "Apply expression must not be null." ); - - if( repl == null ) - throw new NullPointerException( "REPL actor must not be null." ); - - if( failedQuerySet.contains( queryId ) ) - throw new HasFailedException( "Query "+queryId+" has already failed. Dropping request." ); - - if( log.isDebugEnabled() ) - log.debug( "Requesting ticket for "+applyExpr.toString().replace( '\n', ' ') ); - - ce = applyExpr.getTaskExpr(); - try { - - if( ce.getNumAtom() != 1 ) - throw new RuntimeException( "Excepted singular task expression." ); - - if( !( ce.getSingleExpr( 0 ) instanceof ForeignLambdaExpr ) ) - throw new RuntimeException( "Expected foreign lambda expression." ); - - lambda = ( ForeignLambdaExpr )ce.getSingleExpr( 0 ); - } - catch( NotDerivableException e ) { - throw new RuntimeException( "Cannot derive cardinality of task expression." ); - } - - prototype = lambda.getPrototype(); - - for( NameExpr name : prototype.getParamNameSet() ) { - - if( name instanceof ReduceVar ) - continue; - - if( name.getId().equals( CfSemanticModelVisitor.LABEL_TASK ) ) - continue; - - try { - - if( applyExpr.getExpr( name ).getNumAtom() != 1 ) - throw new RuntimeException( "Expected singular application." ); - } - catch( NotDerivableException e ) { - throw new RuntimeException( "Cannot derive cardinality of parameter "+name ); - } - catch( NotBoundException e ) { - throw new RuntimeException( e.getMessage() ); - } - } - - replSet.add( repl ); - - channel = applyExpr.getChannel(); - - - - ticket = new Ticket( lambda, applyExpr, runId ); - - ticketId = ticket.getTicketId(); - - if( cacheMap.containsKey( ticketId ) ) { - - if( log.isDebugEnabled() ) - log.debug( "Returning ticket "+ticketId+" from cache." ); - - ticket = cacheMap.get( ticketId ); - putTicket( queryId, ticket ); - qt = new QualifiedTicket( ticket, channel ); - } - else { - - if( log.isDebugEnabled() ) - log.debug( "Creating new ticket "+ticketId+"." ); - - qt = new QualifiedTicket( ticket, channel ); - putTicket( queryId, ticket ); - cacheMap.put( ticketId, ticket ); - cre.sendMsg( new TicketReadyMsg( this, queryId, ticket ) ); - } - - if( log.isDebugEnabled() ) - log.debug( "Substitute with ticket: "+ticket.toString().replace( '\n', ' ' ) ); - - return qt; - } - - @Override - protected void shutdown() { - // nothing to do - } - - private void putTicket( UUID queryId, Ticket ticket ) { - - Set ticketSet; - Set uuidSet; - - if( queryId == null ) - throw new NullPointerException( "Run must not be null." ); - - if( ticket == null ) - throw new NullPointerException( "Ticket must not be null." ); - - if( ticket.isEvaluated() ) - return; - - ticketSet = queryTicketMap.get( queryId ); - if( ticketSet == null ) { - ticketSet = new HashSet<>(); - queryTicketMap.put( queryId, ticketSet ); - } - - uuidSet = ticketQueryMap.get( ticket ); - if( uuidSet == null ) { - uuidSet = new HashSet<>(); - ticketQueryMap.put( ticket, uuidSet ); - } - - if( !ticket.isNormal() ) - throw new RuntimeException( "Ticket not ready." ); - - ticketSet.add( ticket ); - uuidSet.add( queryId ); - - - } - - private void clearTicket( Ticket ticket ) { - - ticketQueryMap.remove( ticket ); - for( Set ticketSet : queryTicketMap.values() ) - ticketSet.remove( ticket ); - } - - private void clearQuery( UUID queryId ) { - - queryTicketMap.remove( queryId ); - for( Set querySet : ticketQueryMap.values() ) - querySet.remove( queryId ); - } - -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/invoc/JsonReportEntryTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/invoc/JsonReportEntryTest.java deleted file mode 100644 index 9b9c1d1..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/invoc/JsonReportEntryTest.java +++ /dev/null @@ -1,70 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.invoc; - -import java.util.UUID; - -import org.json.JSONException; -import org.junit.Assert; -import org.junit.Test; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.JsonReportEntry; - -public class JsonReportEntryTest { - - @SuppressWarnings("static-method") - @Test - public void testRunIdIsParsed() throws JSONException { - - String line; - JsonReportEntry entry; - - line = "{timestamp:1396610901698,runId:\"ae113134-5b4f-4ab1-8da7-81507f9668c9\",taskId:3122,taskname:null,lang:null,invocId:null,key:\"wf-name\",value:\"word-count.cf\"}"; - - entry = new JsonReportEntry( line ); - Assert.assertEquals( - "Invalid run ID.", - UUID.fromString( "ae113134-5b4f-4ab1-8da7-81507f9668c9" ), - entry.getRunId() ); - } - - @SuppressWarnings("static-method") - @Test - public void testTaskIdCanBeNull() throws JSONException { - - String line; - JsonReportEntry entry; - - line = "{timestamp:1396615800633,runId:\"2281e887-5b0a-40d5-8b00-d55b482de1ef\",taskId:null,taskname:null,lang:null,invocId:null,key:\"wf-name\",value:\"montage_0\"}"; - - entry = new JsonReportEntry( line ); - Assert.assertFalse( "Entry should not have a task id.", entry.hasTaskId() ); - Assert.assertNull( "Task id should be null.", entry.getTaskId() ); - } - - @SuppressWarnings("static-method") - @Test - public void testTaskIdCanBeMissing1() throws JSONException { - - String line; - JsonReportEntry entry; - - line = "{timestamp:1398874529654,runId:\"7dbb006c-2b9b-41ce-9dde-9433d1416de0\",key:\"wf-name\",value:\"variant-call-09-setup.cf\"}"; - - entry = new JsonReportEntry( line ); - Assert.assertFalse( "Entry should not have a task id.", entry.hasTaskId() ); - Assert.assertNull( "Task id should be null.", entry.getTaskId() ); - } - - @SuppressWarnings("static-method") - @Test - public void testTaskIdCanBeMissing2() throws JSONException { - - String line; - JsonReportEntry entry; - - line = "{timestamp:1398874529676,runId:\"7dbb006c-2b9b-41ce-9dde-9433d1416de0\",key:\"hiway-event\",value:{\"priority\":\"0\",\"nodes\":[],\"type\":\"container-requested\",\"vcores\":1,\"memory\":1536}}"; - - entry = new JsonReportEntry( line ); - Assert.assertFalse( "Entry should not have a task id.", entry.hasTaskId() ); - Assert.assertNull( "Task id should be null.", entry.getTaskId() ); - } -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/repl/DynamicNodeVisitorTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/repl/DynamicNodeVisitorTest.java deleted file mode 100644 index 463168d..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/repl/DynamicNodeVisitorTest.java +++ /dev/null @@ -1,209 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.repl; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - -import java.util.UUID; - -import org.junit.Before; -import org.junit.Test; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Block; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CondExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.LambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NativeLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; -import de.huberlin.wbi.cuneiform.core.ticketsrc.NodeVisitorTicketSrc; - -public class DynamicNodeVisitorTest { - - private DynamicNodeVisitor dnv; - private CompoundExpr x; - - @Before - public void setUp() throws HasFailedException, NotDerivableException { - - NodeVisitorTicketSrc ticketSrc; - BaseRepl repl; - TopLevelContext tlc; - QualifiedTicket qt; - - x = new CompoundExpr( new StringExpr( "Z" ) ); - - qt = mock( QualifiedTicket.class ); - when( qt.getOutputValue() ).thenThrow( new NotDerivableException( "blub" ) ); - when( qt.getNumAtom() ).thenThrow( new NotDerivableException( "blub" ) ); - - ticketSrc = mock( NodeVisitorTicketSrc.class ); - when( ticketSrc.requestTicket( any( BaseRepl.class ), any( UUID.class ), any( ApplyExpr.class ) ) ). - thenReturn( qt ); - - repl = mock( BaseRepl.class ); - tlc = new TopLevelContext(); - tlc.putAssign( new NameExpr( "x" ), x ); - - dnv = new DynamicNodeVisitor( ticketSrc, repl, tlc ); - } - - @Test - public void cndFalseShouldEvalElseExpr() throws HasFailedException, NotBoundException { - - CondExpr condExpr; - CompoundExpr thenExpr, elseExpr, result; - - thenExpr = mock( CompoundExpr.class ); - elseExpr = new CompoundExpr( new StringExpr( "B" ) ); - - condExpr = new CondExpr( new CompoundExpr(), thenExpr, elseExpr ); - result = dnv.accept( condExpr ); - - assertEquals( elseExpr, result ); - - } - - @Test - public void cndEvaluatesConditionBeforeDecision1() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - LambdaExpr lam; - ApplyExpr app; - CompoundExpr e, thenExpr, elseExpr; - CondExpr condExpr; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr() ); - - lam = new NativeLambdaExpr( sign, body ); - - app = new ApplyExpr( 1, false ); - app.setTaskExpr( new CompoundExpr( lam ) ); - - thenExpr = new CompoundExpr( new StringExpr( "A" ) ); - elseExpr = new CompoundExpr( new StringExpr( "B" ) ); - - condExpr = new CondExpr( new CompoundExpr( app ), thenExpr, elseExpr ); - - e = new CompoundExpr( condExpr ); - - assertEquals( elseExpr, dnv.accept( e ) ); - } - - @Test - public void cndEvaluatesConditionBeforeDecision2() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - LambdaExpr lam; - ApplyExpr app; - CompoundExpr e, thenExpr, elseExpr; - CondExpr condExpr; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new StringExpr( "true" ) ) ); - - lam = new NativeLambdaExpr( sign, body ); - - app = new ApplyExpr( 1, false ); - app.setTaskExpr( new CompoundExpr( lam ) ); - - thenExpr = new CompoundExpr( new StringExpr( "A" ) ); - elseExpr = new CompoundExpr( new StringExpr( "B" ) ); - - condExpr = new CondExpr( new CompoundExpr( app ), thenExpr, elseExpr ); - - e = new CompoundExpr( condExpr ); - - assertEquals( thenExpr, dnv.accept( e ) ); - } - - @Test - public void cndEvaluatesOnlyOnFinalCondition() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - LambdaExpr lam; - ApplyExpr app; - CompoundExpr e, f, thenExpr, elseExpr; - CondExpr condExpr; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new StringExpr( "true" ) ) ); - - lam = new ForeignLambdaExpr( sign, "bash", "blub" ); - - app = new ApplyExpr( 1, false ); - app.setTaskExpr( new CompoundExpr( lam ) ); - - thenExpr = new CompoundExpr( new StringExpr( "A" ) ); - elseExpr = new CompoundExpr( new StringExpr( "B" ) ); - - condExpr = new CondExpr( new CompoundExpr( app ), thenExpr, elseExpr ); - - e = new CompoundExpr( condExpr ); - - f = dnv.accept( e ); - assertFalse( f.isNormal() ); - assertEquals( 1, f.getNumSingleExpr() ); - assertTrue( f.getSingleExpr( 0 ) instanceof CondExpr ); - } - - @Test - public void cndEvaluatesThenExpression() throws HasFailedException, NotBoundException { - - CompoundExpr e, thenExpr, elseExpr; - CondExpr condExpr; - - - - thenExpr = new CompoundExpr( new NameExpr( "x" ) ); - elseExpr = new CompoundExpr( new StringExpr( "B" ) ); - - condExpr = new CondExpr( new CompoundExpr( new StringExpr( "X" ) ), thenExpr, elseExpr ); - - e = new CompoundExpr( condExpr ); - - assertEquals( x, dnv.accept( e ) ); - } - - @Test - public void cndEvaluatesElseExpression() throws HasFailedException, NotBoundException { - - CompoundExpr e, thenExpr, elseExpr; - CondExpr condExpr; - - - - thenExpr = new CompoundExpr( new StringExpr( "A" ) ); - elseExpr = new CompoundExpr( new NameExpr( "x" ) ); - - condExpr = new CondExpr( new CompoundExpr(), thenExpr, elseExpr ); - - e = new CompoundExpr( condExpr ); - - assertEquals( x, dnv.accept( e ) ); - } - -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/repl/ReferenceTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/repl/ReferenceTest.java deleted file mode 100644 index 12be6e8..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/repl/ReferenceTest.java +++ /dev/null @@ -1,1063 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.repl; - -import static org.junit.Assert.*; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.*; -import static de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr.LANGID_BASH; - -import java.util.UUID; - -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; - -import de.huberlin.wbi.cuneiform.core.semanticmodel.ApplyExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Block; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CompoundExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CondExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.CorrelParam; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ForeignLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.HasFailedException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NameExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NativeLambdaExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotBoundException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.NotDerivableException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.Prototype; -import de.huberlin.wbi.cuneiform.core.semanticmodel.QualifiedTicket; -import de.huberlin.wbi.cuneiform.core.semanticmodel.ReduceVar; -import de.huberlin.wbi.cuneiform.core.semanticmodel.SemanticModelException; -import de.huberlin.wbi.cuneiform.core.semanticmodel.StringExpr; -import de.huberlin.wbi.cuneiform.core.semanticmodel.TopLevelContext; -import de.huberlin.wbi.cuneiform.core.ticketsrc.NodeVisitorTicketSrc; - -public class ReferenceTest { - - private DynamicNodeVisitor dnv; - private TopLevelContext tlc; - private NodeVisitorTicketSrc ticketSrc; - - @Before - public void setUp() throws HasFailedException, NotDerivableException { - - BaseRepl repl; - QualifiedTicket qt; - - - qt = mock( QualifiedTicket.class ); - when( qt.getChannel() ).thenReturn( 1 ); - when( qt.getOutputValue() ).thenThrow( new NotDerivableException( "blub" ) ); - when( qt.getNumAtom() ).thenThrow( new NotDerivableException( "blub" ) ); - - ticketSrc = mock( NodeVisitorTicketSrc.class ); - when( ticketSrc.requestTicket( any( BaseRepl.class ), any( UUID.class ), any( ApplyExpr.class ) ) ). - thenReturn( qt ); - - repl = mock( BaseRepl.class ); - tlc = new TopLevelContext(); - - dnv = new DynamicNodeVisitor( ticketSrc, repl, tlc ); - } - - @Test - public void nilShouldEvalItself() throws HasFailedException, NotBoundException { - - CompoundExpr nil, result; - - nil = new CompoundExpr(); - result = dnv.accept( nil ); - assertEquals( nil, result ); - } - - @Test - public void strShouldEvalItself() throws HasFailedException, NotBoundException { - - CompoundExpr str, result; - - str = new CompoundExpr( new StringExpr( "bla" ) ); - result = dnv.accept( str ); - assertEquals( str, result ); - } - - @Test( expected=NotBoundException.class ) - public void undefVarShouldFail() throws HasFailedException, NotBoundException { - - CompoundExpr freeVar; - - freeVar = new CompoundExpr( new NameExpr( "x" ) ); - dnv.accept( freeVar ); - } - - @Test - public void defVarShouldEvalToBoundValue() throws HasFailedException, NotBoundException { - - CompoundExpr boundVar, result, content; - - content = new CompoundExpr( new StringExpr( "blub" ) ); - tlc.putAssign( new NameExpr( "x" ), content ); - - boundVar = new CompoundExpr( new NameExpr( "x" ) ); - result = dnv.accept( boundVar ); - - assertEquals( content, result ); - } - - @Test - public void defVarShouldCascadeBinding() throws HasFailedException, NotBoundException { - - CompoundExpr result, str; - - str = new CompoundExpr( new StringExpr( "blub" ) ); - - tlc.putAssign( new NameExpr( "x" ), new CompoundExpr( new NameExpr( "y" ) ) ); - tlc.putAssign( new NameExpr( "y" ), str ); - - result = dnv.accept( new CompoundExpr( new NameExpr( "x" ) ) ); - assertEquals( str, result ); - } - - @Test - public void unfinishedTicketShouldEvalToItself() throws HasFailedException, NotBoundException { - - CompoundExpr ce; - QualifiedTicket qt; - - qt = mock( QualifiedTicket.class ); - when( qt.visit( dnv ) ).thenReturn( dnv.accept( qt ) ); - - ce = new CompoundExpr( ); - - assertEquals( ce, dnv.accept( ce ) ); - } - - @Test - public void identityFnShouldEvalArg() throws HasFailedException, NotBoundException { - - CompoundExpr e, f, lamList; - Prototype sign; - Block body; - ApplyExpr ae; - - - e = new CompoundExpr( new StringExpr( "bla" ) ); - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "inp" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new NameExpr( "inp" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - ae.putAssign( new NameExpr( "inp" ), e ); - - f = new CompoundExpr( ae ); - - assertEquals( e, dnv.accept( f ) ); - } - - @Test - public void lamShouldEvalItself() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - CompoundExpr lamList; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new NameExpr( "blub" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - assertEquals( lamList, dnv.accept( lamList ) ); - } - - @Test - public void multipleOutputShouldBeBindable() throws HasFailedException, NotBoundException { - - Prototype sign; - CompoundExpr e1, e2, lamList; - ApplyExpr f1, f2; - Block body; - - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out1" ) ); - sign.addOutput( new NameExpr( "out2" ) ); - sign.addParam( new NameExpr( "task" ) ); - - e1 = new CompoundExpr( new StringExpr( "bla" ) ); - e2 = new CompoundExpr( new StringExpr( "blub" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out1" ), e1 ); - body.putAssign( new NameExpr( "out2" ), e2 ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - f1 = new ApplyExpr( 1, false ); - f1.setTaskExpr( lamList ); - - f2 = new ApplyExpr( 2, false ); - f2.setTaskExpr( lamList ); - - assertEquals( e1, dnv.accept( f1 ) ); - assertEquals( e2, dnv.accept( f2 ) ); - } - - @Test( expected=SemanticModelException.class ) - public void applicationShouldIgnoreCallingContext() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - CompoundExpr lamList; - ApplyExpr ae; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new NameExpr( "x" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - - tlc.putAssign( new NameExpr( "x" ), new CompoundExpr( new StringExpr( "blub" ) ) ); - dnv.accept( new CompoundExpr( ae ) ); - } - - @Test - public void bindingShouldOverrideBody() throws HasFailedException, NotBoundException { - - CompoundExpr f, g, h, lamList; - Prototype sign; - Block body; - ApplyExpr ae; - - f = new CompoundExpr( new StringExpr( "blub" ) ); - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "x" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "x" ), new CompoundExpr( new StringExpr( "bla" ) ) ); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new NameExpr( "x" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - ae.putAssign( new NameExpr( "x" ), f ); - - g = new CompoundExpr( ae ); - h = dnv.accept( g ); - - assertEquals( f, h ); - } - - @Test( expected=NullPointerException.class ) - public void appWithEmptyTaskListShouldFail() throws HasFailedException, NotBoundException { - - ApplyExpr ae; - - ae = new ApplyExpr( 1, false ); - dnv.accept( new CompoundExpr( ae ) ); - } - - @Ignore - @Test - public void crossProductShouldBeDerivableFromSignature() throws HasFailedException, NotBoundException { - - Prototype sign; - CompoundExpr e1, e2, lamList, f1, f2; - Block body; - ApplyExpr ae1, ae2; - - e1 = new CompoundExpr(); - e1.addSingleExpr( new StringExpr( "A" ) ); - e1.addSingleExpr( new StringExpr( "B" ) ); - - e2 = new CompoundExpr(); - e2.addSingleExpr( new StringExpr( "1" ) ); - e2.addSingleExpr( new StringExpr( "2" ) ); - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out1" ) ); - sign.addOutput( new NameExpr( "out2" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "p1" ) ); - sign.addParam( new NameExpr( "p2" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out1" ), new CompoundExpr( new NameExpr( "p1" ) ) ); - body.putAssign( new NameExpr( "out2" ), new CompoundExpr( new NameExpr( "p2" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( lamList ); - ae1.putAssign( new NameExpr( "p1" ), e1 ); - ae1.putAssign( new NameExpr( "p2" ), e2 ); - - ae2 = new ApplyExpr( 2, false ); - ae2.setTaskExpr( lamList ); - ae2.putAssign( new NameExpr( "p1" ), e1 ); - ae2.putAssign( new NameExpr( "p2" ), e2 ); - - f1 = new CompoundExpr(); - f1.addSingleExpr( new StringExpr( "A" ) ); - f1.addSingleExpr( new StringExpr( "A" ) ); - f1.addSingleExpr( new StringExpr( "B" ) ); - f1.addSingleExpr( new StringExpr( "B" ) ); - - f2 = new CompoundExpr(); - f2.addSingleExpr( new StringExpr( "1" ) ); - f2.addSingleExpr( new StringExpr( "2" ) ); - f2.addSingleExpr( new StringExpr( "1" ) ); - f2.addSingleExpr( new StringExpr( "2" ) ); - - assertEquals( f1, dnv.accept( new CompoundExpr( ae1 ) ) ); - assertEquals( f2, dnv.accept( new CompoundExpr( ae2 ) ) ); - } - - public void dotProductShouldBeDerivableFromSignature() throws HasFailedException, NotBoundException { - - Prototype sign; - CorrelParam correl; - CompoundExpr e1, e2, lamList; - Block body; - ApplyExpr ae1, ae2; - - - correl = new CorrelParam(); - correl.addName( new NameExpr( "p1" ) ); - correl.addName( new NameExpr( "p2" ) ); - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out1" ) ); - sign.addOutput( new NameExpr( "out2" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( correl ); - - e1 = new CompoundExpr(); - e1.addSingleExpr( new StringExpr( "A" ) ); - e1.addSingleExpr( new StringExpr( "B" ) ); - - e2 = new CompoundExpr(); - e2.addSingleExpr( new StringExpr( "1" ) ); - e2.addSingleExpr( new StringExpr( "2" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out1" ), new CompoundExpr( new NameExpr( "p1" ) ) ); - body.putAssign( new NameExpr( "out2" ), new CompoundExpr( new NameExpr( "p2" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( lamList ); - ae1.putAssign( new NameExpr( "p1" ), e1 ); - ae1.putAssign( new NameExpr( "p2" ), e2 ); - - ae2 = new ApplyExpr( 2, false ); - ae2.setTaskExpr( lamList ); - ae2.putAssign( new NameExpr( "p1" ), e1 ); - ae2.putAssign( new NameExpr( "p2" ), e2 ); - - assertEquals( e1, dnv.accept( new CompoundExpr( ae1 ) ) ); - assertEquals( e2, dnv.accept( new CompoundExpr( ae2 ) ) ); - } - - @Test - public void aggregateShouldConsumeListAsWhole() throws HasFailedException, NotBoundException { - - Prototype sign; - CompoundExpr e1, e2, e3, e4, lamList; - Block body; - ApplyExpr ae; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new ReduceVar( "inp", null ) ); - - e1 = new CompoundExpr( new StringExpr( "A" ) ); - - e2 = new CompoundExpr(); - e2.addSingleExpr( new StringExpr( "B" ) ); - e2.addSingleExpr( new StringExpr( "C" ) ); - - e3 = new CompoundExpr(); - e3.addCompoundExpr( e1 ); - e3.addSingleExpr( new NameExpr( "inp" ) ); - - - body = new Block(); - body.putAssign( new NameExpr( "out" ), e3 ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - ae.putAssign( new NameExpr( "inp" ), e2 ); - - e4 = new CompoundExpr(); - e4.addCompoundExpr( e1 ); - e4.addCompoundExpr( e2 ); - - assertEquals( e4, dnv.accept( new CompoundExpr( ae ) ) ); - } - - // @Test - public void taskCorrelationShouldWork() throws HasFailedException, NotBoundException { - - Prototype sign; - CorrelParam correl; - Block body1, body2; - CompoundExpr e1, e2, e3, e4, e5, lamList; - ApplyExpr ae; - - correl = new CorrelParam(); - correl.addName( new NameExpr( "task" ) ); - correl.addName( new NameExpr( "c" ) ); - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( correl ); - sign.addParam( new NameExpr( "p" ) ); - - e1 = new CompoundExpr(); - e1.addSingleExpr( new NameExpr( "c" ) ); - e1.addSingleExpr( new NameExpr( "p" ) ); - - e2 = new CompoundExpr(); - e2.addSingleExpr( new NameExpr( "p" ) ); - e2.addSingleExpr( new NameExpr( "c" ) ); - - body1 = new Block(); - body1.putAssign( new NameExpr( "out" ), e1 ); - - body2 = new Block(); - body2.putAssign( new NameExpr( "out" ), e2 ); - - lamList = new CompoundExpr(); - lamList.addSingleExpr( new NativeLambdaExpr( sign, body1 ) ); - lamList.addSingleExpr( new NativeLambdaExpr( sign, body2 ) ); - - e3 = new CompoundExpr(); - e3.addSingleExpr( new StringExpr( "A" ) ); - e3.addSingleExpr( new StringExpr( "B" ) ); - - e4 = new CompoundExpr(); - e4.addSingleExpr( new StringExpr( "1" ) ); - e4.addSingleExpr( new StringExpr( "2" ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - ae.putAssign( new NameExpr( "c" ), e3 ); - ae.putAssign( new NameExpr( "p" ), e4 ); - - e5 = new CompoundExpr(); - e5.addSingleExpr( new StringExpr( "A" ) ); - e5.addSingleExpr( new StringExpr( "1" ) ); - e5.addSingleExpr( new StringExpr( "A" ) ); - e5.addSingleExpr( new StringExpr( "2" ) ); - e5.addSingleExpr( new StringExpr( "1" ) ); - e5.addSingleExpr( new StringExpr( "B" ) ); - e5.addSingleExpr( new StringExpr( "2" ) ); - e5.addSingleExpr( new StringExpr( "B" ) ); - - assertEquals( e5, dnv.accept( new CompoundExpr( ae ) ) ); - } - - @Test - public void cndFalseShouldEvalElseExpr() throws HasFailedException, NotBoundException { - - CompoundExpr e, ifExpr, thenExpr, elseExpr, e1; - - e1 = new CompoundExpr( new StringExpr( "B" ) ); - - ifExpr = new CompoundExpr(); - thenExpr = new CompoundExpr( new StringExpr( "A" ) ); - elseExpr = e1; - - e = new CompoundExpr( new CondExpr( ifExpr, thenExpr, elseExpr ) ); - assertEquals( e1, dnv.accept( e ) ); - } - - @Test - public void cndEvaluatesConditionBeforeDecision1() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - CompoundExpr lamList, e, a, b; - ApplyExpr ae; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr() ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - - a = new CompoundExpr( new StringExpr( "A" ) ); - b = new CompoundExpr( new StringExpr( "B" ) ); - - e = new CompoundExpr( new CondExpr( new CompoundExpr( ae ), a, b ) ); - - assertEquals( b, dnv.accept( e ) ); - } - - - - @Test - public void cndEvaluatesConditionBeforeDecision2() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - CompoundExpr lamList, e, a, b, x, y; - ApplyExpr ae; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - x = new CompoundExpr( new StringExpr( "X" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), x ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - - a = new CompoundExpr( new StringExpr( "A" ) ); - b = new CompoundExpr( new StringExpr( "B" ) ); - - e = new CompoundExpr( new CondExpr( new CompoundExpr( ae ), a, b ) ); - - y = dnv.accept( e ); - - assertEquals( a, y ); - } - - @Test - public void cndEvaluatesOnlyOnFinalCondition() throws HasFailedException, NotBoundException { - - Prototype sign; - CompoundExpr lamList, e, a, b, x; - ApplyExpr ae; - CondExpr ce; - - sign = new Prototype(); - sign.addOutput( new ReduceVar( "out", null ) ); - sign.addParam( new NameExpr( "task" ) ); - - lamList = new CompoundExpr( - new ForeignLambdaExpr( - sign, ForeignLambdaExpr.LANGID_BASH, "blub" ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - - a = new CompoundExpr( new NameExpr( "a" ) ); - b = new CompoundExpr( new NameExpr( "b" ) ); - - e = new CompoundExpr( new CondExpr( new CompoundExpr( ae ), a, b ) ); - - tlc.putAssign( new NameExpr( "a" ), new CompoundExpr( new StringExpr( "A" ) ) ); - tlc.putAssign( new NameExpr( "b" ), new CompoundExpr( new StringExpr( "A" ) ) ); - x = dnv.accept( e ); - - assertTrue( x.getSingleExpr( 0 ) instanceof CondExpr ); - - ce = ( CondExpr )x.getSingleExpr( 0 ); - assertEquals( a, ce.getThenExpr() ); - assertEquals( b, ce.getElseExpr() ); - - assertTrue( ce.getIfExpr().getSingleExpr( 0 ) instanceof QualifiedTicket ); - } - - @Test - public void cndEvaluatesThenExpr() throws HasFailedException, NotBoundException { - - CompoundExpr e, f, x; - - e = new CompoundExpr( new CondExpr( - new CompoundExpr( new StringExpr( "Z" ) ), - new CompoundExpr( new NameExpr( "x" ) ), - new CompoundExpr( new StringExpr( "B" ) ) ) ); - - f = new CompoundExpr( new StringExpr( "A" ) ); - - tlc.putAssign( new NameExpr( "x" ), f ); - x = dnv.accept( e ); - - assertEquals( f, x ); - } - - @Test - public void cndEvaluatesElseExpr() throws HasFailedException, NotBoundException { - - CompoundExpr e, f, x; - - e = new CompoundExpr( new CondExpr( - new CompoundExpr(), - new CompoundExpr( new StringExpr( "A" ) ), - new CompoundExpr( new NameExpr( "x" ) ) ) ); - - f = new CompoundExpr( new StringExpr( "B" ) ); - - tlc.putAssign( new NameExpr( "x" ), f ); - x = dnv.accept( e ); - - assertEquals( f, x ); - } - - - @Test - public void foreignAppWithCndParamIsLeftUntouched() throws HasFailedException, NotBoundException { - - CompoundExpr lamList, e, x; - Prototype sign; - ApplyExpr ae1, ae2, ae3; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "p" ) ); - - lamList = new CompoundExpr( new ForeignLambdaExpr( - sign, LANGID_BASH, "blub" ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( lamList ); - ae1.putAssign( - new NameExpr( "p" ), new CompoundExpr( new StringExpr( "A" ) ) ); - - e = new CompoundExpr( new CondExpr( - new CompoundExpr( ae1 ), new CompoundExpr(), new CompoundExpr() ) ); - - ae2 = new ApplyExpr( 1, false ); - ae2.setTaskExpr( lamList ); - ae2.putAssign( new NameExpr( "p" ), e ); - - x = dnv.accept( new CompoundExpr( ae2 ) ); - - assertTrue( x.getSingleExpr( 0 ) instanceof ApplyExpr ); - - ae3 = ( ApplyExpr )x.getSingleExpr( 0 ); - - assertEquals( lamList, ae3.getTaskExpr() ); - } - - @Test - public void foreignAppWithSelectParamIsLeftUntouched() throws HasFailedException, NotBoundException { - - Prototype sign; - CompoundExpr lamList, x; - ApplyExpr ae1, ae2, ae3; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "p" ) ); - - lamList = new CompoundExpr( new ForeignLambdaExpr( - sign, LANGID_BASH, "blub" ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( lamList ); - ae1.putAssign( new NameExpr( "p" ), new CompoundExpr( new StringExpr( "A" ) ) ); - - ae2 = new ApplyExpr( 1, false ); - ae2.setTaskExpr( lamList ); - ae2.putAssign( new NameExpr( "p" ), new CompoundExpr( ae1 ) ); - - x = dnv.accept( new CompoundExpr( ae2 ) ); - - assertTrue( x.getSingleExpr( 0 ) instanceof ApplyExpr ); - - ae3 = ( ApplyExpr )x.getSingleExpr( 0 ); - - assertEquals( lamList, ae3.getTaskExpr() ); - } - - @Test - public void cascadingAppDoesNotBreakEnum() throws HasFailedException, NotBoundException, NotDerivableException { - - Prototype sign1, sign2; - CompoundExpr lam1, lam2, out1, out2, x; - ApplyExpr ae1, ae2; - Block body2; - QualifiedTicket qt1; - - sign1 = new Prototype(); - sign1.addOutput( new ReduceVar( "out", null ) ); - sign1.addParam( new NameExpr( "task" ) ); - - lam1 = new CompoundExpr( new ForeignLambdaExpr( sign1, LANGID_BASH, "out=(1 2 3)" ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( lam1 ); - - sign2 = new Prototype(); - sign2.addOutput( new NameExpr( "out" ) ); - sign2.addParam( new NameExpr( "task" ) ); - sign2.addParam( new NameExpr( "p1" ) ); - sign2.addParam( new NameExpr( "p2" ) ); - - out1 = new CompoundExpr(); - out1.addSingleExpr( new NameExpr( "p1" ) ); - out1.addSingleExpr( new NameExpr( "p2" ) ); - - body2 = new Block(); - body2.putAssign( new NameExpr( "out" ), out1 ); - - lam2 = new CompoundExpr( new NativeLambdaExpr( sign2, body2 ) ); - - ae2 = new ApplyExpr( 1, false ); - ae2.setTaskExpr( lam2 ); - ae2.putAssign( new NameExpr( "p1" ), new CompoundExpr( new StringExpr( "A" ) ) ); - ae2.putAssign( new NameExpr( "p2" ), new CompoundExpr( ae1 ) ); - - out2 = new CompoundExpr(); - out2.addSingleExpr( new StringExpr( "1" ) ); - out2.addSingleExpr( new StringExpr( "2" ) ); - - qt1 = mock( QualifiedTicket.class ); - when( qt1.getChannel() ).thenReturn( 1 ); - when( qt1.getNumAtom() ).thenReturn( 2 ); - when( qt1.getOutputValue() ).thenReturn( out2 ); - when( qt1.getStringExprValue( 0 ) ).thenReturn( new StringExpr( "1" ) ); - when( qt1.getStringExprValue( 1 ) ).thenReturn( new StringExpr( "2" ) ); - - when( ticketSrc.requestTicket( - any( BaseRepl.class ), - any( UUID.class ), - any( ApplyExpr.class ) ) ) - .thenReturn( qt1 ); - - x = dnv.accept( new CompoundExpr( ae2 ) ); - - assertEquals( "'A' '1' 'A' '2'", x.toString() ); - } - - @Test - public void appTaskParamIsEvaluated() throws HasFailedException, NotBoundException { - - Prototype sign; - Block body; - CompoundExpr lamList, x; - ApplyExpr ae; - - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new StringExpr( "A" ) ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( new CompoundExpr( new NameExpr( "f" ) ) ); - - tlc.putAssign( new NameExpr( "f" ), lamList ); - - x = dnv.accept( new CompoundExpr( ae ) ); - - assertEquals( new CompoundExpr( new StringExpr( "A" ) ), x ); - } - - @Test - public void appNonFinalResultPreservesApp() throws NotDerivableException, HasFailedException, NotBoundException { - - Prototype sign; - Block body; - QualifiedTicket qt1; - CompoundExpr lamList, x; - ApplyExpr ae; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - qt1 = mock( QualifiedTicket.class ); - when( qt1.getNumAtom() ).thenThrow( new NotDerivableException( "blub" ) ); - when( qt1.getOutputValue() ).thenThrow( new NotDerivableException( "blub" ) ); - when( qt1.visit( dnv ) ).thenReturn( new CompoundExpr( qt1 ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( qt1 ) ); - - lamList = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lamList ); - - x = dnv.accept( new CompoundExpr( ae ) ); - assertEquals( new CompoundExpr( ae ), x ); - } - - @Test - public void appNonFinalResultPreservesAppWithNewLam() throws HasFailedException, NotBoundException { - - Prototype cSign, sign; - CompoundExpr cLam, lam, cnd, x, val, varx; - ApplyExpr cAe, ae, ae1; - Block body, body1; - NativeLambdaExpr lam1; - CondExpr cnd1; - - - cSign = new Prototype(); - cSign.addOutput( new ReduceVar( "out", null ) ); - cSign.addParam( new NameExpr( "task" ) ); - - cLam = new CompoundExpr( new ForeignLambdaExpr( cSign, LANGID_BASH, "blub" ) ); - - cAe = new ApplyExpr( 1, false ); - cAe.setTaskExpr( cLam ); - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - varx = new CompoundExpr( new NameExpr( "x" ) ); - - cnd = new CompoundExpr( new CondExpr( new CompoundExpr( cAe ), varx, varx ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), cnd ); - body.putAssign( new NameExpr( "x" ), new CompoundExpr( new StringExpr( "A" ) ) ); - - lam = new CompoundExpr( new NativeLambdaExpr( sign, body ) ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( lam ); - - x = dnv.accept( new CompoundExpr( ae ) ); - - assertEquals( 1, x.getNumSingleExpr() ); - assertTrue( x.getSingleExpr( 0 ) instanceof ApplyExpr ); - - ae1 = ( ApplyExpr )x.getSingleExpr( 0 ); - - assertEquals( 1, ae1.getChannel() ); - assertTrue( ae1.getTaskExpr().getSingleExpr( 0 ) instanceof NativeLambdaExpr ); - - lam1 = ( NativeLambdaExpr )ae1.getTaskExpr().getSingleExpr( 0 ); - - assertEquals( sign, lam1.getPrototype() ); - - body1 = lam1.getBodyBlock(); - val = body1.getExpr( new NameExpr( "out" ) ); - - assertEquals( 1, val.getNumSingleExpr() ); - assertTrue( val.getSingleExpr( 0 ) instanceof CondExpr ); - - cnd1 = ( CondExpr )val.getSingleExpr( 0 ); - - assertEquals( varx, cnd1.getThenExpr() ); - assertEquals( varx, cnd1.getElseExpr() ); - assertTrue( cnd1.getIfExpr().getSingleExpr( 0 ) instanceof QualifiedTicket ); - } - - @Test - public void nestedAppUndergoesReduction() throws HasFailedException, NotBoundException, NotDerivableException { - - Prototype sign; - CompoundExpr lam1, lam2, x, y, a; - ApplyExpr ae1, ae2; - Block body2; - QualifiedTicket qt1; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - - lam1 = new CompoundExpr( new ForeignLambdaExpr( sign, LANGID_BASH, "blub" ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( lam1 ); - - body2 = new Block(); - body2.putAssign( new NameExpr( "out" ), new CompoundExpr( ae1 ) ); - - lam2 = new CompoundExpr( new NativeLambdaExpr( sign, body2 ) ); - - ae2 = new ApplyExpr( 1, false ); - ae2.setTaskExpr( lam2 ); - - qt1 = mock( QualifiedTicket.class ); - when( qt1.visit( dnv ) ).thenReturn( dnv.accept( qt1 ) ); - when( qt1.getChannel() ).thenReturn( 1 ); - when( qt1.getNumAtom() ).thenThrow( new NotDerivableException( "bla" ) ); - when( qt1.getOutputValue() ).thenThrow( new NotDerivableException( "blub" ) ); - - when( ticketSrc.requestTicket( - any( BaseRepl.class ), - any( UUID.class ), - any( ApplyExpr.class ) ) ).thenReturn( qt1 ); - - x = dnv.accept( new CompoundExpr( ae2 ) ); - - assertEquals( 1, x.getNumSingleExpr() ); - assertTrue( x.getSingleExpr( 0 ) instanceof ApplyExpr ); - - a = new CompoundExpr( new StringExpr( "A" ) ); - - reset( qt1 ); - when( qt1.visit( dnv ) ).thenReturn( dnv.accept( qt1 ) ); - when( qt1.getChannel() ).thenReturn( 1 ); - when( qt1.getNumAtom() ).thenReturn( 1 ); - when( qt1.getOutputValue() ).thenReturn( a ); - - y = dnv.accept( x ); - assertEquals( a.normalize(), y.normalize() ); - } - - /*recursion_terminates( CreateTicket ) -> - DecSign = {sign, [{param, "i1", false, false}, - {param, "converge", false, true}], - [], - [{param, "i", false, false}]}, - DecBody = {forbody, bash, "blub"}, - DecLam = [{lam, ?LOC, DecSign, DecBody}], - RecSign = {sign, [{param, "out", false, true}], - [], [{param, "i", false, false}]}, - RecBody = {natbody, - #{"i1" => [{app, ?LOC, 1, DecLam, #{"i" => [{var, "i"}]}}], - "converge" => [{app, ?LOC, 2, DecLam, #{"i" => [{var, "i"}]}}], - "out" => [{cnd, [{var, "converge"}], - [{str, "last"}], - [{str, "next to"}, - {app, ?LOC, 1, [{var, "rec"}], - #{"i" => [{var, "i1"}]}}]}]}}, - RecLam = [{lam, ?LOC, RecSign, RecBody}], - App = [{app, ?LOC, 1, [{var, "rec"}], #{"i" => [{str, "2"}]}}], - Context0 = put( "rec", RecLam, #{} ), - R = eval( App, Context0, CreateTicket ), - [Ref0] = get_ticket_id_list( R ), - Context1 = put( {2, Ref0}, [], Context0 ), - S = eval( R, Context1, CreateTicket ), - [Ref1] = get_ticket_id_list( S ), - Context2 = put( {1, Ref1}, [{str, "1"}], Context1 ), - T = eval( S, Context2, CreateTicket ), - [Ref2] = get_ticket_id_list( T ), - Context3 = put( {2, Ref2}, [{str, "true"}], Context2 ), - U = eval( T, Context3, CreateTicket ), - ?_assertEqual( [{str, "next to"}, {str, "last"}], U ). */ - - @Ignore - @Test - public void recursionTerminates() throws HasFailedException, NotBoundException, NotDerivableException { - - Prototype decSign, recSign; - ForeignLambdaExpr decLam; - Block recBody; - ApplyExpr ae1, ae2, recCall, ae; - CondExpr cnd; - CompoundExpr nextToCall, r, s; - NativeLambdaExpr recLam; - QualifiedTicket qt1, qt2; - - decSign = new Prototype(); - decSign.addOutput( new NameExpr( "i1" ) ); - decSign.addOutput( new ReduceVar( "converge", null ) ); - decSign.addParam( new NameExpr( "task" ) ); - decSign.addParam( new NameExpr( "i" ) ); - - decLam = new ForeignLambdaExpr( decSign, LANGID_BASH, "blub" ); - - recSign = new Prototype(); - recSign.addOutput( new ReduceVar( "out", null ) ); - recSign.addParam( new NameExpr( "task" ) ); - recSign.addParam( new NameExpr( "i" ) ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( new CompoundExpr( decLam ) ); - ae1.putAssign( new NameExpr( "i" ), new CompoundExpr( new NameExpr( "i" ) ) ); - - ae2 = new ApplyExpr( 2, false ); - ae2.setTaskExpr( new CompoundExpr( decLam ) ); - ae2.putAssign( new NameExpr( "i" ), new CompoundExpr( new NameExpr( "i" ) ) ); - - recCall = new ApplyExpr( 1, false ); - recCall.setTaskExpr( new CompoundExpr( new NameExpr( "rec" ) ) ); - recCall.putAssign( new NameExpr( "i" ), new CompoundExpr( new NameExpr( "i1" ) ) ); - - nextToCall = new CompoundExpr(); - nextToCall.addSingleExpr( new StringExpr( "next to" ) ); - nextToCall.addSingleExpr( recCall ); - - cnd = new CondExpr( - new CompoundExpr( new NameExpr( "converge" ) ), - new CompoundExpr( new StringExpr( "last" ) ), - nextToCall ); - - recBody = new Block(); - recBody.putAssign( new NameExpr( "i1" ), new CompoundExpr( ae1 ) ); - recBody.putAssign( new NameExpr( "converge" ), new CompoundExpr( ae2 ) ); - recBody.putAssign( new NameExpr( "out" ), new CompoundExpr( cnd ) ); - - recLam = new NativeLambdaExpr( recSign, recBody ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( new CompoundExpr( new NameExpr( "rec" ) ) ); - ae.putAssign( new NameExpr( "i" ), new CompoundExpr( new StringExpr( "2" ) ) ); - - tlc.putAssign( new NameExpr( "rec" ), new CompoundExpr( recLam ) ); - System.out.println( tlc ); - - qt1 = mock( QualifiedTicket.class ); - when( qt1.visit( dnv ) ).thenReturn( dnv.accept( qt1 ) ); - when( qt1.getChannel() ).thenReturn( 2 ); - when( qt1.getNumAtom() ).thenThrow( new NotDerivableException( "bla" ) ); - when( qt1.getOutputValue() ).thenThrow( new NotDerivableException( "blub" ) ); - - reset( ticketSrc ); - when( ticketSrc.requestTicket( any( BaseRepl.class ), any( UUID.class ), any( ApplyExpr.class ) ) ) - .thenReturn( qt1 ); - - r = dnv.accept( new CompoundExpr( ae ) ); - - reset( qt1 ); - when( qt1.visit( dnv ) ).thenReturn( dnv.accept( qt1 ) ); - when( qt1.getChannel() ).thenReturn( 2 ); - when( qt1.getNumAtom() ).thenReturn( 0 ); - when( qt1.getOutputValue() ).thenReturn( new CompoundExpr() ); - - qt2 = mock( QualifiedTicket.class ); - when( qt2.visit( dnv ) ).thenReturn( dnv.accept( qt2 ) ); - when( qt2.getChannel() ).thenReturn( 1 ); - when( qt2.getNumAtom() ).thenReturn( 1 ); - when( qt2.getOutputValue() ).thenReturn( new CompoundExpr( new StringExpr( "1" ) ) ); - - reset( ticketSrc ); - when( ticketSrc.requestTicket( any( BaseRepl.class ), any( UUID.class ), any( ApplyExpr.class ) ) ) - .thenReturn( qt2 ); - - s = dnv.accept( r ); - System.out.println( s ); - } - -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ApplyExprTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ApplyExprTest.java deleted file mode 100644 index d69e702..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/ApplyExprTest.java +++ /dev/null @@ -1,37 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import static org.junit.Assert.*; - -import org.junit.Test; - -public class ApplyExprTest { - - @SuppressWarnings("static-method") - @Test - public void copyConstructorShouldWork() { - - ApplyExpr ae1, ae2; - NativeLambdaExpr lam; - Prototype sign; - Block body; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "inp" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new NameExpr( "inp" ) ) ); - - lam = new NativeLambdaExpr( sign, body ); - - ae1 = new ApplyExpr( 1, false ); - ae1.setTaskExpr( new CompoundExpr( lam ) ); - ae1.putAssign( new NameExpr( "inp" ), new CompoundExpr( new StringExpr( "A" ) ) ); - - ae2 = new ApplyExpr( ae1 ); - - assertEquals( ae1, ae2 ); - assertNotSame( ae1, ae2 ); - } -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CompoundExprTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CompoundExprTest.java deleted file mode 100644 index 5c767b7..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CompoundExprTest.java +++ /dev/null @@ -1,111 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import static org.junit.Assert.*; - -import org.junit.Test; - -public class CompoundExprTest { - - @SuppressWarnings("static-method") - @Test - public void equalsReturnsTrueOnCompoundExprWithEqualContent() { - - CompoundExpr a, b; - - a = new CompoundExpr( new StringExpr( "A" ) ); - b = new CompoundExpr( new StringExpr( "A" ) ); - - assertEquals( a, b ); - assertEquals( b, a ); - } - - @SuppressWarnings("static-method") - @Test - public void equalsReturnsFalseOnCompoundExprWithNonEqualContent() { - - CompoundExpr a, b; - - a = new CompoundExpr( new StringExpr( "A" ) ); - b = new CompoundExpr( new StringExpr( "B" ) ); - - assertNotEquals( a, b ); - assertNotEquals( b, a ); - } - - @SuppressWarnings("static-method") - @Test - public void equalsReturnsFalseOnNull() { - - CompoundExpr a; - - a = new CompoundExpr(); - assertNotEquals( a, null ); - } - - @SuppressWarnings("static-method") - @Test - public void equalsReturnsFalseOnCompoundExprWithDifferingSize() { - - CompoundExpr a, b; - - a = new CompoundExpr(); - b = new CompoundExpr( new StringExpr( "B" ) ); - - assertNotEquals( a, b ); - assertNotEquals( b, a ); - } - - @SuppressWarnings("static-method") - @Test - public void equalsReturnsFalseOnNonCompoundExpr() { - - CompoundExpr a; - - a = new CompoundExpr(); - assertNotEquals( a, "blub" ); - } - - @SuppressWarnings("static-method") - @Test - public void condIsNonNormal() { - - CondExpr cnd; - CompoundExpr ce; - - cnd = new CondExpr( new CompoundExpr(), new CompoundExpr(), new CompoundExpr() ); - ce = new CompoundExpr( cnd ); - - assertFalse( ce.isNormal() ); - } - - @SuppressWarnings("static-method") - @Test - public void copyApplyExprShouldWork() { - - ApplyExpr ae; - NativeLambdaExpr lam; - Prototype sign; - Block body; - CompoundExpr ce1, ce2; - - sign = new Prototype(); - sign.addOutput( new NameExpr( "out" ) ); - sign.addParam( new NameExpr( "task" ) ); - sign.addParam( new NameExpr( "inp" ) ); - - body = new Block(); - body.putAssign( new NameExpr( "out" ), new CompoundExpr( new NameExpr( "inp" ) ) ); - - lam = new NativeLambdaExpr( sign, body ); - - ae = new ApplyExpr( 1, false ); - ae.setTaskExpr( new CompoundExpr( lam ) ); - ae.putAssign( new NameExpr( "inp" ), new CompoundExpr( new StringExpr( "A" ) ) ); - - ce1 = new CompoundExpr( ae ); - ce2 = new CompoundExpr( ce1 ); - - assertEquals( ce1, ce2 ); - assertNotSame( ce1, ce2 ); - } -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CondExprTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CondExprTest.java deleted file mode 100644 index 7a609b5..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/CondExprTest.java +++ /dev/null @@ -1,66 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import org.junit.Test; - -import static org.junit.Assert.*; -import static org.mockito.Mockito.*; - -public class CondExprTest { - - @SuppressWarnings("static-method") - @Test - public void constructorSetsThreeExpr() { - - CondExpr condExpr; - CompoundExpr ifExpr, thenExpr, elseExpr; - - ifExpr = mock( CompoundExpr.class ); - thenExpr = mock( CompoundExpr.class ); - elseExpr = mock( CompoundExpr.class ); - - condExpr = new CondExpr( ifExpr, thenExpr, elseExpr ); - assertEquals( ifExpr, condExpr.getIfExpr() ); - assertEquals( thenExpr, condExpr.getThenExpr() ); - assertEquals( elseExpr, condExpr.getElseExpr() ); - } - - @SuppressWarnings({ "static-method", "unused" }) - @Test( expected=IllegalArgumentException.class ) - public void constructorThrowsIaeOnNullIfExpr() { - - CondExpr condExpr; - CompoundExpr thenExpr, elseExpr; - - thenExpr = mock( CompoundExpr.class ); - elseExpr = mock( CompoundExpr.class ); - - condExpr = new CondExpr( null, thenExpr, elseExpr ); - } - - @SuppressWarnings({ "static-method", "unused" }) - @Test( expected=IllegalArgumentException.class ) - public void constructorThrowsIaeOnNullThenExpr() { - - CondExpr condExpr; - CompoundExpr ifExpr, elseExpr; - - ifExpr = mock( CompoundExpr.class ); - elseExpr = mock( CompoundExpr.class ); - - condExpr = new CondExpr( ifExpr, null, elseExpr ); - } - - @SuppressWarnings({ "static-method", "unused" }) - @Test( expected=IllegalArgumentException.class ) - public void constructorThrowsIaeOnNullElseExpr() { - - CondExpr condExpr; - CompoundExpr ifExpr, thenExpr; - - ifExpr = mock( CompoundExpr.class ); - thenExpr = mock( CompoundExpr.class ); - - condExpr = new CondExpr( ifExpr, thenExpr, null ); - } - -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/EnumHelperTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/EnumHelperTest.java deleted file mode 100644 index c7dc345..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/EnumHelperTest.java +++ /dev/null @@ -1,210 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import org.junit.Test; - -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -public class EnumHelperTest { - - @SuppressWarnings({ "static-method", "unused" }) - @Test( expected = IllegalArgumentException.class ) - public void ConstructorShouldThrowIaeOnNullTaskExpr() { - - EnumHelper eh; - BaseBlock bindingBlock; - - bindingBlock = mock( BaseBlock.class ); - - eh = new EnumHelper( null, bindingBlock ); - } - - @SuppressWarnings({ "static-method", "unused" }) - @Test( expected = IllegalArgumentException.class ) - public void ConstructorShouldThrowIaeOnNullBindingBlock() { - - EnumHelper eh; - CompoundExpr ce; - - ce = mock( CompoundExpr.class ); - - eh = new EnumHelper( ce, null ); - } - - @SuppressWarnings({ "static-method", "unused" }) - @Test( expected = IllegalArgumentException.class ) - public void ConstructorShouldThrowIaeOnNil() { - - EnumHelper eh; - CompoundExpr ce; - BaseBlock bindingBlock; - - bindingBlock = mock( BaseBlock.class ); - ce = new CompoundExpr(); - - eh = new EnumHelper( ce, bindingBlock ); - } - - @SuppressWarnings({ "unused", "static-method" }) - @Test( expected = IllegalArgumentException.class ) - public void ConstructorShouldThrowIaeOnNonLambdaExpr() { - - EnumHelper eh; - CompoundExpr ce; - BaseBlock bindingBlock; - - bindingBlock = mock( BaseBlock.class ); - ce = new CompoundExpr( mock( NameExpr.class ) ); - - eh = new EnumHelper( ce, bindingBlock ); - } - - @SuppressWarnings("static-method") - @Test - public void getCardinalityShouldWorkWithTwoCorrelatedParameters() throws NotDerivableException { - - Prototype prototype; - CorrelParam cp; - CompoundExpr taskExpr; - LambdaExpr le; - Block bindingBlock; - NameExpr a, b, c, d; - CompoundExpr ce; - EnumHelper eh; - - // create name expressions for four parameters - a = new NameExpr( "a" ); - b = new NameExpr( "b" ); - c = new NameExpr( "c" ); - d = new NameExpr( "d" ); - - // generate prototype - prototype = new Prototype(); - - cp = new CorrelParam(); - cp.addName( a ); - cp.addName( b ); - prototype.addParam( cp ); - - cp = new CorrelParam(); - cp.addName( c ); - cp.addName( d ); - prototype.addParam( cp ); - - // create lambda expression - le = mock( LambdaExpr.class ); - when( le.getPrototype() ).thenReturn( prototype ); - - // create task expression - taskExpr = new CompoundExpr( le ); - - // bind parameter values - bindingBlock = new Block(); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "a1" ) ); - ce.addSingleExpr( new StringExpr( "a2" ) ); - ce.addSingleExpr( new StringExpr( "a3" ) ); - bindingBlock.putAssign( a, ce ); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "b1" ) ); - ce.addSingleExpr( new StringExpr( "b2" ) ); - ce.addSingleExpr( new StringExpr( "b3" ) ); - bindingBlock.putAssign( b, ce ); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "c1" ) ); - ce.addSingleExpr( new StringExpr( "c2" ) ); - ce.addSingleExpr( new StringExpr( "c3" ) ); - ce.addSingleExpr( new StringExpr( "c4" ) ); - ce.addSingleExpr( new StringExpr( "c5" ) ); - bindingBlock.putAssign( c, ce ); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "d1" ) ); - ce.addSingleExpr( new StringExpr( "d2" ) ); - ce.addSingleExpr( new StringExpr( "d3" ) ); - ce.addSingleExpr( new StringExpr( "d4" ) ); - ce.addSingleExpr( new StringExpr( "d5" ) ); - bindingBlock.putAssign( d, ce ); - - eh = new EnumHelper( taskExpr, bindingBlock ); - assertEquals( 15, eh.getCardinality() ); - } - - @SuppressWarnings("static-method") - @Test - public void getCardinalityShouldWorkWithTaskCorrelation() throws NotDerivableException { - - Prototype prototype; - CorrelParam cp; - CompoundExpr taskExpr; - LambdaExpr le1, le2; - Block bindingBlock; - NameExpr task, b, c, d; - CompoundExpr ce; - EnumHelper eh; - - // create name expressions for four parameters - task = new NameExpr( "task" ); - b = new NameExpr( "a" ); - c = new NameExpr( "b" ); - d = new NameExpr( "c" ); - - // generate prototype - prototype = new Prototype(); - - cp = new CorrelParam(); - cp.addName( task ); - cp.addName( b ); - prototype.addParam( cp ); - - cp = new CorrelParam(); - cp.addName( c ); - cp.addName( d ); - prototype.addParam( cp ); - - // create lambda expression - le1 = mock( LambdaExpr.class ); - when( le1.getPrototype() ).thenReturn( prototype ); - when( le1.getNumAtom() ).thenReturn( 1 ); - - le2 = mock( LambdaExpr.class ); - when( le2.getPrototype() ).thenReturn( prototype ); - when( le2.getNumAtom() ).thenReturn( 1 ); - - // create task expression - taskExpr = new CompoundExpr( le1 ); - taskExpr.addSingleExpr( le2 ); - - // bind parameter values - bindingBlock = new Block(); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "b1" ) ); - ce.addSingleExpr( new StringExpr( "b2" ) ); - bindingBlock.putAssign( b, ce ); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "c1" ) ); - ce.addSingleExpr( new StringExpr( "c2" ) ); - ce.addSingleExpr( new StringExpr( "c3" ) ); - ce.addSingleExpr( new StringExpr( "c4" ) ); - ce.addSingleExpr( new StringExpr( "c5" ) ); - bindingBlock.putAssign( c, ce ); - - ce = new CompoundExpr(); - ce.addSingleExpr( new StringExpr( "d1" ) ); - ce.addSingleExpr( new StringExpr( "d2" ) ); - ce.addSingleExpr( new StringExpr( "d3" ) ); - ce.addSingleExpr( new StringExpr( "d4" ) ); - ce.addSingleExpr( new StringExpr( "d5" ) ); - bindingBlock.putAssign( d, ce ); - - eh = new EnumHelper( taskExpr, bindingBlock ); - assertEquals( 10, eh.getCardinality() ); - } - -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HashHelperTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HashHelperTest.java deleted file mode 100644 index b8ad669..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/HashHelperTest.java +++ /dev/null @@ -1,60 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import org.junit.Test; -import static org.junit.Assert.assertNotEquals; - -import java.util.UUID; - - -public class HashHelperTest { - - - @SuppressWarnings("static-method") - @Test - public void hashPreservesDifferencesBetweenStrings() { - - String a, b; - long h1, h2; - - a = "bla"; - b = "blub"; - - h1 = HashHelper.add( 0L, a ); - h2 = HashHelper.add( 0L, b ); - - assertNotEquals( h1, h2 ); - } - - @SuppressWarnings("static-method") - @Test - public void hashPreservesDifferencesBetweenTickets() { - - Ticket a, b; - long h1, h2; - UUID uuid; - ForeignLambdaExpr lambda1, lambda2; - Block bindingBlock; - Prototype prototype; - String lang; - String body1, body2; - - uuid = UUID.randomUUID(); - bindingBlock = new Block(); - - prototype = new Prototype(); - lang = ForeignLambdaExpr.LANGID_BASH; - body1 = "bla"; - body2 = "blub"; - - lambda1 = new ForeignLambdaExpr( prototype, lang, body1 ); - lambda2 = new ForeignLambdaExpr( prototype, lang, body2 ); - - a = new Ticket( lambda1, bindingBlock, uuid ); - b = new Ticket( lambda2, bindingBlock, uuid ); - - h1 = HashHelper.add( 0L, a ); - h2 = HashHelper.add( 0L, b ); - - assertNotEquals( h1, h2 ); - } -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NativeLambdaExprTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NativeLambdaExprTest.java deleted file mode 100644 index 9e73c97..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/NativeLambdaExprTest.java +++ /dev/null @@ -1,5 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -public class NativeLambdaExprTest { - -} diff --git a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/StringExprTest.java b/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/StringExprTest.java deleted file mode 100644 index 2b60123..0000000 --- a/cuneiform-core/src/test/java/de/huberlin/wbi/cuneiform/core/semanticmodel/StringExprTest.java +++ /dev/null @@ -1,21 +0,0 @@ -package de.huberlin.wbi.cuneiform.core.semanticmodel; - -import static org.junit.Assert.*; - -import org.junit.Test; - -public class StringExprTest { - - @SuppressWarnings("static-method") - @Test - public void EqualReturnsTrueOnStringExprWithEqualContent() { - - StringExpr a, b; - - a = new StringExpr( "A" ); - b = new StringExpr( "A" ); - - assertEquals( a, b ); - assertEquals( b, a ); - } -} diff --git a/cuneiform-dist/.gitignore b/cuneiform-dist/.gitignore deleted file mode 100644 index 6afedc5..0000000 --- a/cuneiform-dist/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/target -/bin -/.classpath -/.settings -/.project diff --git a/cuneiform-dist/pom.xml b/cuneiform-dist/pom.xml deleted file mode 100644 index 73314e7..0000000 --- a/cuneiform-dist/pom.xml +++ /dev/null @@ -1,129 +0,0 @@ - - 4.0.0 - - - de.hu-berlin.wbi.cuneiform - cuneiform - 2.0.4-RELEASE - - - cuneiform-dist - - - - de.hu-berlin.wbi.cuneiform - cuneiform-cmdline - ${project.version} - - - de.hu-berlin.wbi.cuneiform - cuneiform-cfide - ${project.version} - - - de.hu-berlin.wbi.cuneiform - cuneiform-logview - ${project.version} - - - - - - - - src/main/scripts - - log4j.properties - - - - - - - maven-assembly-plugin - 2.4 - - - generate-dist - package - - single - - - - - - src/main/assemblies/bin.xml - - false - - - - org.apache.maven.plugins - maven-jar-plugin - 2.6 - - - true - - true - - - development - ${project.url} - de.huberlin.wbi.cuneiform.cmdline.main.Main - - - - - - org.apache.maven.plugins - maven-shade-plugin - 2.4.2 - - - - - src/main/scripts/log4j.properties - log4j.properties - - - false - true - - - - package - - shade - - - - - - org.apache.maven.plugins - maven-javadoc-plugin - 2.10.3 - - - attach-javadocs - - jar - - - - - true - - de.hu-berlin.wbi.cuneiform:* - - - - - - - diff --git a/cuneiform-dist/src/main/assemblies/bin.xml b/cuneiform-dist/src/main/assemblies/bin.xml deleted file mode 100644 index d47c473..0000000 --- a/cuneiform-dist/src/main/assemblies/bin.xml +++ /dev/null @@ -1,51 +0,0 @@ - - bin - - dir - tar.gz - zip - - - - - src/main/scripts - cuneiform-${project.version} - - - - false - cuneiform-${project.version} - - - - - cuneiform-${project.version}/lib - false - true - false - false - - de.hu-berlin.wbi.cuneiform:cuneiform-cmdline - de.hu-berlin.wbi.cuneiform:cuneiform-cfide - de.hu-berlin.wbi.cuneiform:cuneiform-logview - - - - - cuneiform-${project.version}/bin - false - true - false - false - - de.hu-berlin.wbi.cuneiform:cuneiform-cmdline - de.hu-berlin.wbi.cuneiform:cuneiform-cfide - de.hu-berlin.wbi.cuneiform:cuneiform-logview - - - - - diff --git a/cuneiform-dist/src/main/cuneiform/variant-call11.cf b/cuneiform-dist/src/main/cuneiform/variant-call11.cf deleted file mode 100644 index cfad539..0000000 --- a/cuneiform-dist/src/main/cuneiform/variant-call11.cf +++ /dev/null @@ -1,193 +0,0 @@ -% VARIANT-CALL -% -% A variant calling workflow. -% -% Sample data can be obtained from: -% ftp://ftp.1000genomes.ebi.ac.uk/vol1/ftp/phase3/data/HG02025/sequence_read/ -% -% The HG38 reference genome can be downloaded from -% http://hgdownload.soe.ucsc.edu/goldenPath/hg38/chromosomes/ -% -% An Annovar HG38 database is expected to reside in -% /opt/data/annodb_hg38 -% -% In addition to a Cuneiform interpreter the following tools need to be -% installed to run this analysis: -% - FastQC 0.11.4 -% - Bowtie2 2.2.6 -% - SAMtools 1.2 -% - VarScan 2.3.9 -% - Annovar - -%% ============================================================ -%% Task definitions -%% ============================================================ - -% untar -deftask untar( : tar( File ) )in bash *{ - tar xf $tar - out=`tar tf $tar` -}* - -% gunzip -deftask gunzip( out( File ) : gz( File ) )in bash *{ - gzip -c -d $gz > $out -}* - -// fastqc -deftask fastqc( zip( File ) : fastq( File ) )in bash *{ - fastqc -f fastq --noextract -o ./ $fastq - zip=`ls *.zip` -}* - -% bowtie2 -deftask bowtie2-build( idx( File ) : fa( File ) ) -in bash *{ - bowtie2-build $fa bt2idx - tar cf $idx --remove-files bt2idx.* -}* - -deftask bowtie2-align( - bam( File ) -: idx( File ) [fastq1( File ) fastq2( File )] )in bash *{ - - tar xf $idx - bowtie2 -D 5 -R 1 -N 0 -L 22 -i S,0,2.50 --no-unal -x bt2idx \ - -1 $fastq1 -2 $fastq2 -S - | samtools view -b - > $bam - rm bt2idx.* -}* - -% samtools -deftask samtools-faidx( fai( File ) : fa( File ) )in bash *{ - samtools faidx $fa - fai=$fa.fai -}* - -deftask samtools-sort( sortedbam( File ) : bam( File ) )in bash *{ - sortedbam=alignment-sorted.bam - samtools sort -m 1G $bam alignment-sorted -}* - -deftask samtools-mpileup( - mpileup( File ) -: bam( File ) [fa( File ) fai( File )] )in bash *{ - - ln -sf $fai $fa.fai - samtools mpileup -f $fa $bam > $mpileup -}* - -deftask samtools-merge( merged( File ) : )in bash *{ - if [ ${#bam[@]} -eq "0" ] - then - echo "No files to merge." >&2 - exit -1 - else - if [ ${#bam[@]} -eq "1" ] - then - merged=$bam - else - samtools merge -f $merged ${bam[@]} - fi - fi -}* - -% varscan -deftask varscan( vcf( File ) : mpileup( File ) )in bash *{ - varscan mpileup2snp $mpileup --output-vcf --p-value 99e-02 > $vcf -}* - -% annovar -deftask annovar( - fun( File ) exonicfun( File ) -: db buildver )in bash *{ - - fun=table.variant_function - exonicfun=table.exonic_variant_function - cat ${vcf[@]} | \ - convert2annovar.pl -format vcf4 - | \ - annotate_variation.pl -buildver $buildver -geneanno -outfile table - $db -}* - -deftask per-chromosome( - vcf( File ) - : [fa( File ) fai( File ) idx( File )] - ) { - - sortedbam = per-fastq( - fa: fa, - idx: idx - fastq1: fastq1, - fastq2: fastq2 ); - - mergedbam = samtools-merge( bam: sortedbam ); - - mpileup = samtools-mpileup( - bam: mergedbam - fa: fa - fai: fai ); - - vcf = varscan( mpileup: mpileup ); -} - -deftask per-fastq( - sortedbam - : idx( File ) [fastq1( File ) fastq2( File )] ) { - - bam = bowtie2-align( - idx: idx, - fastq1: fastq1, - fastq2: fastq2 ); - - sortedbam = samtools-sort( bam: bam ); -} - - -%% ============================================================ -%% Input data -%% ============================================================ - -hg38-tar = 'hg38/hg38.tar'; - -fastq1-gz = 'kgenomes/SRR359188_1.filt.fastq.gz' - 'kgenomes/SRR359195_1.filt.fastq.gz'; - -fastq2-gz = 'kgenomes/SRR359188_2.filt.fastq.gz' - 'kgenomes/SRR359195_2.filt.fastq.gz'; - -db = '/opt/data/annodb_hg38'; - - -%% ============================================================ -%% Workflow definition -%% ============================================================ - -fa = untar( tar: hg38-tar ); -% fa = gunzip( gz: 'hg38/chr22.fa.gz' ); - -fastq1 = gunzip( gz: fastq1-gz ); -fastq2 = gunzip( gz: fastq2-gz ); - -qc = fastqc( fastq: fastq1 fastq2 ); - -bt2idx = bowtie2-build( fa: fa ); -fai = samtools-faidx( fa: fa ); - -vcf = per-chromosome( - fa: fa, - fai: fai, - idx: bt2idx, - fastq1: fastq1, - fastq2: fastq2 ); - -fun exonicfun = annovar( - vcf: vcf - db: db - buildver: 'hg38' ); - - -%% ============================================================ -%% Query -%% ============================================================ - -fun exonicfun qc; - diff --git a/cuneiform-dist/src/main/scripts/log4j.properties b/cuneiform-dist/src/main/scripts/log4j.properties deleted file mode 100644 index ad2b6d9..0000000 --- a/cuneiform-dist/src/main/scripts/log4j.properties +++ /dev/null @@ -1,15 +0,0 @@ -log4j.rootLogger=INFO, StdOut - -log4j.logger.statLogger=DEBUG, StatLog -log4j.additivity.statLogger=false - -log4j.appender.StdOut=org.apache.log4j.ConsoleAppender -log4j.appender.StdOut.layout=org.apache.log4j.PatternLayout -log4j.appender.StdOut.layout.ConversionPattern=%-5p %m%n - -log4j.appender.StatLog=org.apache.log4j.RollingFileAppender -log4j.appender.StatLog.File=/tmp/cuneiform-stat.log -log4j.appender.StatLog.MaxFileSize=100GB -log4j.appender.StatLog.MaxBackupIndex=1 -log4j.appender.StatLog.layout=org.apache.log4j.PatternLayout -log4j.appender.StatLog.layout.ConversionPattern=%m%n diff --git a/include/cuneiform.hrl b/include/cuneiform.hrl new file mode 100644 index 0000000..dd0492e --- /dev/null +++ b/include/cuneiform.hrl @@ -0,0 +1,13 @@ +-define( BUILD, "2016-04-11" ). + +-define( RED( Str ), "\e[31m" ++ Str ++ "\e[0m" ). +-define( BRED( Str ), "\e[1;31m" ++ Str ++ "\e[0m" ). +-define( GRN( Str ), "\e[32m" ++ Str ++ "\e[0m" ). +-define( YLW( Str ), "\e[33m" ++ Str ++ "\e[0m" ). +-define( BYLW( Str ), "\e[1;33m" ++ Str ++ "\e[0m" ). +-define( BLU( Str ), "\e[1;34m" ++ Str ++ "\e[0m" ). + +-define( BASEDIR, "/tmp/cf" ). +-define( WORK, "work" ). +-define( REPO, "repo" ). + diff --git a/pom.xml b/pom.xml deleted file mode 100644 index dee1185..0000000 --- a/pom.xml +++ /dev/null @@ -1,137 +0,0 @@ - - - - - - - 4.0.0 - de.hu-berlin.wbi.cuneiform - cuneiform - 2.0.4-RELEASE - pom - cuneiform - A Functional Workflow Language. - -Cuneiform is a workflow specification language which makes it easy to integrate heterogeneous tools and libraries and exploit data parallelism. Users do not have to create heavy-weight wrappers for establised tools or to reimplement them. Instead, they apply their existing software to partitioned data. Using the Hi-WAY application master Cuneiform can be executed on Hadoop YARN which makes it suitable for large scale data analysis. - -Cuneiform comes in the form of a functional programming language with a Foreign Function Interface (FFI) that lets users create functions in any suitable scripting language and apply these functions in a uniform way. - -Data paralelism is expressed by applying map, cross-product, dot-product, or combinations of the aforementioned algorithmic skeletons to collections of black-box data. - https://github.com/joergen7/cuneiform - - - Apache License, Version 2.0 - http://www.apache.org/licenses/LICENSE-2.0.txt - repo - - - - https://github.com/joergen7/cuneiform.git - - - - - Jörgen Brandt - brandjoe@informatik.hu-berlin.de - Humboldt-Universität zu Berlin - https://www.hu-berlin.de/ - - - Marc Bux - buxmarcn@informatik.hu-berlin.de - Humboldt-Universität zu Berlin - https://www.hu-berlin.de/ - - - Ulf Leser - leser@informatik.hu-berlin.de - Humboldt-Universität zu Berlin - https://www.hu-berlin.de/ - - - - - - ossrh - https://oss.sonatype.org/content/repositories/snapshots/ - - - ossrh - https://oss.sonatype.org/service/local/staging/deploy/maven2/ - - - - - UTF-8 - - - - cuneiform-core - cuneiform-addons - cuneiform-dist - cuneiform-cmdline - - - - - - - org.apache.maven.plugins - maven-compiler-plugin - 3.2 - - 1.7 - 1.7 - - - - - - - - - release-sign-artifacts - - - performRelease - true - - - - - - org.apache.maven.plugins - maven-gpg-plugin - 1.5 - - - sign-artifacts - verify - - sign - - - - - - - - - \ No newline at end of file diff --git a/rebar.config b/rebar.config new file mode 100644 index 0000000..646d6ee --- /dev/null +++ b/rebar.config @@ -0,0 +1,10 @@ +{cover_enabled, true}. +{deps, [ + {effi, "0.1.0-release", + {git, "https://github.com/joergen7/effi.git", + {branch, "master"}}}, + {getopt, "0.8.2", + {git, "https://github.com/jcomellas/getopt.git", + {branch, "master"}}} + ]}. +{escript_incl_apps, [getopt]}. diff --git a/src/cf_cre.erl b/src/cf_cre.erl new file mode 100644 index 0000000..b7d7112 --- /dev/null +++ b/src/cf_cre.erl @@ -0,0 +1,262 @@ +%% -*- erlang -*- +% +% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +% +% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. + +%% @author Jörgen Brandt +%% @doc The Cuneiform Runtime Environment (CRE). +%% ```+-------+ +-------+ +%% | Query | | Query | +%% +-------+ +-------+ +%% \ / +%% +-----+ +%% | CRE | +%% +-----+''' + + +-module( cf_cre ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + +-behaviour( gen_server ). + +%% ============================================================================= +%% Function Exports +%% ============================================================================= + +-export( [start_link/3, submit/2] ). + +-export( [code_change/3, handle_cast/2, handle_info/2, init/1, terminate/2, + handle_call/3] ). + +%% ============================================================================= +%% Includes +%% ============================================================================= + +-ifdef( TEST ). +-include_lib( "eunit/include/eunit.hrl" ). +-endif. + +%% ============================================================================= +%% Callback Function Declarations +%% ============================================================================= + + +-callback init( Arg::term() ) -> {ok, State::term()}. + +-callback handle_submit( Lam, Fa, R, DataDir, LibMap, ModState ) -> ok +when Lam :: cf_sem:lam(), + Fa :: #{string() => [cf_sem:str()]}, + R :: pos_integer(), + DataDir :: string(), + LibMap :: #{cf_sem:lang() => [string()]}, + ModState :: term(). + +%% ============================================================================= +%% Type Definitions +%% ============================================================================= + +-type ckey() :: {cf_sem:lam(), #{string() => [cf_sem:str()]}, string()}. + +-type response() :: {failed, pos_integer(), atom(), term()} + | {finished, #{atom() => term()}}. + +-type cre_state() :: {Mod::atom(), + SubscrMap::#{pos_integer() => sets:set( pid() )}, + ReplyMap::#{pos_integer() => response()}, + Cache::#{ckey() => cf_sem:fut()}, + R::pos_integer(), + LibMap::#{cf_sem:lang() => [string()]}, + ModState::term()}. + +-type submit() :: {submit, App::cf_sem:app(), Cwd::string()}. + +%% ============================================================================= +%% Generic Server Functions +%% ============================================================================= + +code_change( _OldVsn, State, _Extra ) -> {ok, State}. +handle_cast( Request, _State ) -> error( {bad_request, Request} ). +terminate( _Reason, _State ) -> ok. + +%% @doc Generates the initial state of the CRE. + +-spec init( {Mod, ModArg, LibMap} ) -> {ok, cre_state()} +when Mod :: atom(), + ModArg :: term(), + LibMap :: #{cf_sem:lang() => [string()]}. + +init( {Mod, ModArg, LibMap} ) +when is_atom( Mod ), is_map( LibMap ) -> + + SubscrMap = #{}, % mapping of a future to a set of subscriber pids + ReplyMap = #{}, % mapping of a future to a response + Cache = #{}, % cache mapping a cache key to a future + R = 1, % next id + + % initialize CRE + {ok, ModState} = apply( Mod, init, [ModArg] ), + + {ok, {Mod, SubscrMap, ReplyMap, Cache, R, LibMap, ModState}}. + +%% @doc On receiving a call containing a subission, a future is generated and +%% returned. +%% +%% When a submission request is received by the CRE, it returns a future by +%% either creating a future, and spawning a corresponding job, or by +%% returning a future from its cache, if a corresponding job has already +%% been started. +%% +%% Additionally, the source of the call is added to the subscriber list for +%% the corresponding future. All subscribers to a future eventually +%% receive a notification about the result of the submission. +%% +%% If the future is served from the cache and a reply has already been +%% received, the subscriber is immediately notified. +%% +-spec handle_call( Request, From, State ) -> {reply, cf_sem:fut(), cre_state()} +when Request :: submit(), + From :: {pid(), term()}, + State :: cre_state(). + +handle_call( {submit, App, Cwd}, + {Pid, _Tag}, + {Mod, SubscrMap, ReplyMap, Cache, R, LibMap, ModState} ) + +when is_tuple( App ), + is_list( Cwd ), + is_pid( Pid ), + is_atom( Mod ), + is_map( SubscrMap ), + is_map( ReplyMap ), + is_map( Cache ), + is_integer( R ), R > 0, + is_map( LibMap ) -> + + {app, _AppLine, _Channel, Lam, Fa} = App, + {lam, _LamLine, LamName, {sign, Lo, _Li}, _Body} = Lam, + + % construct cache key + Ckey = {Lam, Fa, Cwd}, + + case maps:is_key( Ckey, Cache ) of + + false -> + + % create new future + Fut = {fut, LamName, R, Lo}, + + % start process + apply( Mod, handle_submit, [Lam, Fa, R, Cwd, LibMap, ModState] ), + + SubscrMap1 = SubscrMap#{R => sets:from_list( [Pid] )}, + Cache1 = Cache#{Ckey => Fut}, + R1 = R+1, + + {reply, Fut, {Mod, SubscrMap1, ReplyMap, Cache1, R1, LibMap, ModState}}; + + true -> + + % retrieve future from cache + Fut = maps:get( Ckey, Cache ), + + {fut, _, S, _} = Fut, + #{S := SubscrSet} = SubscrMap, + + SubscrMap1 = SubscrMap#{S => sets:add_element( Pid, SubscrSet )}, + + case maps:is_key( S, ReplyMap ) of + false -> ok; + true -> Pid ! maps:get( S, ReplyMap ) + end, + + {reply, Fut, {Mod, SubscrMap1, ReplyMap, Cache, R, LibMap, ModState}} + end. + +%% Info Handler %% + +-spec handle_info( Info, State ) -> {noreply, cre_state()} +when Info :: response(), + State :: cre_state(). + +handle_info( Info={failed, Reason, S, Data}, + {Mod, SubscrMap, ReplyMap, Cache, R, LibMap, ModState} ) +when is_atom( Reason ), + is_integer( S ), S > 0 -> + + % retrieve subscriber set + #{S := SubscrSet} = SubscrMap, + + % notify subscribers + lists:foreach( fun( Subscr ) -> + Subscr ! {failed, Reason, S, Data} + end, + sets:to_list( SubscrSet ) ), + + ReplyMap1 = ReplyMap#{S => Info}, + + {noreply, {Mod, SubscrMap, ReplyMap1, Cache, R, LibMap, ModState}}; + +handle_info( Info={finished, Sum}, + {Mod, SubscrMap, ReplyMap, Cache, R, LibMap, ModState} ) -> + + % retrieve subscriber set + #{id := S} = Sum, + #{S := SubscrSet} = SubscrMap, + + % notify subscribers + lists:foreach( fun( Subscr ) -> + Subscr ! {finished, Sum} + end, + sets:to_list( SubscrSet ) ), + + + ReplyMap1 = ReplyMap#{S => Info}, + + {noreply, {Mod, SubscrMap, ReplyMap1, Cache, R, LibMap, ModState}}. + +%% ============================================================================= +%% API Functions +%% ============================================================================= + +-spec start_link( Mod, ModArg, LibMap ) -> {ok, pid()} | ignore | {error, Error} +when Mod :: atom(), + ModArg :: term(), + LibMap :: #{cf_sem:lang() => [string()]}, + Error :: {already_started, pid()} | term(). + +start_link( Mod, ModArg, LibMap ) +when is_atom( Mod ), is_map( LibMap ) -> + gen_server:start_link( {local, cre}, ?MODULE, {Mod, ModArg, LibMap}, [] ). + + +-spec submit( App, Cwd ) -> cf_sem:fut() +when App :: cf_sem:app(), + Cwd :: string(). + +submit( App, Cwd ) +when is_tuple( App ), is_list( Cwd ) -> + gen_server:call( cre, {submit, App, Cwd} ). + +%% ============================================================================= +%% Internal Functions +%% ============================================================================= + + +-ifdef( TEST ). + + +-endif. \ No newline at end of file diff --git a/src/cf_parse.yrl b/src/cf_parse.yrl new file mode 100644 index 0000000..e824e99 --- /dev/null +++ b/src/cf_parse.yrl @@ -0,0 +1,323 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +%% ============================================================================= +%% Symbol Declaration +%% ============================================================================= + +Nonterminals + script stat assign exprlist expr binding assignlist sign query app cnd + paramlist param inparamlist inparam namelist name defun lang bindinglist + compoundexpr. + +Terminals + intlit strlit body bash beginif colon comma deftask else endif eq file in + python r lbrace lparen lsquarebr ltag nil rbrace rparen rsquarebr rtag + semicolon string then id perl. + + +%% ============================================================================= +%% Syntax Definition +%% ============================================================================= + +Rootsymbol script. + + +script -> stat : '$1'. +script -> stat script : combine( '$1', '$2' ). + +stat -> query : {'$1', #{}, #{}}. +stat -> assign : {undef, '$1', #{}}. +stat -> defun : {undef, #{}, '$1'}. + +query -> compoundexpr semicolon : '$1'. + +assign -> exprlist eq compoundexpr semicolon : mk_assign( get_line( '$2' ), '$1', '$3', 1 ). + +assignlist -> assign : '$1'. +assignlist -> assign assignlist : maps:merge( '$1', '$2' ). + +defun -> deftask id sign lbrace assignlist rbrace : mk_natlam( '$1', '$2', '$3', '$5' ). +defun -> deftask id sign in lang body : mk_forlam( '$1', '$2', '$3', '$5', '$6' ). + +lang -> bash : bash. +lang -> perl : perl. +lang -> python : python. +lang -> r : r. + +compoundexpr -> nil : []. +compoundexpr -> exprlist : '$1'. + +expr -> intlit : mk_str( '$1' ). +expr -> strlit : mk_str( '$1' ). +expr -> id : mk_var( '$1' ). +expr -> cnd : '$1'. +expr -> app : '$1'. + +exprlist -> expr : ['$1']. +exprlist -> expr exprlist : ['$1'|'$2']. + +cnd -> beginif compoundexpr then compoundexpr + else compoundexpr endif : {cnd, get_line( '$1' ), '$2', '$4', '$6'}. + +app -> id lparen rparen : {app, get_line( '$1' ), 1, mk_var( '$1' ), #{}}. +app -> id lparen bindinglist rparen : {app, get_line( '$1' ), 1, mk_var( '$1' ), '$3'}. + +binding -> id colon compoundexpr : mk_binding( '$1', '$3' ). + +bindinglist -> binding : '$1'. +bindinglist -> binding comma bindinglist : maps:merge( '$1', '$3' ). + +sign -> lparen paramlist colon rparen : {sign, '$2', []}. +sign -> lparen paramlist colon inparamlist rparen : {sign, '$2', '$4'}. + +inparam -> param : '$1'. +inparam -> lsquarebr namelist rsquarebr : {correl, '$2'}. + +inparamlist -> inparam : ['$1']. +inparamlist -> inparam inparamlist : ['$1'|'$2']. + +param -> name : {param, '$1', false}. +param -> ltag name rtag : {param, '$2', true}. + +paramlist -> param : ['$1']. +paramlist -> param paramlist : ['$1'|'$2']. + +name -> id : {name, get_name( '$1' ), false}. +name -> id lparen string rparen : {name, get_name( '$1' ), false}. +name -> id lparen file rparen : {name, get_name( '$1' ), true}. + +namelist -> name : ['$1']. +namelist -> name namelist : ['$1'|'$2']. + +%% ============================================================================= +%% Erlang Code +%% ============================================================================= + +Erlang code. + +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-export( [string/1, file/1] ). + +-ifdef( TEST ). +-include_lib( "eunit/include/eunit.hrl" ). +-endif. + +string( S ) -> + case cf_scan:string( S ) of + {error, ScanErrorInfo, _} -> {error, ScanErrorInfo}; + {ok, TokenLst, _} -> + try parse( TokenLst ) of + Ret -> Ret + catch + throw:E -> {error, E} + end + end. + +file( Filename ) -> + case file:read_file( Filename ) of + {error, Reason} -> {error, Reason}; + {ok, B} -> + S = binary_to_list( B ), + string( S ) + end. + + + +combine( {Target1, Rho1, Global1}, {Target2, Rho2, Global2} ) -> + case Target1 of + undef -> {Target2, maps:merge( Rho1, Rho2 ), maps:merge( Global1, Global2 )}; + _ -> + case Target2 of + undef -> {undef, maps:merge( Rho1, Rho2 ), maps:merge( Global1, Global2 )}; + _ -> {Target1++Target2, maps:merge( Rho1, Rho2 ), maps:merge( Global1, Global2 )} + end + end. + +mk_var( {id, Line, Name} ) -> {var, Line, Name}. + +mk_str( {intlit, _Line, N} ) -> {str, N}; +mk_str( {strlit, _Line, S} ) -> {str, S}. + +get_line( {_, Line, _} ) -> Line. + +get_name( {id, _Line, Name} ) -> Name. + +mk_binding( {id, _, Name}, ExprList ) -> + #{Name => ExprList}. + +mk_assign( _Line, [], _ExprList, _Channel ) -> #{}; + +mk_assign( Line, [{var, _Line, Name}|Rest], ExprList, Channel ) -> + Rho = mk_assign( Line, Rest, ExprList, Channel+1 ), + Value = lists:flatmap( fun( E ) -> set_channel( Line, E, Channel ) end, ExprList ), + Rho#{Name => Value}; + +mk_assign( Line, [_E|_Rest], _ExprList, _Channel ) -> + throw( {Line, cf_parse, "non-variable expression on left-hand side of assignment"} ). + + +set_channel( _Line, {app, AppLine, _Channel, LamList, Binding}, N ) -> [{app, AppLine, N, LamList, Binding}]; +set_channel( _Line, E, 1 ) -> [E]; +set_channel( Line, _, _ ) -> throw( {Line, cf_parse, "multiple value bind on non-application expression"} ). + +mk_natlam( {deftask, Line, _}, {id, _, Name}, Sign, Block ) -> + #{Name => {lam, Line, Name, Sign, {natbody, Block}}}. + +mk_forlam( {deftask, Line, _}, {id, _, Name}, Sign, Lang, {body, _, Code} ) -> + #{Name => {lam, Line, Name, Sign, {forbody, Lang, Code}}}. + +%% ============================================================================= +%% Unit Tests +%% ============================================================================= + +-ifdef( TEST ). + +nil_should_be_recognized_test() -> + ?assertEqual( {ok, {[], #{}, #{}}}, string( "nil;" ) ). + +var_should_be_recognized_test() -> + ?assertEqual( {ok, {[{var, 1, "blub"}], #{}, #{}}}, string( "blub;" ) ). + +multi_element_compoundexpr_should_be_recognized_test() -> + ?assertEqual( {ok, {[{var, 1, "bla"}, {var, 1, "blub"}], #{}, #{}}}, + string( "bla blub;" ) ). + +multiple_queries_should_be_joined_test() -> + ?assertEqual( {ok, {[{var, 1, "bla"}, {var, 1, "blub"}], #{}, #{}}}, + string( "bla; blub;" ) ). + +strlit_should_be_recognized_test() -> + ?assertEqual( {ok, {[{str, "bla"}], #{}, #{}}}, string( "\"bla\";" ) ). + +intlit_should_be_recognized_test() -> + ?assertEqual( {ok, {[{str, "-5"}], #{}, #{}}}, string( "-5;" ) ). + +cnd_should_be_recognized_test() -> + ?assertEqual( {ok, {[{cnd, 1, [], [{str, "bla"}], [{str, "blub"}]}], + #{}, #{}}}, + string( "if nil then \"bla\" else \"blub\" end;" ) ). + +app_should_be_recognized_test() -> + [?assertEqual( {ok, {[{app, 1, 1, {var, 1, "f"}, #{}}], #{}, #{}}}, + string( "f();" ) ), + ?assertEqual( {ok, {[{app, 1, 1, {var, 1, "f"}, #{"x" => [{var, 1, "x"}]}}], + #{}, #{}}}, string( "f( x: x );" ) ), + ?assertEqual( {ok, {[{app, 1, 1, {var, 1, "f"}, + #{"x" => [{var, 1, "x"}], + "y" => [{str, "y"}]}}], #{}, #{}}}, + string( "f( x: x, y: \"y\" );" ) )]. + + +assign_should_be_recognized_test() -> + [?assertEqual( {ok, {undef, #{"x" => [{str, "x"}]}, #{}}}, + string( "x = \"x\";" ) ), + ?assertEqual( {ok, {undef, #{"x" => [{app, 1, 1, {var, 1, "f"}, #{}}], + "y" => [{app, 1, 2, {var, 1, "f"}, #{}}]}, + #{}}}, string( "x y = f();" ) ), + ?assertMatch( {error, {1, cf_parse, _}}, string( "x y = \"A\";" ) ), + ?assertMatch( {error, {1, cf_parse, _}}, string( "\"a\" = \"A\";" ) )]. + +native_deftask_should_be_recognized_test() -> + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + []}, + {natbody, #{"out" => [{str, "A"}]}}}}}}, + string( "deftask f( out : ) { out = \"A\"; }" ) ). + +foreign_deftask_should_be_recognized_test() -> + [?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + []}, + {forbody, bash, "out=A"}}}}}, + string( "deftask f( out : )in bash *{out=A}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + []}, + {forbody, r, "out=\"A\""}}}}}, + string( "deftask f( out : )in R *{out=\"A\"}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + []}, + {forbody, python, ""}}}}}, + string( "deftask f( out : )in python *{}*" ) )]. + +sign_with_inparam_should_be_recognized_test() -> + [?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + [{param, {name, "inp", false}, false}]}, + {forbody, python, "(defparameter out \"A\")"}}}}}, + string( "deftask f( out : inp )in python *{(defparameter out \"A\")}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + [{param, {name, "a", false}, false}, + {param, {name, "b", false}, false}]}, + {forbody, python, "(defparameter out \"A\")"}}}}}, + string( "deftask f( out : a b )in python *{(defparameter out \"A\")}*" ) )]. + +param_should_be_recognized_test() -> + [?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + [{param, {name, "inp", false}, false}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( out( String ) : inp( String ) )in bash *{blub}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", true}, false}], + [{param, {name, "inp", true}, false}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( out( File ) : inp( File ) )in bash *{blub}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, true}], + [{param, {name, "inp", false}, true}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( : )in bash *{blub}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, true}], + [{param, {name, "inp", false}, true}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( : )in bash *{blub}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", true}, true}], + [{param, {name, "inp", true}, true}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( : )in bash *{blub}*" ) )]. + +correl_inparam_should_be_recognized_test() -> + [?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + [{correl, [{name, "a", true}, + {name, "b", true}]}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( out : [a( File ) b( File )] )in bash *{blub}*" ) ), + ?assertEqual( {ok, {undef, #{}, #{"f" => {lam, 1, "f", + {sign, [{param, {name, "out", false}, false}], + [{correl, [{name, "a", true}, + {name, "b", true}]}, + {param, {name, "c", false}, false}]}, + {forbody, bash, "blub"}}}}}, + string( "deftask f( out : [a( File ) b( File )] c )in bash *{blub}*" ) )]. + +-endif. + + diff --git a/src/cf_prescan.xrl b/src/cf_prescan.xrl new file mode 100644 index 0000000..2046ba6 --- /dev/null +++ b/src/cf_prescan.xrl @@ -0,0 +1,72 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +%% ============================================================================= +%% Definitions +%% ============================================================================= + +Definitions. + +CWD = cwd +HELP = help +NONWS = . +QUIT = quit +SEMICOLON = ; +LBRACE = \{ +LMMECB = \*\{ +RBRACE = \} +RMMECB = \}\* +STATE = state +TASKS = tasks +WS = [\000-\s] + + +%% ============================================================================= +%% Rules +%% ============================================================================= + +Rules. + +{CWD} : {token, cwd}. +{TASKS} : {token, tasks}. +{STATE} : {token, state}. +{QUIT} : {token, quit}. +{HELP} : {token, help}. +{SEMICOLON} : {token, semicolon}. +{LBRACE} : {token, lbrace}. +{RBRACE} : {token, rbrace}. +{LMMECB} : {token, lmmecb}. +{RMMECB} : {token, rmmecb}. +{WS} : skip_token. +{NONWS} : {token, nonws}. + +%% ============================================================================= +%% Erlang Code +%% ============================================================================= + + +Erlang code. + +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-export( [yyrev/2] ). \ No newline at end of file diff --git a/src/cf_scan.xrl b/src/cf_scan.xrl new file mode 100644 index 0000000..f8181dc --- /dev/null +++ b/src/cf_scan.xrl @@ -0,0 +1,265 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +%% ============================================================================= +%% Definitions +%% ============================================================================= + + +Definitions. + +BASH = [Bb]ash +PERL = [Pp]erl +PYTHON = [Pp]ython +R = [Rr] + +BEGINIF = if +COLON = : +COMMA = , +DEFTASK = deftask +ELSE = else +ENDIF = end +EQ = = +FILE = File +IN = in +LBRACE = \{ +LPAREN = \( +LSQUAREBR = \[ +LTAG = < +NIL = nil +RBRACE = \} +RPAREN = \) +RSQUAREBR = \] +RTAG = > +SEMICOLON = ; +STRING = String +THEN = then + +ID = [A-Za-z][A-Za-z0-9.\-_]* +INTLIT = -?([0-9]|[1-9][0-9]*) +STRLIT = "[^"]*" +BODY = \*\{([^}]|\}+[^}*])*\}+\* + + +COMMENT1 = (#|//|%).* +COMMENT2 = /\*([^*]|\*+[^/*])*\*+/ +WS = [\000-\s] + + + + +%% ============================================================================= +%% Rules +%% ============================================================================= + +Rules. + +{BASH} : {token, {bash, TokenLine, TokenChars}}. +{PERL} : {token, {perl, TokenLine, TokenChars}}. +{PYTHON} : {token, {python, TokenLine, TokenChars}}. +{R} : {token, {r, TokenLine, TokenChars}}. + +{INTLIT} : {token, {intlit, TokenLine, TokenChars}}. +{STRLIT} : {token, {strlit, TokenLine, trim_strlit( TokenChars )}}. +{BODY} : {token, {body, TokenLine, trim_body( TokenChars )}}. +{BEGINIF} : {token, {beginif, TokenLine, TokenChars}}. +{COLON} : {token, {colon, TokenLine, TokenChars}}. +{COMMA} : {token, {comma, TokenLine, TokenChars}}. +{DEFTASK} : {token, {deftask, TokenLine, TokenChars}}. +{ELSE} : {token, {else, TokenLine, TokenChars}}. +{ENDIF} : {token, {endif, TokenLine, TokenChars}}. +{EQ} : {token, {eq, TokenLine, TokenChars}}. +{FILE} : {token, {file, TokenLine, TokenChars}}. +{IN} : {token, {in, TokenLine, TokenChars}}. +{LBRACE} : {token, {lbrace, TokenLine, TokenChars}}. +{LPAREN} : {token, {lparen, TokenLine, TokenChars}}. +{LSQUAREBR} : {token, {lsquarebr, TokenLine, TokenChars}}. +{LTAG} : {token, {ltag, TokenLine, TokenChars}}. +{NIL} : {token, {nil, TokenLine, TokenChars}}. +{RBRACE} : {token, {rbrace, TokenLine, TokenChars}}. +{RPAREN} : {token, {rparen, TokenLine, TokenChars}}. +{RSQUAREBR} : {token, {rsquarebr, TokenLine, TokenChars}}. +{RTAG} : {token, {rtag, TokenLine, TokenChars}}. +{SEMICOLON} : {token, {semicolon, TokenLine, TokenChars}}. +{STRING} : {token, {string, TokenLine, TokenChars}}. +{THEN} : {token, {then, TokenLine, TokenChars}}. + +{ID} : {token, {id, TokenLine, TokenChars}}. + +{COMMENT1} : skip_token. +{COMMENT2} : skip_token. +{WS} : skip_token. + + + + +%% ============================================================================= +%% Erlang Code +%% ============================================================================= + + +Erlang code. + +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-export( [yyrev/2] ). + +-ifdef( TEST ). +-include_lib( "eunit/include/eunit.hrl" ). +-endif. + +trim_body( S ) -> + string:substr( S, 3, length( S )-4 ). + +trim_strlit( S ) -> + string:substr( S, 2, length( S )-2 ). + + + +%% ============================================================================= +%% Unit Tests +%% ============================================================================= + + +-ifdef( TEST ). + + + +bash_style_comment_should_be_recognized_test() -> + ?assertEqual( {ok, [], 1}, string( "#this is a comment" ) ). + +c_style_comment_should_be_recognized_test() -> + ?assertEqual( {ok, [], 1}, string( "//this is a comment" ) ). + +erlang_style_comment_should_be_recognized_test() -> + ?assertEqual( {ok, [], 1}, string( "%this is a comment" ) ). + + +multiline_comment_should_be_recognized_test() -> + [?assertEqual( {ok, [], 1}, string( "/**/" ) ), + ?assertEqual( {ok, [], 1}, string( "/*x*/" ) ), + ?assertEqual( {ok, [], 1}, string( "/*xy*/" ) ), + ?assertEqual( {ok, [], 2}, string( "/*x\ny*/" ) )]. + +bash_should_be_recognized_test() -> + ?assertEqual( {ok, [{bash, 1, "bash"}], 1}, string( "bash" ) ). + +beginif_should_be_recognized_test() -> + ?assertEqual( {ok, [{beginif, 1, "if"}], 1}, string( "if" ) ). + +colon_should_be_recognized_test() -> + ?assertEqual( {ok, [{colon, 1, ":"}], 1}, string( ":" ) ). + +deftask_should_be_recognized_test() -> + ?assertEqual( {ok, [{deftask, 1, "deftask"}], 1}, string( "deftask" ) ). + +else_should_be_recognized_test() -> + ?assertEqual( {ok, [{else, 1, "else"}], 1}, string( "else" ) ). + +endif_should_be_recognized_test() -> + ?assertEqual( {ok, [{endif, 1, "end"}], 1}, string( "end" ) ). + +eq_should_be_recognized_test() -> + ?assertEqual( {ok, [{eq, 1, "="}], 1}, string( "=" ) ). + +file_should_be_recognized_test() -> + ?assertEqual( {ok, [{file, 1, "File"}], 1}, string( "File" ) ). + +in_should_be_recognized_test() -> + ?assertEqual( {ok, [{in, 1, "in"}], 1}, string( "in" ) ). + +id_should_be_recognized_test() -> + [?assertEqual( {ok, [{id, 1, "inp"}], 1}, string( "inp" ) ), + ?assertEqual( {ok, [{id, 1, "a0"}], 1}, string( "a0" ) ), + ?assertEqual( {ok, [{id, 1, "a9"}], 1}, string( "a9" ) ), + ?assertEqual( {ok, [{id, 1, "a."}], 1}, string( "a." ) ), + ?assertEqual( {ok, [{id, 1, "a-"}], 1}, string( "a-" ) ), + ?assertEqual( {ok, [{id, 1, "a_"}], 1}, string( "a_" ) )]. + +python_should_be_recognized_test() -> + ?assertEqual( {ok, [{python, 1, "python"}], 1}, string( "python" ) ). + +r_should_be_recognized_test() -> + [?assertEqual( {ok, [{r, 1, "r"}], 1}, string( "r" ) ), + ?assertEqual( {ok, [{r, 1, "R"}], 1}, string( "R" ) )]. + +lbrace_should_be_recognized_test() -> + ?assertEqual( {ok, [{lbrace, 1, "{"}], 1}, string( "{" ) ). + +lparen_should_be_recognized_test() -> + ?assertEqual( {ok, [{lparen, 1, "("}], 1}, string( "(" ) ). + +lsquarebr_should_be_recognized_test() -> + ?assertEqual( {ok, [{lsquarebr, 1, "["}], 1}, string( "[" ) ). + +ltag_should_be_recognized_test() -> + ?assertEqual( {ok, [{ltag, 1, "<"}], 1}, string( "<" ) ). + +nil_should_be_recognized_test() -> + ?assertEqual( {ok, [{nil, 1, "nil"}], 1}, string( "nil" ) ). + +rbrace_should_be_recognized_test() -> + ?assertEqual( {ok, [{rbrace, 1, "}"}], 1}, string( "}" ) ). + +rparen_should_be_recognized_test() -> + ?assertEqual( {ok, [{rparen, 1, ")"}], 1}, string( ")" ) ). + +rsquarebr_should_be_recognized_test() -> + ?assertEqual( {ok, [{rsquarebr, 1, "]"}], 1}, string( "]" ) ). + +rtag_should_be_recognized_test() -> + ?assertEqual( {ok, [{rtag, 1, ">"}], 1}, string( ">" ) ). + +semicolon_should_be_recognized_test() -> + ?assertEqual( {ok, [{semicolon, 1, ";"}], 1}, string( ";" ) ). + +string_should_be_recognized_test() -> + ?assertEqual( {ok, [{string, 1, "String"}], 1}, string( "String" ) ). + +then_should_be_recognized_test() -> + ?assertEqual( {ok, [{then, 1, "then"}], 1}, string( "then" ) ). + +intlit_should_be_recognized_test() -> + [?assertEqual( {ok, [{intlit, 1, "10"}], 1}, string( "10" ) ), + ?assertEqual( {ok, [{intlit, 1, "-10"}], 1}, string( "-10" ) ), + ?assertEqual( {ok, [{intlit, 1, "9"}], 1}, string( "9" ) ), + ?assertEqual( {ok, [{intlit, 1, "0"}], 1}, string( "0" ) )]. + +body_should_be_recognized_test() -> + [?assertEqual( {ok, [{body, 1, ""}], 1}, string( "*{}*" ) ), + ?assertEqual( {ok, [{body, 1, "x"}], 1}, string( "*{x}*" ) ), + ?assertEqual( {ok, [{body, 1, "xy"}], 1}, string( "*{xy}*" ) ), + ?assertEqual( {ok, [{body, 1, "x\ny"}], 2}, string( "*{x\ny}*" ) )]. + +strlit_should_be_recognized_test() -> + [?assertEqual( {ok, [{strlit, 1, ""}], 1}, string( "\"\"" ) ), + ?assertEqual( {ok, [{strlit, 1, "x"}], 1}, string( "\"x\"" ) ), + ?assertEqual( {ok, [{strlit, 1, "xy"}], 1}, string( "\"xy\"" ) )]. + + + + + +-endif. + + diff --git a/src/cf_sem.erl b/src/cf_sem.erl new file mode 100644 index 0000000..29084f7 --- /dev/null +++ b/src/cf_sem.erl @@ -0,0 +1,585 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( cf_sem ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-export( [eval/2, pfinal/1] ). + +-ifdef( TEST ). +-include_lib( "eunit/include/eunit.hrl" ). +-endif. + +%% ============================================================================= +%% Abstract Syntax +%% ============================================================================= + +%% Expression %% =============================================================== + +-type expr() :: str() | var() | select() | cnd() | app(). % (1) +-type str() :: {str, S::string()}. % (2) +-type var() :: {var, Line::pos_integer(), N::string()}. % (3) +-type select() :: {select, AppLine::pos_integer(), C::pos_integer(), U::fut()}.% (4) +-type fut() :: {fut, LamName::string(), R::pos_integer(), Lo::[param()]}. % (5) +-type cnd() :: {cnd, Line::pos_integer(), % (6) + Xc::[expr()], Xt::[expr()], Xe::[expr()]}. +-type app() :: {app, AppLine::pos_integer(), C::pos_integer(), % (7) + Lambda::lam() | var(), Fa::#{string() => [expr()]}}. + +%% Lambda %% =================================================================== + +-type lam() :: {lam, Line::pos_integer(), Name::string(), % (8) + S::sign(), B::body()}. + +% Task Signature +-type sign() :: {sign, Lo::[param()], Li::[inparam()]}. % (9) +-type param() :: {param, M::name(), Pl::boolean()}. % (10) +-type name() :: {name, N::string(), Pf::boolean()}. +-type inparam() :: param() | correl(). % (11) +-type correl() :: {correl, Lc::[name()]}. % (12) + +% Body +-type body() :: natbody() | forbody(). % (13) +-type natbody() :: {natbody, Fb::#{string() => [expr()]}}. % (14) +-type forbody() :: {forbody, L::lang(), S::string()}. % (15) +-type lang() :: bash | python | r. % (16) + +%% Argument Pair %% ============================================================ + +-type argpair() :: {L0::[inparam()], F::#{string() => [expr()]}}. % (17) + +%% Evaluation Context %% ======================================================= + +-type ctx() :: {Rho :: #{string() => [expr()]}, % (18) + Mu :: fun( ( app() ) -> fut() ), + Gamma :: #{string() => lam()}, + Omega :: #{{string(), pos_integer()} => [expr()]}}. + +%% ============================================================================= +%% Predicates +%% ============================================================================= + +%% Finality %% ================================================================= + +-spec pfinal( X ) -> boolean() % (19) +when X :: #{string() => [expr()]} | [expr()] | expr(). + +pfinal( F ) when is_map( F ) -> pfinal( maps:values( F ) ); % (20) +pfinal( L ) when is_list( L ) -> lists:all( fun pfinal/1, L ); % (21,22) +pfinal( {str, _S} ) -> true; % (23) +pfinal( _T ) -> false. + +%% Singularity %% ============================================================== + +-spec psing( A::app() ) -> boolean(). % (24) + +psing( {app, Line, _C, {lam, _, LamName, {sign, _Lo, Li}, _B}, Fa} ) -> % (25) + case psing_argpair( {Li, Fa} ) of + {ok, P} -> P; + {error, N} -> + throw( {Line, cf_sem, + "argument "++N++" is unbound in application of "++LamName} ) + end. + +-spec psing_argpair( Z::argpair() ) -> {ok, boolean()} | {error, string()}. % (26) + +psing_argpair( {[], _F} ) -> {ok, true}; % (27) +psing_argpair( {[{param, _, Pl}|T], F} ) % (28) +when Pl -> + psing_argpair( {T, F} ); +psing_argpair( {[{param, {name, N, _}, _Pl}|T], F} ) -> % (29) + case maps:is_key( N, F ) of + false -> {error, N}; + true -> + case length( maps:get( N, F ) ) of + 1 -> psing_argpair( {T, F} ); + _ -> {ok, false} + end + end; +psing_argpair( _Z ) -> {ok, false}. + +%% Enumerability %% ============================================================ + +-spec pen( X::expr()|[expr()] ) -> boolean(). % (30) + +pen( X )when is_list( X ) -> lists:all( fun pen/1, X ); % (31,32) +pen( {str, _S} ) -> true; % (33) +pen( {cnd, _, _Xc, Xt, Xe} ) when length( Xt ) =:= 1, length( Xe ) =:= 1 -> % (34) + pen( Xt ) andalso pen( Xe ); +pen( X={app, _, C, {lam, _, _, {sign, Lo, _Li}, _B}, _Fb} ) -> % (35) + case psing( X ) of + false -> false; + true -> + {param, _, Pl} = lists:nth( C, Lo ), + not Pl + end; +pen( {select, _, C, {fut, _, _R, Lo}} ) -> % (36) + {param, _, Pl} = lists:nth( C, Lo ), + not Pl; +pen( _T ) -> false. + +-spec pindep( X ) -> boolean() % (37) +when X :: #{string() => [expr()]} | [expr()] | expr(). + +pindep( Fa ) when is_map( Fa ) -> pindep( maps:values( Fa ) ); % (38) +pindep( X ) when is_list( X ) -> lists:all( fun pindep/1, X ); % (39,40) +pindep( {str, _} ) -> true; % (41) +pindep( {select, _, _, _} ) -> true; % (42) +pindep( {cnd, _, Xc, Xt, Xe} ) -> % (43) + pindep( Xc ) andalso pindep( Xt ) andalso pindep( Xe ); +pindep( {app, _, _, _, Fa} ) -> pindep( Fa ); % (44) +pindep( _ ) -> false. + +%% ============================================================================= +%% Evaluation +%% ============================================================================= + +%% The eval Function %% ======================================================== + +-spec eval( X::[expr()], Theta::ctx() ) -> [expr()]. % (37) + +eval( X, Theta ) -> + X1 = step( X, Theta ), + case X1 of + X -> X; % (38) + _ -> eval( X1, Theta ) % (39) + end. + +%% Reduction Rules %% ========================================================== + +-spec step_assoc( F, Theta ) -> #{string() => [expr()]} % (40) +when F :: #{string() => [expr()]}, + Theta :: ctx(). + +step_assoc( F, Theta ) when is_map( F ) -> % (41) + maps:map( fun( _N, X ) -> step( X, Theta ) end, F ). + + +-spec step( X, Theta ) -> [expr()] % (42) +when X :: #{string() => [expr()]} | [expr()] | expr(), + Theta :: ctx(). + + +% Expression List +step( X, Theta ) when is_list( X ) -> % (43,44) + lists:flatmap( fun( Y ) -> step( Y, Theta ) end, X ); + +% String Literal +step( X={str, _S}, _Theta ) -> [X]; % (45) + +% Variable +step( {var, _, N}, {Rho, _Mu, _Gamma, _Omega} ) -> % (46) + case maps:is_key( N, Rho ) of + true -> maps:get( N, Rho ); + false -> throw( {1, cf_sem, "unbound variable "++N} ) + end; + +% Future Channel Selection +step( S={select, _, C, {fut, _, R, Lo}}, {_Rho, _Mu, _Gamma, Omega} ) -> % (47,48) + {param, {name, N, _}, _} = lists:nth( C, Lo ), + maps:get( {N, R}, Omega, [S] ); + +% Conditional +step( {cnd, _, [], _Xt, Xe}, _Theta ) -> Xe; % (49) +step( {cnd, Line, Xc=[_|_], Xt, Xe}, Theta ) -> + case pfinal( Xc ) of + false -> [{cnd, Line, step( Xc, Theta ), Xt, Xe}]; % (50) + true -> Xt % (51) + end; + +% Application +step( {app, Line, C, {var, _, N}, Fa}, {_Rho, _Mu, Gamma, _Omega} ) -> % (52) + case maps:is_key( N, Gamma ) of + true -> [{app, Line, C, maps:get( N, Gamma ), Fa}]; + false -> throw( {Line, cf_sem, "undefined task "++N} ) + end; + +step( X={app, AppLine, C, + Lambda={lam, LamLine, LamName, S={sign, Lo, _Li}, B}, Fa}, + Theta={_Rho, Mu, Gamma, Omega} ) -> + case psing( X ) of + false -> enum_app( {app, AppLine, C, Lambda, step_assoc( Fa, Theta )} ); % (53) + true -> + case pfinal( Fa ) of + false -> [{app, AppLine, C, Lambda, step_assoc( Fa, Theta )}]; % (54) + true -> + case B of + {forbody, _L, _Z} -> [{select, AppLine, C, apply( Mu, [X] )}]; % (55) + {natbody, Fb} -> + {param, {name, N, _}, Pl} = lists:nth( C, Lo ), + V0 = maps:get( N, Fb ), + V1 = step( V0, {maps:merge( Fb, Fa ), Mu, Gamma, Omega} ), + case pindep( V1 ) of + false -> [{app, AppLine, C, % (56) + {lam, LamLine, LamName, S, + {natbody, Fb#{ N => V1}}}, + Fa}]; + true -> + case Pl orelse length( V1 ) =:= 1 of + true -> V1; % (57) + false -> error( output_sign_mismatch ) + end + end + end + end + end. + + + +%% ============================================================================= +%% Enumeration +%% ============================================================================= + +%% The enum Function %% + +-spec enum_app( A::app() ) -> [app()]. % (58) + +enum_app( {app, AppLine, C, {lam, LamLine, LamName, {sign, Lo, Li}, B}, Fa} ) ->% (59) + [{app, AppLine, C, + {lam, LamLine, LamName, + {sign, Lo, L1}, B}, F1} || {L1, F1} <- enum( [{Li, Fa}] )]. + +-spec enum( Z::[argpair()] ) -> [argpair()]. % (60) + +enum( Z ) -> + Z1 = estep( Z ), + case Z1 of + Z -> Z; % (61) + _ -> enum( Z1 ) % (62) + end. + +%% Enumeration Rules %% + +-spec estep( Z::[argpair()] ) -> [argpair()]. % (63) + +estep( Z ) -> % (64,65) + lists:flatmap( fun( {Li, F} ) -> estep( Li, F ) end, Z ). + +-spec estep( Li::[inparam()], F::#{string() => [expr()]} ) -> [argpair()]. % (66) + +estep( [], F ) -> [{[], F}]; % (67) +estep( [H={param, _, Pl}|T], F ) when Pl -> aug( estep( T, F ), H ); % (68) +estep( L=[H={param, {name, N, _}, _Pl}|T], F ) -> + V = maps:get( N, F ), + case pen( V ) of + false -> [{L, F}]; % (69) + true -> + case length( V ) of + 1 -> aug( estep( T, F ), H ); % (70) + _ -> [{L, maps:put( N, [X], F )} || X <- V] % (71) + end + end; +estep( L=[H={correl, Lc}|T], F ) when length( Lc ) > 1 -> + Pen = pen( [maps:get( N, F ) || {name, N, _} <- Lc] ), + case Pen of + false -> [{L, F}]; % (72) + true -> + Z = corr( Lc, F ), + aug( [{T, G} || G <- Z], H ) % (73) + end. + + + +%% Augmentation %% + +-spec aug( Z::[argpair()], A::inparam() ) -> [argpair()]. % (74) + +aug( Z, A ) -> [aug_argpair( Y, A ) || Y <- Z]. % (75) + +-spec aug_argpair( Y::argpair(), A::inparam() ) -> argpair(). % (76) + +aug_argpair( {L0, F}, A={param, _, _Pl} ) -> {[A|L0], F}; % (77) +aug_argpair( {L0, F}, {correl, Lc} ) -> % (78) + L1 = [{param, M, false} || M <- Lc], + {L1++L0, F}. + +%% Correlation %% + +-spec corr( Lc, F ) -> [#{string() => [expr()]}] +when Lc :: [name()], + F :: #{string() => [expr()]}. + +corr( Lc, F ) -> + case corrstep( Lc, F, F ) of + [] -> []; + [H,T] -> [H|corr( Lc, T )] + end. + +-spec corrstep( Lc, Facc, F0 ) -> [#{string() => [expr()]}] % (79) +when Lc :: [name()], + Facc :: #{string() => [expr()]}, + F0 :: #{string() => [expr()]}. + +corrstep( [], Facc, F0 ) -> [Facc, F0]; % (80) +corrstep( [{name, H, _}|T], Facc, F0 ) -> + case maps:get( H, F0 ) of + [] -> []; % (81) + [A|B] -> corrstep( T, Facc#{H => [A]}, F0#{H => B}) % (82) + end. + + + + +%% ============================================================================= +%% Unit Tests +%% ============================================================================= + +-ifdef( TEST ). + + +%% ============================================================================= +%% Predicates +%% ============================================================================= + +%% Finality %% + +str_should_be_final_test() -> + S = {str, "blub"}, + ?assert( pfinal( S ) ). + +app_should_not_be_final_test() -> + A = {app, 12, 1, {var, "f"}, #{}}, + ?assertNot( pfinal( A ) ). + +cnd_should_not_be_final_test() -> + C = {cnd, 12, [{str, "a"}], [{str, "b"}], [{str, "c"}]}, + ?assertNot( pfinal( C ) ). + +select_should_not_be_final_test() -> + Fut = {fut, "f", 1234, [{param, "out", false}]}, + S = {select, 12, 1, Fut}, + ?assertNot( pfinal( S ) ). + +var_should_not_be_final_test() -> + V = {var, 12, "x"}, + ?assertNot( pfinal( V ) ). + +all_str_should_be_final_test() -> + X = [{str, "bla"}, {str, "blub"}], + ?assert( pfinal( X ) ). + +empty_lst_should_be_final_test() -> + ?assert( pfinal( [] ) ). + +one_var_lst_should_not_be_final_test() -> + X = [{str, "bla"}, {str, "blub"}, {var, 12, "x"}], + ?assertNot( pfinal( X ) ). + +all_var_lst_should_not_be_final_test() -> + X = [{var, 10, "bla"}, {var, 11, "blub"}, {var, 12, "x"}], + ?assertNot( pfinal( X ) ). + +empty_map_should_be_final_test() -> + ?assert( pfinal( #{} ) ). + +only_str_map_should_be_final_test() -> + M = #{"x" => [{str, "bla"}, {str, "blub"}], "y" => [{str, "shalala"}]}, + ?assert( pfinal( M ) ). + +one_var_map_should_not_be_final_test() -> + M = #{"x" => [{str, "bla"}, {str, "blub"}], + "y" => [{str, "shalala"}, {var, 12, "x"}]}, + ?assertNot( pfinal( M ) ). + +all_var_map_should_not_be_final_test() -> + M = #{"x" => [{var, 10, "bla"}, {var, 11, "blub"}], + "y" => [{var, 12, "shalala"}, {var, 13, "x"}]}, + ?assertNot( pfinal( M ) ). + +%% Singularity %% + +app_without_arg_should_be_singular_test() -> + S = {sign, [{param, {name, "out", false}, false}], []}, + B = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", S, B}, + App = {app, 13, 1, Lam, #{}}, + ?assert( psing( App ) ). + +app_binding_single_str_should_be_singular_test() -> + S = {sign, [{param, {name, "out", false}, false}], [{param, {name, "x", false}, false}]}, + B = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", S, B}, + App = {app, 13, 1, Lam, #{"x" => [{str, "bla"}]}}, + ?assert( psing( App ) ). + +app_binding_str_lst_should_not_be_singular_test() -> + S = {sign, [{param, {name, "out", false}, false}], [{param, {name, "x", false}, false}]}, + B = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", S, B}, + App = {app, 13, 1, Lam, #{"x" => [{str, "bla"}, {str, "blub"}]}}, + ?assertNot( psing( App ) ). + +app_with_only_aggregate_args_should_be_singular_test() -> + S = {sign, [{param, {name, "out", false}, false}], [{param, {name, "x", false}, true}]}, + B = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", S, B}, + App = {app, 13, 1, Lam, #{"x" => [{str, "bla"}, {str, "blub"}]}}, + ?assert( psing( App ) ). + +%% Enumerability %% + +empty_lst_should_be_enumerable_test() -> + ?assert( pen( [] ) ). + +str_lst_should_be_enumerable_test() -> + ?assert( pen( [{str, "a"}, {str, "b"}] ) ). + +one_var_lst_should_not_be_enumerable_test() -> + ?assertNot( pen( [{str, "a"}, {var, 12, "x"}] ) ). + +all_var_lst_should_not_be_enumerable_test() -> + ?assertNot( pen( [{var, 12, "x"}, {var, 13, "y"}] ) ). + +str_should_be_enumerable_test() -> + ?assert( pen( {str, "blub"} ) ). + +single_str_branch_cnd_should_be_enumerable_test() -> + ?assert( pen( {cnd, 12, [], [{str, "a"}], [{str, "b"}]} ) ). + +empty_then_branch_cnd_should_not_be_enumerable_test() -> + ?assertNot( pen( {cnd, 12, [], [], [{str, "b"}]} ) ). + +empty_else_branch_cnd_should_not_be_enumerable_test() -> + ?assertNot( pen( {cnd, 12, [], [{str, "a"}]} ) ). + +select_non_lst_app_should_be_enumerable_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", Sign, Body}, + App = {app, 13, 1, Lam, #{}}, + ?assert( pen( App ) ). + +select_lst_app_should_not_be_enumerable_test() -> + Sign = {sign, [{param, {name, "out", false}, true}], []}, + Body = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", Sign, Body}, + App = {app, 13, 1, Lam, #{}}, + ?assertNot( pen( App ) ). + +non_singular_app_should_not_be_enumerable_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], [{param, {name, "x", false}, false}]}, + Body = {forbody, bash, "shalala"}, + Lam = {lam, 12, "f", Sign, Body}, + Fa = #{"x" => [{str, "bla"}, {str, "blub"}]}, + App = {app, 13, 1, Lam, Fa}, + ?assertNot( pen( App ) ). + +non_lst_select_should_be_enumerable_test() -> + Lo = [{param, {name, "out", false}, false}], + Fut = {fut, "f", 1234, Lo}, + Select = {select, 12, 1, Fut}, + ?assert( pen( Select ) ). + +lst_select_should_not_be_enumerable_test() -> + Lo = [{param, {name, "out", false}, true}], + Fut = {fut, "f", 1234, Lo}, + Select = {select, 12, 1, Fut}, + ?assertNot( pen( Select ) ). + +%% ============================================================================= +%% Enumeration +%% ============================================================================= + +%% The enum Function %% + +enum_app_without_app_does_nothing_test() -> + S = {sign, [{param, {param, "out", false}, false}], []}, + B = {forbody, bash, "shalala"}, + Lam = {lam, 1, "f", S, B}, + App = {app, 2, 1, Lam, #{}}, + ?assertEqual( [App], enum_app( App ) ). + +%% Enumeration Rules %% + +%% Augmentation %% + +can_aug_argpair_with_param_test() -> + L0 = [{param, {name, "b", false}, false}, {param, {name, "c", false}, false}], + F = #{"a" => [{str, "1"}], "b" => [{str, "2"}], "c" => [{str, "3"}]}, + Pair = {L0, F}, + I = {param, {name, "a", false}, false}, + L1 = [I|L0], + ?assertEqual( {L1, F}, aug_argpair( Pair, I ) ). + +can_aug_argpair_with_correl_test() -> + L0 = [{param, {name, "b", false}, false}, {param, {name, "c", false}, false}], + F = #{"a1" => [{str, "11"}], + "a2" => [{str, "12"}], + "b" => [{str, "2"}], + "c" => [{str, "3"}]}, + Pair = {L0, F}, + I = {correl, [{name, "a1", false}, {name, "a2", false}]}, + L1 = [{param, {name, "a1", false}, false}, {param, {name, "a2", false}, false}|L0], + ?assertEqual( {L1, F}, aug_argpair( Pair, I ) ). + +can_augment_empty_argpairlist_with_param_test() -> + F1 = #{"a" => [{str, "x1"}]}, + F2 = #{"a" => [{str, "y1"}]}, + PairList = [{[], F1}, {[], F2}], + I = {param, {name, "a", false}, false}, + L1 = [{param, {name, "a", false}, false}], + ?assertEqual( [{L1, F1}, {L1, F2}], aug( PairList, I ) ). + +can_augment_argpairlist_with_param_test() -> + L0 = [{param, {name, "b", false}, false}, {param, {name, "c", false}, false}], + F1 = #{"a" => [{str, "x1"}], "b" => [{str, "x2"}], "c" => [{str, "x3"}]}, + F2 = #{"a" => [{str, "y1"}], "b" => [{str, "y2"}], "c" => [{str, "y3"}]}, + PairList = [{L0, F1}, {L0, F2}], + I = {param, {name, "a", false}, false}, + L1 = [{param, {name, "a", false}, false}|L0], + ?assertEqual( [{L1, F1}, {L1, F2}], aug( PairList, I ) ). + +can_augment_argpairlist_with_correl_test() -> + L0 = [{param, {name, "b", false}, false}, {param, {name, "c", false}, false}], + F1 = #{"a1" => [{str, "x11"}], + "a2" => [{str, "x12"}], + "b" => [{str, "x2"}], + "c" => [{str, "x3"}]}, + F2 = #{"a1" => [{str, "y11"}], + "a2" => [{str, "y12"}], + "b" => [{str, "y2"}], + "c" => [{str, "y3"}]}, + PairList = [{L0, F1}, {L0, F2}], + I = {correl, [{name, "a1", false}, {name, "a2", false}]}, + L1 = [{param, {name, "a1", false}, false}, {param, {name, "a2", false}, false}|L0], + ?assertEqual( [{L1, F1}, {L1, F2}], aug( PairList, I ) ). + +%% Correlation %% + +corrstep_should_separate_first_value_single_test() -> + Lc = [{name, "a", false}], + F0 = #{"a" => [{str, "1"}, {str, "2"}, {str, "3"}]}, + Y = [#{"a" => [{str, "1"}]}, #{"a" => [{str, "2"}, {str, "3"}]}], + X = corrstep( Lc, F0, F0 ), + ?assertEqual( Y, X ). + +corrstep_should_separate_first_value_two_test() -> + Lc = [{name, "a", false}, {name, "b", false}], + F0 = #{"a" => [{str, "1"}, {str, "2"}, {str, "3"}], "b" => [{str, "A"}, {str, "B"}, {str, "C"}]}, + Y = [#{"a" => [{str, "1"}], "b" => [{str, "A"}]}, #{"a" => [{str, "2"}, {str, "3"}], "b" => [{str, "B"}, {str, "C"}]}], + X = corrstep( Lc, F0, F0 ), + ?assertEqual( Y, X ). + + + +-endif. \ No newline at end of file diff --git a/src/cf_shell.erl b/src/cf_shell.erl new file mode 100644 index 0000000..c006405 --- /dev/null +++ b/src/cf_shell.erl @@ -0,0 +1,166 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( cf_shell ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +%% ============================================================================= +%% Function Exports +%% ============================================================================= + +-export( [server/1] ). + +%% ============================================================================= +%% Includes +%% ============================================================================= + +-include( "cuneiform.hrl" ). + +%% ============================================================================= +%% Macro Definitions +%% ============================================================================= + +-define( PROMPT, "> " ). + +%% ============================================================================= +%% Internal Functions +%% ============================================================================= + +-spec server( Cwd::string() ) -> ok. + +server( Cwd ) -> + io:format( "~s~n~n~n", [get_banner()] ), + server_loop( #{}, #{}, Cwd ). + +-spec server_loop( Rho, Gamma, Cwd ) -> ok +when Rho :: #{string() => [cuneiform:expr()]}, + Gamma :: #{string() => fun()}, + Cwd :: string(). + +server_loop( Rho, Gamma, Cwd ) -> + case read_expression( ?PROMPT ) of + {ctl, quit} -> + ok; + {ctl, cwd} -> + io:format( "~s~n", [filename:absname( Cwd )] ), + server_loop( Rho, Gamma, Cwd ); + {ctl, help} -> + io:format( "~s~n", [get_help()] ), + server_loop( Rho, Gamma, Cwd ); + {ctl, state} -> + io:format( "~p~n", [Rho] ), + server_loop( Rho, Gamma, Cwd ); + {ctl, tasks} -> + io:format( "~p~n", [Gamma] ), + server_loop( Rho, Gamma, Cwd ); + {error, ErrorInfo} -> + S = cuneiform:format_error( ErrorInfo ), + io:format( "~s~n", [S] ), + server_loop( Rho, Gamma, Cwd ); + {ok, {Query, DRho, DGamma}} -> + Rho1 = maps:merge( Rho, DRho ), + Gamma1 = maps:merge( Gamma, DGamma ), + case Query of + undef -> server_loop( Rho1, Gamma1, Cwd ); + _ -> + try cuneiform:reduce( Query, Rho1, Gamma1, "." ) of + X -> io:format( "~s~n", [cuneiform:format_result( X )] ) + catch + throw:T -> io:format( "~s~n", [cuneiform:format_error( T )] ) + end, + server_loop( Rho1, Gamma1, Cwd ) + end + end. + +read_expression( Prompt ) -> + io:format( Prompt ), + read( [] ). + +read( Buf ) -> + case io:get_line( "" ) of + eof -> {ctl, quit}; + S -> + T = Buf++S, + {ok, TokenLst, _} = cf_prescan:string( T ), + case TokenLst of + [] -> + case Buf of + [] -> {ok, {undef, #{}, #{}}}; % nothing entered or buffered + [_|_] -> read( Buf ) % nothing new entered + end; + [_|_] -> % something was entered + case is_open( TokenLst ) of + true -> read( T ); % closing paren missing + false -> + case lists:last( TokenLst ) of + semicolon -> cf_parse:string( T ); + rbrace -> cf_parse:string( T ); + rmmecb -> cf_parse:string( T ); + + lbrace -> cf_parse:string( T ); % these are in fact error + lmmecb -> cf_parse:string( T ); % cases but cf_parse will + % figure it out + + nonws -> read( T ); + C -> {ctl, C} + end + end + end + end. + +is_open( TokenLst ) -> + NOpen1 = length( [lbrace || lbrace <- TokenLst] ), + NClose1 = length( [rbrace || rbrace <- TokenLst] ), + NOpen2 = length( [lmmecb || lmmecb <- TokenLst] ), + NClose2 = length( [rmmecb || rmmecb <- TokenLst] ), + NOpen1 > NClose1 orelse NOpen2 > NClose2. + + + +-spec get_help() -> iolist(). + +get_help() -> + string:join( + [?YLW( "help" )++" show this usage info", + ?YLW( "state" )++" show variable bindings", + ?YLW( "tasks" )++" show task definitions", + ?YLW( "cwd" )++" current working directory", + ?YLW( "quit" )++" quit the shell" + ], "\n" ). + + +-spec get_banner() -> iolist(). + +get_banner() -> + string:join( + [" ___", + " @@WB Cuneiform", + " @@E_____ "++cuneiform:get_vsn()++" "++?BUILD, + " _g@@@@@WWWWWWL", + " g@@#*`3@B "++?YLW( "Type " )++?BYLW( "help" )++?YLW( " for usage info." ), + " @@P 3@B", + " @N____ 3@B "++?BLU( "http://www.cuneiform-lang.org" ), + " \"W@@@WF3@B" + ], "\n" ). + + diff --git a/src/cf_sup.erl b/src/cf_sup.erl new file mode 100644 index 0000000..857c1e8 --- /dev/null +++ b/src/cf_sup.erl @@ -0,0 +1,71 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( cf_sup ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-behaviour( supervisor ). + +%% API +-export( [start_link/0, start_cre/3] ). + +%% Supervisor callbacks +-export([init/1]). + + +%% ============================================================================= +%% API functions +%% ============================================================================= + +start_link() -> + supervisor:start_link( {local, ?MODULE}, ?MODULE, [] ). + +%% ============================================================================= +%% Supervisor callbacks +%% ============================================================================= + +init( [] ) -> + + RestartStrategy = one_for_one, + MaxRestarts = 5, + MaxSecondsBetweenRestarts = 10, + SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, + + + {ok, {SupFlags, []}}. + +-spec start_cre( Mod, ModArg, LibMap ) -> {ok, pid()} +when Mod :: atom(), + ModArg :: term(), + LibMap :: #{cf_sem:lang() => [string()]}. + +start_cre( Mod, ModArg, LibMap ) +when is_atom( Mod ), is_map( LibMap ) -> + + Restart = temporary, + Shutdown = 2000, + Type = worker, + + Cre = {cre, {cf_cre, start_link, [Mod, ModArg, LibMap]}, Restart, Shutdown, Type, [cf_cre]}, + + supervisor:start_child( cf_sup, Cre ). \ No newline at end of file diff --git a/src/condor.erl b/src/condor.erl new file mode 100644 index 0000000..669f5ee --- /dev/null +++ b/src/condor.erl @@ -0,0 +1,174 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt +%% @author Irina Guberman + +-module( condor ). +-author( "Jorgen Brandt " ). +-author( "Irina Guberman " ). +-vsn( "2.2.0-release" ). + + +-include( "cuneiform.hrl" ). + +-behaviour( cf_cre ). + +-export( [init/1, handle_submit/6] ). + + +init( _ModArg ) -> + BaseDir = local:create_basedir( ?BASEDIR, 1 ), + {ok, BaseDir}. + +handle_submit( Lam, Fa, R, DataDir, LibMap, BaseDir ) -> + CreRef = self(), + _Pid = spawn_link( fun() -> CreRef ! stage( Lam, Fa, R, DataDir, LibMap, BaseDir ) end ), + ok. + + + +stage( Lam={lam, _LamLine, _LamName, {sign, Lo, Li}, _Body}, Fa, R, DataDir, + LibMap, BaseDir ) -> + + Dir = local:create_workdir( BaseDir, ?WORK, R ), + + RepoDir = string:join( [BaseDir, ?REPO], "/" ), + + % resolve input files + Triple1 = lib_refactor:get_refactoring( Li, Fa, Dir, [DataDir, RepoDir], R ), + {RefactorLst1, MissingLst1, Fa1} = Triple1, + + case MissingLst1 of + [_|_] -> {failed, precond, R, MissingLst1}; + [] -> + + % link in input files + lib_refactor:apply_refactoring( RefactorLst1 ), + + + LogFile = string:join( [Dir, "_log.txt"], "/" ), + OutputFile = string:join( [Dir, "_output.txt"], "/" ), + SubmitFile = string:join( [Dir, "_job.sub"], "/" ), + EffiSumFile = string:join( [Dir, "_summary.effi"], "/" ), + EffiRequestFile = string:join( [Dir, "_request.effi"], "/" ), + + SubmitMap = #{universe => "VANILLA", + executable => "/usr/local/bin/effi", + arguments => string:join( [EffiRequestFile, + EffiSumFile], " " ), + should_transfer_files => "IF_NEEDED", + when_to_transfer_output => "ON_EXIT", + log => LogFile, + output => OutputFile, + environment => ["\"", + string:join( ["HOME=.", + "PATH=/bin:/usr/bin:/usr/local/bin"], " " ), + "\""], + initialdir => Dir + }, + + % write Effi submit file + EffiRequest = {Lam, Fa1, R, LibMap}, + EffiRequestStr = io_lib:format( "~p.~n", [EffiRequest] ), + ok = file:write_file( EffiRequestFile, EffiRequestStr ), + + % write HTCondor submit file + SubmitStr = format_submit( SubmitMap ), + ok = file:write_file( SubmitFile, SubmitStr ), + error_logger:info_msg( "~s~n", [SubmitStr] ), + + % submit job + case lib_os:cmd( string:join( ["condor_submit", SubmitFile], " " ), Dir ) of + + {error, B1} -> {failed, cre_error, R, B1}; + ok -> + + % wait until the job terminates + case lib_os:cmd( string:join( ["condor_wait", LogFile], " " ), Dir ) of + + {error, B2} -> {failed, cre_error, R, B2}; + ok -> + + {ok, B} = file:read_file( EffiSumFile ), + {ok, Tokens, _} = erl_scan:string( binary_to_list( B ) ), + {ok, Sum} = erl_parse:parse_term( Tokens ), + + case maps:get( state, Sum ) of + + ok -> + + % resolve output files + RMap = maps:get( ret, Sum ), + {lam, _Line, _LamName, Sign, _Body} = Lam, + {sign, Lo, _Li} = Sign, + {RefactorLst, [], RMap1} = lib_refactor:get_refactoring( Lo, RMap, RepoDir, [Dir], R ), + ok = lib_refactor:apply_refactoring( RefactorLst ), + + {finished, maps:put( ret, RMap1, Sum )}; + + script_error -> + + #{actscript := ActScript, out := Out} = Sum, + {failed, script_error, R, {ActScript, Out}}; + + R1 -> + + #{missing := MissingLst2} = Sum, + {failed, R1, R, MissingLst2} + end + end + end + end. + + + +%% @doc Takes a Condor submit map and formats it as a binary. +%% +%% Takes the Condor submit map and produces a string having 'key = value' +%% format. The order of key-value pairs in the output binary is undefined. +%% The resulting binary is a complete, valid condor submit file leaving it +%% to the caller to write the content to disk. +%% +-spec format_submit( Condorparams0 ) -> binary() +when Condorparams0::#{atom() => string() | [string()]}. + +format_submit(CondorParams0) -> + CondorParams1 = input_files_to_cs_string(CondorParams0), + LineSep = io_lib:nl(), + SubmitFileStr = maps:fold(fun(K, V, Acc) -> [Acc, io_lib:format("~p = ~s", [K,V]), LineSep] end, "", CondorParams1), + iolist_to_binary([LineSep, lists:flatten(SubmitFileStr), LineSep, "Queue", LineSep]). + +%% @doc Converts the transfer_input_files field from a Condor submit map to a +%% comma-separated string. +%% +%% The only parameter that might come here as a list instead of complete +%% cs-string is transfer_input_files because it might be generated by +%% Cuneiform and it needs file validation. Therefore any declared +%% transfer_input_files params need to be first parsed from cs-string +%% into a list, and once they are validated and possibly merged, they are +%% converted back to cs-string here. +%% +-spec input_files_to_cs_string( CondorParams ) -> #{atom() => string()} +when CondorParams :: #{atom() => string() | [string()]}. + +input_files_to_cs_string(#{transfer_input_files := InputFiles} = CondorParams) -> + InputFilesCS = string:join(InputFiles, ", "), + maps:put(transfer_input_files, InputFilesCS, CondorParams); + +input_files_to_cs_string(CondorParams) -> CondorParams. \ No newline at end of file diff --git a/src/cuneiform.app.src b/src/cuneiform.app.src new file mode 100644 index 0000000..ae5b7a0 --- /dev/null +++ b/src/cuneiform.app.src @@ -0,0 +1,13 @@ + +{application, cuneiform, + [ + {description, "Cuneiform: A Functional Language for Large Scale Scientific Data Analysis"}, + {vsn, "2.2.0-release"}, + {registered, []}, + {applications, [ + kernel, + stdlib + ]}, + {mod, {cuneiform_app, []}}, + {env, []} + ]}. \ No newline at end of file diff --git a/src/cuneiform.erl b/src/cuneiform.erl new file mode 100644 index 0000000..8268ef1 --- /dev/null +++ b/src/cuneiform.erl @@ -0,0 +1,290 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( cuneiform ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + +% API +-export( [main/1, file/2, start/3, reduce/4, get_vsn/0, format_result/1, format_error/1] ). + + +-include( "cuneiform.hrl" ). + +%% ============================================================================= +%% API functions +%% ============================================================================= + +-spec main( CmdLine::list() ) -> ok. + +main( CmdLine ) -> + case getopt:parse( get_optspec_lst(), CmdLine ) of + {error, {Reason, Data}} -> error( {Reason, Data} ); + {ok, {OptLst, NonOptLst}} -> + case lists:member( version, OptLst ) of + true -> print_vsn(); + false -> + case lists:member( help, OptLst ) of + true -> print_usage(); + false -> + case lists:member( cite, OptLst ) of + true -> print_bibtex(); + false -> + {workdir, Cwd} = lists:keyfind( workdir, 1, OptLst ), + {nthread, NSlot} = lists:keyfind( nthread, 1, OptLst ), + {platform, Platform} = lists:keyfind( platform, 1, OptLst ), + LibMap = get_libmap( OptLst ), + {ok, CrePid} = start( Platform, NSlot, LibMap ), + link( CrePid ), + case NonOptLst of + [] -> cf_shell:server( Cwd ); + [File] -> + case file( File, Cwd ) of + {ok, Ret} -> io:format( "~s~n", [format_result( Ret )] ); + {error, T} -> io:format( "~s~n", [format_error( T )] ) + end + end + end + end + end + end. + +-spec file( File, Cwd ) -> {ok, [cf_sem:str()]} | {error, term()} +when File :: string(), + Cwd :: string(). + +file( File, Cwd ) -> + case cf_parse:file( File ) of + {error, ErrorInfo} -> + {error, ErrorInfo}; + {ok, {Query, Rho, Gamma}} -> + try cuneiform:reduce( Query, Rho, Gamma, Cwd ) of + X -> {ok, X} + catch + throw:T -> {error, T} + end + end. + + +start( Mod, ModArg, LibMap ) +when is_atom( Mod ), is_map( LibMap ) -> + application:start( cuneiform ), + cf_sup:start_cre( Mod, ModArg, LibMap ). + + + + +%% ============================================================================= +%% Internal Functions +%% ============================================================================= + +get_libmap( OptLst ) -> + RLib = [P || {rlib, P} <- OptLst], + PyLib = [P || {pylib, P} <- OptLst], + #{python => PyLib, r => RLib}. + + +%% Reduction %% + +-spec reduce( X0, Rho, Gamma, Cwd ) -> [cf_sem:str()] +when X0 :: [cf_sem:expr()], + Rho :: #{string() => [cf_sem:expr()]}, + Gamma :: #{string() => cf_sem:lam()}, + Cwd :: string(). + +reduce( X0, Rho, Gamma, Cwd ) -> + Mu = fun( A ) -> cf_cre:submit( A, Cwd ) end, + reduce( X0, {Rho, Mu, Gamma, #{}}, Cwd ). + +-spec reduce( X0, Theta, Cwd ) -> [cf_sem:str()] +when X0 :: [cf_sem:expr()], + Theta :: cf_sem:ctx(), + Cwd :: string(). + +reduce( X0, {Rho, Mu, Gamma, Omega}, Cwd ) -> + + X1 = cf_sem:eval( X0, {Rho, Mu, Gamma, Omega} ), + case cf_sem:pfinal( X1 ) of + true -> X1; + false -> + receive + + {failed, R2, R, Data} -> + {AppLine, LamName} = hd( find_select( R, X1 ) ), + throw( {AppLine, cuneiform, {R2, LamName, R, Data}} ); + + + {finished, Summary} -> + Ret = maps:get( ret, Summary ), + R = maps:get( id, Summary ), + Delta = lists:foldl( + fun( N, Delta0 ) -> + acc_delta( N, Delta0, Ret, R ) + end, + #{}, maps:keys( Ret ) ), + + reduce( X1, {Rho, Mu, Gamma, maps:merge( Omega, Delta )}, Cwd ); + + Msg -> error( {bad_msg, Msg} ) + + end + end. + +-spec acc_delta( N, Delta0, Ret, R ) -> #{string() => [cf_sem:str()]} +when N :: string(), + Delta0 :: #{string() => [cf_sem:str()]}, + Ret :: #{string() => [string()]}, + R :: pos_integer(). + +acc_delta( N, Delta0, Ret, R ) -> + Delta0#{{N, R} => maps:get( N, Ret )}. + + +find_select( R, L ) when is_list( L ) -> + lists:flatmap( fun( X ) -> find_select( R, X ) end, L ); + +find_select( R, Fa ) when is_map( Fa ) -> + find_select( R, maps:values( Fa ) ); + +find_select( R, {select, AppLine, _C, {fut, LamName, R, _Lo}} ) -> + [{AppLine, LamName}]; + +find_select( R, {app, _AppLine, _C, _Lambda, Fa} ) -> + find_select( R, Fa ); + +find_select( _, _ ) -> + []. + + + +%% opt_spec_list/0 +% +-spec get_optspec_lst() -> [{atom(), char(), string(), undefined, string()}]. + +get_optspec_lst() -> + NSlot = case erlang:system_info( logical_processors_available ) of + unknown -> 1; + N -> N + end, + [ + {version, $v, "version", undefined, "show Cuneiform version"}, + {help, $h, "help", undefined, "show command line options"}, + {cite, $c, "cite", undefined, "show Bibtex entry for citation"}, + {workdir, $w, "workdir", {string, "."}, "working directory"}, + {nthread, $t, "nthread", {integer, NSlot}, "number of threads in local mode"}, + {platform, $p, "platform", {atom, local}, "platform to use: local, condor"}, + {rlib, undefined, "rlib", string, "include R library path"}, + {pylib, undefined, "pylib", string, "include Python library path"} + ]. + +-spec get_vsn() -> string(). + +get_vsn() -> + {vsn, Vsn} = lists:keyfind( vsn, 1, module_info( attributes ) ), + Vsn. + + +%% print_bibtex/0 +% +-spec print_bibtex() -> ok. + +print_bibtex() -> io:format( "~n~s~n~n", [get_bibtex()] ). + + +%% get_bibtex/0 +% +-spec get_bibtex() -> iolist(). + +get_bibtex() -> + string:join( ["@InProceedings{Brandt2015,", + " Title = {Cuneiform: A Functional Language for Large Scale Scientific Data Analysis},", + " Author = {Brandt, J{\"o}rgen and Bux, Marc and Leser, Ulf},", + " Booktitle = {Proceedings of the Workshops of the EDBT/ICDT},", + " Year = {2015},", + " Address = {Brussels, Belgium},", + " Month = {March},", + " Pages = {17--26},", + " Volume = {1330},", + " Abstract = {The need to analyze massive scientific data sets on the one hand and the availability of distributed compute resources with an increasing number of CPU cores on the other hand have promoted the development of a variety of languages and systems for parallel, distributed data analysis. Among them are data-parallel query languages such as Pig Latin or Spark as well as scientific workflow languages such as Swift or Pegasus DAX. While data-parallel query languages focus on the exploitation of data parallelism, scientific workflow languages focus on the integration of external tools and libraries. However, a language that combines easy integration of arbitrary tools, treated as black boxes, with the ability to fully exploit data parallelism does not exist yet. Here, we present Cuneiform, a novel language for large-scale scientific data analysis. We highlight its functionality with respect to a set of desirable features for such languages, introduce its syntax and semantics by example, and show its flexibility and conciseness with use cases, including a complex real-life workflow from the area of genome research. Cuneiform scripts are executed dynamically on the workflow execution platform Hi-WAY which is based on Hadoop YARN. The language Cuneiform, including tool support for programming, workflow visualization, debugging, logging, and provenance-tracing, and the parallel execution engine Hi-WAY are fully implemented.},", + " Doi = {10.13140/RG.2.1.3547.6561},", + " Url = {http://ceur-ws.org/Vol-1330/paper-03.pdf}", + "}"], "\n" ). + +%% print_usage/0 +% +-spec print_usage() -> ok. + +print_usage() -> getopt:usage( get_optspec_lst(), "cuneiform", "[]" ). + + +%% print_vsn/0 +% +-spec print_vsn() -> ok. + +print_vsn() -> io:format( "~s~n", [get_vsn()] ). + +-spec format_result( StrLst::[cf_sem:str()] ) -> iolist(). + +format_result( StrLst ) -> + F = fun( {str, S}, AccIn ) -> + io_lib:format( "~s\"~s\" ", [AccIn, S] ) + end, + io_lib:format( ?GRN( "~s" ), [lists:foldl( F, "", StrLst )] ). + + +-spec format_out( [binary()] ) -> iolist(). + +format_out( Out ) -> + [io_lib:format( "~s~n", [Line] ) || Line <- Out]. + + + +-spec format_script( ActScript::iolist() ) -> iolist(). + +format_script( ActScript ) -> + {_, S} = lists:foldl( fun( Line, {N, S} ) -> + {N+1, io_lib:format( "~s~4.B ~s~n", [S, N, Line] )} + end, + {1, []}, re:split( ActScript, "\n" ) ), + S. + + +format_error( {Line, cf_scan, {illegal, Token}} ) -> + io_lib:format( ?RED( "Line ~p: " )++?BRED( "illegal token ~s" ), [Line, Token] ); + +format_error( {Line, cf_parse, S} ) -> + io_lib:format( ?RED( "Line ~p: " )++?BRED( "~s" ), [Line, S] ); + +format_error( {Line, cf_sem, S} ) -> + io_lib:format( ?RED( "Line ~p: " )++?BRED( "~s" ), [Line, S] ); + +format_error( {AppLine, cuneiform, {script_error, LamName, R, {ActScript, Out}}} ) -> + io_lib:format( + ?BYLW( "[out]~n" )++?YLW( "~s~n" ) + ++?BYLW( "[script]~n" )++?YLW( "~s~n" ) + ++?RED( "Line ~p: " )++?BRED( "script error in call to ~s (id: ~p)" ), + [format_out( Out ), format_script( ActScript ), AppLine, LamName, R] ); + +format_error( {AppLine, cuneiform, {precond, LamName, R, MissingLst}} ) -> + io_lib:format( ?RED( "Line ~p: " )++?BRED( "precondition not met in call to ~s (~p)~n" )++?RED( "Missing files: ~p" ), [AppLine, LamName, R, MissingLst] ); + +format_error( ErrorInfo ) -> + io_lib:format( ?RED( "Error: " )++?BRED( "~p" ), [ErrorInfo] ). \ No newline at end of file diff --git a/src/cuneiform_app.erl b/src/cuneiform_app.erl new file mode 100644 index 0000000..b09d25e --- /dev/null +++ b/src/cuneiform_app.erl @@ -0,0 +1,37 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( cuneiform_app ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-behaviour( application ). + +% Application +-export( [start/2, stop/1] ). + +%% ============================================================================= +%% Application callbacks +%% ============================================================================= + +start( normal, [] ) -> cf_sup:start_link(). +stop( _State ) -> ok. \ No newline at end of file diff --git a/src/gen_queue.erl b/src/gen_queue.erl new file mode 100644 index 0000000..86465cb --- /dev/null +++ b/src/gen_queue.erl @@ -0,0 +1,102 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( gen_queue ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + +-behaviour( gen_server ). + +%% ============================================================================= +%% Function Exports +%% ============================================================================= + +-export( [start_link/1, stage_reply/2] ). + +-export( [code_change/3, handle_cast/2, handle_info/2, init/1, terminate/2, + handle_call/3] ). + +%% ============================================================================= +%% Type Definitions +%% ============================================================================= + +-type job() :: {Mod::atom(), Fun::atom(), ArgLst::[term()]}. +-type request() :: {request, pid(), job()}. +-type queue_state() :: {NSlot::non_neg_integer(), Q::[request()]}. + +%% ============================================================================= +%% Generic Server Functions +%% ============================================================================= + +code_change( _OldVsn, State, _Extra ) -> {ok, State}. +handle_call( Request, _From, _State ) -> error( {bad_request, Request} ). +terminate( _Reason, _State ) -> ok. +handle_info( Info, _State ) -> error( {bad_msg, Info} ). + +-spec init( NSlot::pos_integer() ) -> {ok, queue_state()}. + +init( NSlot ) -> {ok, {NSlot,[]}}. + +-spec handle_cast( Request, State ) -> {noreply, queue_state()} +when Request :: request(), + State :: queue_state(). + +handle_cast( continue, {NSlot, []} ) -> + {noreply, {NSlot+1, []}}; + +handle_cast( continue, {NSlot, [H|T]} ) -> + QueueRef = self(), + spawn_link( fun() -> stage_reply( QueueRef, H ) end ), + {noreply, {NSlot, T}}; + +handle_cast( Request, {0, Q} ) -> + {noreply, {0, [Request|Q]}}; + +handle_cast( Request, {NSlot, []} ) -> + QueueRef = self(), + spawn_link( fun() -> stage_reply( QueueRef, Request ) end ), + {noreply, {NSlot-1, []}}. + +%% ============================================================================= +%% API Functions +%% ============================================================================= + +-spec start_link( NSlot ) -> {ok, pid()} | ignore | {error, term()} +when NSlot :: pos_integer(). + +start_link( NSlot ) +when is_integer( NSlot ), NSlot > 0 -> + gen_server:start_link( ?MODULE, NSlot, [] ). + +-spec stage_reply( QueueRef, Request ) -> ok +when QueueRef :: pid(), + Request :: request(). + +stage_reply( QueueRef, {request, Pid, {Mod, Fun, ArgLst}} ) +when is_pid( QueueRef ), + is_pid( Pid ), + is_atom( Mod ), + is_atom( Fun ), + is_list( ArgLst ) -> + + Reply = apply( Mod, Fun, ArgLst ), + Pid ! Reply, + gen_server:cast( QueueRef, continue ). \ No newline at end of file diff --git a/src/lib_os.erl b/src/lib_os.erl new file mode 100644 index 0000000..63bf915 --- /dev/null +++ b/src/lib_os.erl @@ -0,0 +1,77 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + +-module( lib_os ). +-vsn( "2.2.0-release" ). +-author( "Jorgen Brandt " ). + +-export( [cmd/2] ). + +-define( BUF_SIZE, 1024 ). + + + + +-spec cmd( Cmd::string(), Dir::string() ) -> ok | {error, binary()}. + +cmd( Cmd, Dir ) when is_list( Cmd ), is_list( Dir ) -> + + Port = open_port( {spawn, Cmd}, + [exit_status, + stderr_to_stdout, + binary, + {cd, Dir}, + {line, ?BUF_SIZE}] ), + + listen_port( Port, <<>> ). + + + + + +-spec listen_port( Port::port(), OutAcc::binary() ) -> ok | {error, binary()}. + +listen_port( Port, OutAcc ) when is_port( Port ), is_binary( OutAcc ) -> + + receive + + % no line feed, buffer line and continue + {Port, {data, {noeol, PartLine}}} -> + OutAcc1 = <>, + listen_port( Port, OutAcc1 ); + + % line feed encountered + {Port, {data, {eol, PartLine}}} -> + OutAcc1 = <>, + listen_port( Port, OutAcc1 ); + + % process succeeded + {Port, {exit_status, 0}} -> + ok; + + % process failed + {Port, {exit_status, _}} -> + {error, OutAcc}; + + % if nothing matches, raise error + Msg -> + error( {bad_msg, Msg} ) + + end. diff --git a/src/local.erl b/src/local.erl new file mode 100644 index 0000000..c14faee --- /dev/null +++ b/src/local.erl @@ -0,0 +1,151 @@ +%% -*- erlang -*- +%% +%% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +%% +%% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. + +%% @author Jörgen Brandt + + +-module( local ). +-author( "Jorgen Brandt " ). +-vsn( "2.2.0-release" ). + + +-include( "cuneiform.hrl" ). + +-behaviour( cf_cre ). +-export( [init/1, handle_submit/6, stage/6, create_basedir/2, + create_workdir/3] ). + +-spec create_basedir( Prefix::string(), I::pos_integer() ) -> iolist(). + +create_basedir( Prefix, I ) +when is_list( Prefix ), + is_integer( I ), I > 0 -> + + BaseDir = [Prefix, "-", integer_to_list( I )], + + case filelib:is_file( BaseDir ) of + true -> create_basedir( Prefix, I+1 ); + false -> + case filelib:ensure_dir( [BaseDir, "/"] ) of + {error, Reason} -> error( {Reason, ensure_dir, [BaseDir, "/"]} ); + ok -> BaseDir + end + end. + +-spec init( NSlot::pos_integer() ) -> {ok, {iolist(), pid()}}. + +init( NSlot ) when is_integer( NSlot ), NSlot > 0 -> + BaseDir = create_basedir( ?BASEDIR, 1 ), + {ok, QueueRef} = gen_queue:start_link( NSlot ), + {ok, {BaseDir, QueueRef}}. + + +-spec handle_submit( Lam, Fa, R, DataDir, LibMap, {BaseDir, QueueRef} ) -> ok +when Lam :: cre:lam(), + Fa :: #{string() => [cre:str()]}, + R :: pos_integer(), + DataDir :: string(), + LibMap :: #{cf_sem:lang() => [string()]}, + BaseDir :: iolist(), + QueueRef :: pid(). + +handle_submit( Lam, Fa, R, DataDir, LibMap, {BaseDir, QueueRef} ) +when is_tuple( Lam ), + is_map( Fa ), + is_integer( R ), R > 0, + is_list( DataDir ), + is_map( LibMap ), + is_list( BaseDir ), + is_pid( QueueRef ) -> + gen_server:cast( QueueRef, {request, self(), + {?MODULE, stage, [Lam, Fa, R, DataDir, LibMap, BaseDir]}} ). + + +-spec create_workdir( BaseDir, Work, R ) -> string() +when BaseDir :: string(), + Work :: string(), + R :: pos_integer(). + +create_workdir( BaseDir, Work, R ) +when is_list( BaseDir ), + is_list( Work ), + is_integer( R ), R > 0 -> + + Dir = string:join( [BaseDir, Work, integer_to_list( R )], "/" ), + + % create working directory + case filelib:ensure_dir( [Dir, "/"] ) of + {error, R1} -> error( {R1, ensure_dir, [Dir, "/"]} ); + ok -> Dir + end. + + +-spec stage( Lam, Fa, R, DataDir, LibMap, BaseDir ) -> cre:response() +when Lam :: cre:lam(), + Fa :: #{string() => [cre:str()]}, + R :: pos_integer(), + DataDir :: string(), + LibMap :: #{cf_sem:lang() => [string()]}, + BaseDir :: iolist(). + +stage( Lam={lam, _LamLine, _LamName, {sign, Lo, Li}, _Body}, + Fa, R, DataDir, LibMap, BaseDir ) +when is_list( Lo ), + is_list( Li ), + is_map( Fa ), + is_integer( R ), R > 0, + is_list( DataDir ), + is_map( LibMap ), + is_list( BaseDir ) -> + + + Dir = create_workdir( BaseDir, ?WORK, R ), + + RepoDir = string:join( [BaseDir, ?REPO], "/" ), + + % resolve input files + Triple1 = lib_refactor:get_refactoring( Li, Fa, Dir, [DataDir, RepoDir], R ), + {RefactorLst1, MissingLst1, Fa1} = Triple1, + + case MissingLst1 of + [_|_] -> {failed, precond, R, MissingLst1}; + [] -> + + % link in input files + lib_refactor:apply_refactoring( RefactorLst1 ), + + % start effi + case effi:check_run( Lam, Fa1, R, Dir, LibMap ) of + + {failed, R2, R, Data} -> {failed, R2, R, Data}; + + {finished, Sum} -> + + Ret1 = maps:get( ret, Sum ), + + % resolve output files + Triple2 = lib_refactor:get_refactoring( Lo, Ret1, RepoDir, [Dir], R ), + {RefactorLst2, [], Ret2} = Triple2, + + % link out output files + lib_refactor:apply_refactoring( RefactorLst2 ), + + % update result map + {finished, Sum#{ret => Ret2}} + end + end. \ No newline at end of file diff --git a/test/cf_sem_test.erl b/test/cf_sem_test.erl new file mode 100644 index 0000000..c046368 --- /dev/null +++ b/test/cf_sem_test.erl @@ -0,0 +1,353 @@ +%% -*- erlang -*- +% +% Cuneiform: A Functional Language for Large Scale Scientific Data Analysis +% +% Copyright 2016 Jörgen Brandt, Marc Bux, and Ulf Leser +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. + +-module( cf_sem_test ). +-author( "Jorgen Brandt " ). + +-include_lib( "eunit/include/eunit.hrl" ). + +-define( THETA0, {#{}, fun mu/1, #{}, #{}} ). + +mu( {app, _AppLine, _C, {lam, _LamLine, LamName, {sign, Lo, _Li}, _B}, _Fa} ) -> + {fut, LamName, random:uniform( 1000000000 ), Lo}. + +nil_should_eval_itself_test() -> + ?assertEqual( [], cf_sem:eval( [], ?THETA0 ) ). + +str_should_eval_itself_test() -> + E = [{str, "bla"}], + ?assertEqual( E, cf_sem:eval( E, ?THETA0 ) ). + +undef_var_should_fail_test() -> + E = [{var, 1, "x"}], + ?assertThrow( {1, cf_sem, _}, cf_sem:eval( E, ?THETA0 ) ). + +def_var_should_eval_to_bound_value_test() -> + E = [{str, "blub"}], + X = cf_sem:eval( [{var, 2, "x"}], {#{"x" => E}, fun mu/1, #{}, #{}} ), + ?assertEqual( E, X ). + +def_var_should_cascade_binding_test() -> + E = [{str, "blub"}], + Theta = {#{"x" => [{var, 2, "y"}], "y" => E}, fun mu/1, #{}, #{}}, + X = cf_sem:eval( [{var, 3, "x"}], Theta ), + ?assertEqual( E, X ). + +def_var_should_cascade_binding_twice_test() -> + A = [{str, "A"}], + Rho = #{"x" => [{var, 2, "y"}], "y" => [{var, 3, "z"}], "z" => A}, + ?assertEqual( A, cf_sem:eval( [{var, 4, "x"}], {Rho, fun mu/1, #{}, #{}} ) ). + +unfinished_fut_should_eval_to_itself_test() -> + Fut = {fut, "f", 1234, [{param, {name, "out", false}, false}]}, + E = [{select, 2, 1, Fut}], + X = cf_sem:eval( E, ?THETA0 ), + ?assertEqual( E, X ). + + +finished_fut_should_eval_to_result_test() -> + Fut = {fut, "f", 1234, [{param, {name, "out", false}, false}]}, + S = {select, 2, 1, Fut}, + F = [{str, "blub"}], + Theta = {#{}, fun mu/1, #{}, #{{"out", 1234} => F}}, + X = cf_sem:eval( [S], Theta ), + ?assertEqual( F, X ). + +noarg_fn_should_eval_plain_test() -> + E = [{str, "bla"}], + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {natbody, #{"out" => E}}, + Lam = {lam, 2, "f", Sign, Body}, + F = [{app, 3, 1, Lam, #{}}], + ?assertEqual( E, cf_sem:eval( F, ?THETA0 ) ). + +noarg_fn_should_eval_body_test() -> + E = [{str, "bla"}], + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {natbody, #{"out" => [{var, 2, "x"}], "x" => E}}, + Lam = {lam, 3, "f", Sign, Body}, + F = [{app, 4, 1, Lam, #{}}], + ?assertEqual( E, cf_sem:eval( F, ?THETA0 ) ). + +fn_call_should_insert_lam_test() -> + E = [{str, "bla"}], + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {natbody, #{"out" => E}}, + Lam = {lam, 2, "f", Sign, Body}, + F = [{app, 3, 1, {var, 4, "f"}, #{}}], + Theta = {#{}, fun mu/1, #{"f" => Lam}, #{}}, + ?assertEqual( E, cf_sem:eval( F, Theta ) ). + +app_with_unbound_lam_should_fail_test() -> + F = [{app, 1, 1, {var, 2, "f"}, #{}}], + ?assertThrow( {1, cf_sem, _}, cf_sem:eval( F, ?THETA0 ) ). + +identity_fn_should_eval_arg_test() -> + E = [{str, "bla"}], + Sign = {sign, [{param, {name, "out", false}, false}], [{param, {name, "inp", false}, false}]}, + Body = {natbody, #{"out" => [{var, 2, "inp"}]}}, + Lam = {lam, 3, "f", Sign, Body}, + F = [{app, 4, 1, Lam, #{"inp" => E}}], + ?assertEqual( E, cf_sem:eval( F, ?THETA0 ) ). + +multiple_output_should_be_bindable_test() -> + Sign = {sign, [{param, {name, "out1", false}, false}, {param, {name, "out2", false}, false}], []}, + E1 = [{str, "bla"}], + E2 = [{str, "blub"}], + Body = {natbody, #{"out1" => E1, "out2" => E2}}, + Lam = {lam, 3, "f", Sign, Body}, + F1 = [{app, 4, 1, Lam, #{}}], + F2 = [{app, 5, 2, Lam, #{}}], + [?assertEqual( E1, cf_sem:eval( F1, ?THETA0 ) ), + ?assertEqual( E2, cf_sem:eval( F2, ?THETA0 ) )]. + +app_should_ignore_calling_context_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {natbody, #{"out" => [{var, 1, "x"}]}}, + Lam = {lam, 2, "f", Sign, Body}, + X = [{app, 3, 1, Lam, #{}}], + Rho = #{"x" => [{str, "blub"}]}, + ?assertThrow( {1, cf_sem, _}, cf_sem:eval( X, {Rho, fun mu/1, #{}, #{}} ) ). + +app_should_hand_down_gamma_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {natbody, #{"out" => [{app, 1, 1, {var, 2, "f"}, #{}}]}}, + Lam = {lam, 3, "g", Sign, Body}, + X = [{app, 4, 1, Lam, #{}}], + E = [{str, "blub"}], + Gamma = #{"f" => {lam, 6, "f", Sign, {natbody, #{"out" => E}}}}, + Theta = {#{}, fun mu/1, Gamma, #{}}, + ?assertEqual( E, cf_sem:eval( X, Theta ) ). + +binding_should_override_body_test() -> + F = [{str, "blub"}], + Sign = {sign, [{param, {name, "out", false}, false}], [{param, {name, "x", false}, false}]}, + Body = {natbody, #{"x" => [{str, "bla"}], "out" => [{var, 3, "x"}]}}, + Lam = {lam, 4, "f", Sign, Body}, + X = [{app, 5, 1, Lam, #{"x" => F}}], + ?assertEqual( F, cf_sem:eval( X, ?THETA0 ) ). + +returning_empty_list_on_nonlist_output_channel_should_fail_test() -> + S = {sign, [{param, {name, "out", false}, false}], []}, + B = {natbody, #{"out" => []}}, + Lam = {lam, 1, "f", S, B}, + X = [{app, 2, 1, Lam, #{}}], + ?assertError( output_sign_mismatch, cf_sem:eval( X, ?THETA0 ) ). + +cross_product_should_be_derivable_test() -> + Sign = {sign, [{param, {name, "out1", false}, false}, {param, {name, "out2", false}, false}], + [{param, {name, "p1", false}, false}, {param, {name, "p2", false}, false}]}, + E1 = [{str, "A"}, {str, "B"}], + E2 = [{str, "1"}, {str, "2"}], + Body = {natbody, #{"out1" => [{var, 5, "p1"}], "out2" => [{var, 6, "p2"}]}}, + Lam = {lam, 7, "f", Sign, Body}, + Binding = #{"p1" => E1, "p2" => E2}, + App1 = [{app, 8, 1, Lam, Binding}], + App2 = [{app, 9, 2, Lam, Binding}], + F1 = [{str, "A"}, {str, "A"}, {str, "B"}, {str, "B"}], + F2 = [{str, "1"}, {str, "2"}, {str, "1"}, {str, "2"}], + [?assertEqual( F1, cf_sem:eval( App1, ?THETA0 ) ), + ?assertEqual( F2, cf_sem:eval( App2, ?THETA0 ) )]. + +dot_product_should_be_derivable1_test() -> + Sign = {sign, [{param, {name, "out1", false}, false}, {param, {name, "out2", false}, false}], + [{correl, [{name, "p1", false}, {name, "p2", false}]}]}, + E1 = [{str, "A"}], + E2 = [{str, "1"}], + Body = {natbody, #{"out1" => [{var, 5, "p1"}], "out2" => [{var, 6, "p2"}]}}, + Lam = {lam, 7, "f", Sign, Body}, + Binding = #{"p1" => E1, "p2" => E2}, + App1 = [{app, 8, 1, Lam, Binding}], + App2 = [{app, 9, 2, Lam, Binding}], + [?assertEqual( E1, cf_sem:eval( App1, ?THETA0 ) ), + ?assertEqual( E2, cf_sem:eval( App2, ?THETA0 ) )]. + +dot_product_should_be_derivable2_test() -> + Sign = {sign, [{param, {name, "out1", false}, false}, {param, {name, "out2", false}, false}], + [{correl, [{name, "p1", false}, {name, "p2", false}]}]}, + E1 = [{str, "A"}, {str, "B"}], + E2 = [{str, "1"}, {str, "2"}], + Body = {natbody, #{"out1" => [{var, 5, "p1"}], "out2" => [{var, 6, "p2"}]}}, + Lam = {lam, 7, "f", Sign, Body}, + Binding = #{"p1" => E1, "p2" => E2}, + App1 = [{app, 8, 1, Lam, Binding}], + App2 = [{app, 9, 2, Lam, Binding}], + [?assertEqual( E1, cf_sem:eval( App1, ?THETA0 ) ), + ?assertEqual( E2, cf_sem:eval( App2, ?THETA0 ) )]. + +dot_product_should_be_derivable3_test() -> + Sign = {sign, [{param, {name, "out1", false}, false}, {param, {name, "out2", false}, false}], + [{correl, [{name, "p1", false}, {name, "p2", false}]}]}, + E1 = [{str, "A"}, {str, "B"}, {str, "C"}], + E2 = [{str, "1"}, {str, "2"}, {str, "3"}], + Body = {natbody, #{"out1" => [{var, 5, "p1"}], "out2" => [{var, 6, "p2"}]}}, + Lam = {lam, 7, "f", Sign, Body}, + Binding = #{"p1" => E1, "p2" => E2}, + App1 = [{app, 8, 1, Lam, Binding}], + App2 = [{app, 9, 2, Lam, Binding}], + [?assertEqual( E1, cf_sem:eval( App1, ?THETA0 ) ), + ?assertEqual( E2, cf_sem:eval( App2, ?THETA0 ) )]. + +aggregate_should_consume_whole_list_test() -> + Sign = {sign, [{param, {name, "out", false}, true}], + [{param, {name, "inp", false}, true}]}, + E1 = [{str, "A"}], + E2 = [{str, "B"}, {str, "C"}], + Body = {natbody, #{"out" => E1++[{var, 4, "inp"}]}}, + Lam = {lam, 5, "f", Sign, Body}, + Binding = #{"inp" => E2}, + App = [{app, 6, 1, Lam, Binding}], + ?assertEqual( E1++E2, cf_sem:eval( App, ?THETA0 ) ). + +cnd_false_should_eval_else_expr_test() -> + E = [{cnd, 1, [], [{str, "A"}], [{str, "B"}]}], + ?assertEqual( [{str, "B"}], cf_sem:eval( E, ?THETA0 ) ). + +cnd_evaluates_condition_before_decision1_test() -> + Sign = {sign, [{param, {name, "out", false}, true}], []}, + Body = {natbody, #{"out" => []}}, + Lam = {lam, 1, "f", Sign, Body}, + App = [{app, 2, 1, Lam, #{}}], + E = [{cnd, 3, App, [{str, "A"}], [{str, "B"}]}], + ?assertEqual( [{str, "B"}], cf_sem:eval( E, ?THETA0 ) ). + +cnd_evaluates_condition_before_decision2_test() -> + Sign = {sign, [{param, {name, "out", false}, true}], []}, + Body = {natbody, #{"out" => [{str, "X"}]}}, + Lam = {lam, 2, "f", Sign, Body}, + App = [{app, 3, 1, Lam, #{}}], + E = [{cnd, 4, App, [{str, "A"}], [{str, "B"}]}], + ?assertEqual( [{str, "A"}], cf_sem:eval( E, ?THETA0 ) ). + +cnd_evaluates_only_on_final_condition_test() -> + Sign = {sign, [{param, {name, "out", false}, true}], []}, + Lam = {lam, 1, "f", Sign, {forbody, bash, "shalala"}}, + App = [{app, 2, 1, Lam, #{}}], + A = [{var, 3, "a"}], + B = [{var, 4, "b"}], + E = [{cnd, 5, App, A, B}], + Rho = #{"a" => [{str, "A"}], "b" => [{str, "B"}]}, + X = cf_sem:eval( E, {Rho, fun mu/1, #{}, #{}} ), + ?assertMatch( [{cnd, 5, [{select, 2, 1, _}], A, B}], X ). + +cnd_evaluates_then_expr_test() -> + E = [{cnd, 1, [{str, "Z"}], [{var, 3, "x"}], [{str, "B"}]}], + F = [{str, "A"}], + Theta = {#{"x" => F}, fun mu/1, #{}, #{}}, + ?assertEqual( F, cf_sem:eval( E, Theta ) ). + +cnd_evaluates_else_expr_test() -> + E = [{cnd, 1, [], [{str, "B"}], [{var, 3, "x"}]}], + F = [{str, "A"}], + Theta = {#{"x" => F}, fun mu/1, #{}, #{}}, + ?assertEqual( F, cf_sem:eval( E, Theta ) ). + +foreign_app_with_cnd_param_is_left_untouched_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], [{param, {name, "p", false}, false}]}, + Lam = {lam, 1, "f", Sign, {forbody, bash, "shalala"}}, + App1 = [{app, 2, 1, Lam, #{"p" => [{str, "A"}]}}], + E = [{cnd, 4, App1, [], []}], + App2 = [{app, 5, 1, Lam, #{"p" => E}}], + X = cf_sem:eval( App2, ?THETA0 ), + ?assertMatch( [{app, 5, 1, Lam, _}], X ). + +foreign_app_with_select_param_is_left_untouched_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], + [{param, {name, "p", false}, false}]}, + Lam = {lam, 1, "f", Sign, {forbody, bash, "shalala"}}, + App1 = [{app, 2, 1, Lam, #{"p" => [{str, "A"}]}}], + App2 = [{app, 4, 1, Lam, #{"p" => App1}}], + X = cf_sem:eval( App2, ?THETA0 ), + ?assertMatch( [{app, 4, 1, Lam, _}], X ). + +app_non_final_result_preserves_app_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Select = [{select, 1, 1, {fut, "f", 1234, [{param, {name, "out", false}, false}]}}], + Cnd = [{cnd, 1, Select, [{var, 2, "x"}], [{var, 3, "y"}]}], + Body = {natbody, #{"out" => Cnd}}, + Lam = {lam, 3, "g", Sign, Body}, + App = [{app, 4, 1, Lam, #{}}], + X = cf_sem:eval( App, ?THETA0 ), + ?assertMatch( App, X ). + +app_tail_recursion_is_optimized_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Select = [{select, 1, 1, {fut, "f", 1234, [{param, {name, "out", false}, false}]}}], + Body = {natbody, #{"out" => Select}}, + Lam = {lam, 3, "g", Sign, Body}, + App = [{app, 4, 1, Lam, #{}}], + X = cf_sem:eval( App, ?THETA0 ), + ?assertMatch( Select, X ). + +app_non_final_result_preserves_app_with_new_lam_test() -> + CSign = {sign, [{param, {name, "out", false}, true}], []}, + CLam = {lam, 1, "f", CSign, {forbody, bash, "shalala"}}, + CApp = [{app, 2, 1, CLam, #{}}], + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Body = {natbody, #{"out" => [{cnd, 3, CApp, [{var, 4, "x"}], [{var, 5, "x"}]}], + "x" => [{str, "A"}]}}, + Lam = {lam, 7, "f", Sign, Body}, + App = [{app, 8, 1, Lam, #{}}], + X = cf_sem:eval( App, ?THETA0 ), + [{app, 8, 1, {lam, 7, "f", Sign, {natbody, BodyMap1}}, #{}}] = X, + Val = maps:get( "out", BodyMap1 ), + ?assertMatch( [{cnd, 3, [{select, 2, 1, _}], [{var, 4, "x"}], [{var, 5, "x"}]}], Val ). + +nested_app_undergoes_reduction_test() -> + Sign = {sign, [{param, {name, "out", false}, false}], []}, + Lam1 = {lam, 1, "f", Sign, {forbody, bash, "shalala"}}, + App1 = [{app, 2, 1, Lam1, #{}}], + Body2 = {natbody, #{"out" => App1}}, + Lam2 = {lam, 3, "g", Sign, Body2}, + App2 = [{app, 4, 1, Lam2, #{}}], + X = cf_sem:eval( App2, ?THETA0 ), + [{select, 2, 1, {fut, "f", R, _}}] = X, + Omega = #{{"out", R} => [{str, "A"}]}, + Y = cf_sem:eval( X, {#{}, fun mu/1, #{}, Omega} ), + ?assertEqual( [{str, "A"}], Y ). + +app_select_param_is_enumerated_test() -> + Sign1 = {sign, [{param, {name, "out", false}, false}], []}, + Lam1 = {lam, 1, "f", Sign1, {forbody, bash, "shalala"}}, + Sign2 = {sign, [{param, {name, "out", false}, false}], + [{param, {name, "inp", false}, false}]}, + Body2 = {natbody, #{"out" => [{var, 2, "inp"}]}}, + Lam2 = {lam, 3, "g", Sign2, Body2}, + A0 = {app, 4, 1, Lam1, #{}}, + A = [A0, A0], + B = [{app, 5, 1, Lam2, #{"inp" => A}}], + X = cf_sem:eval( B, ?THETA0 ), + ?assertMatch( [{app, 5, 1, _, _}, {app, 5, 1, _, _}], X ). + +% deftask find_clusters( cls( File ) : state( File ) ) { +% cls = state; +% } +% mu0 = 1; +% cls = find_clusters( state: mu0 ); +% cls; +identity_fn_should_resolve_var_test() -> + Lo = [{param, {name, "cls", false}, false}], + Li = [{param, {name, "state", false}, false}], + Sign = {sign, Lo, Li}, + Body = {natbody, #{"cls" => [{var, 1, "state"}]}}, + Lam = {lam, 2, "find_clusters", Sign, Body}, + Fa = #{"state" => [{var, 3, "mu0"}]}, + App = {app, 4, 1, Lam, Fa}, + Rho = #{"cls" => [App], "mu0" => [{str, "1"}]}, + X = [{var, 6, "cls"}], + ?assertEqual( [{str, "1"}], cf_sem:eval( X, {Rho, fun sem:mu/1, #{}, #{}} ) ). \ No newline at end of file