diff --git a/StorletSamples/.gitignore b/StorletSamples/.gitignore new file mode 100644 index 00000000..e8401735 --- /dev/null +++ b/StorletSamples/.gitignore @@ -0,0 +1,2 @@ +/*/bin/* +/bin/ diff --git a/StorletSamples/ExecDepStorlet/build.xml b/StorletSamples/ExecDepStorlet/build.xml new file mode 100644 index 00000000..13b8cb22 --- /dev/null +++ b/StorletSamples/ExecDepStorlet/build.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/StorletSamples/ExecDepStorlet/src/com/ibm/storlet/execdep/ExecDepStorlet.java b/StorletSamples/ExecDepStorlet/src/com/ibm/storlet/execdep/ExecDepStorlet.java new file mode 100644 index 00000000..36e0904d --- /dev/null +++ b/StorletSamples/ExecDepStorlet/src/com/ibm/storlet/execdep/ExecDepStorlet.java @@ -0,0 +1,114 @@ +/*---------------------------------------------------------------------------- + * Copyright IBM Corp. 2015, 2015 All Rights Reserved + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + * --------------------------------------------------------------------------- + */ + +/*============================================================================ + 03-Sep-2014 evgenyl Initial implementation. + ===========================================================================*/ + +package com.ibm.storlet.execdep; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Map.Entry; + +import com.ibm.storlet.common.IStorlet; +import com.ibm.storlet.common.StorletException; +import com.ibm.storlet.common.StorletInputStream; +import com.ibm.storlet.common.StorletLogger; +import com.ibm.storlet.common.StorletObjectOutputStream; +import com.ibm.storlet.common.StorletOutputStream; +import com.ibm.storlet.common.StorletUtils; + + +/*---------------------------------------------------------------------------- + * ExecDepStorlet + * + * This class invokes another executable. + * The idea is to check that the dependencies are set up correctly, i.e. + * copied and "chmod"-ed. + * */ +public class ExecDepStorlet implements IStorlet +{ + private final int nExpectedReturnCode_ = 42; + @Override + public void invoke( ArrayList inputStreams, + ArrayList outputStreams, + Map arg2, + StorletLogger log ) + throws StorletException + { + StorletInputStream sinob = null; + StorletObjectOutputStream sout = null; + try + { + String strContent = "...:::== Inside ExecDepStorlet ==:::..."; + String strTimeStamp = new SimpleDateFormat("dd-MM-yyy HH:mm:ss"). + format(new Date()); + log.emitLog( strContent ); + log.emitLog( strTimeStamp ); + + sinob = inputStreams.get(0); + HashMap md = sinob.getMetadata(); + sout = (StorletObjectOutputStream)outputStreams.get(0); + Iterator> ii = md.entrySet().iterator(); + while( ii.hasNext() ) + { + @SuppressWarnings("rawtypes") + Map.Entry kv = (Map.Entry) ii.next(); + log.emitLog( "[ " + kv.getKey() + " ] = " + kv.getValue() ); + } + // Get the source location of this class image + String strJarPath = StorletUtils.getClassFolder(this.getClass()); + + // Combine the invocation string + String strExec = strJarPath + java.io.File.separator + "get42"; + log.emitLog( "Exec = " + strExec ); + // Start process, wait for it to finish, get the exit code + Process ExecProc = new ProcessBuilder( strExec ).start(); + int nExitCode = ExecProc.waitFor(); + String strInvRes = "Exit code = " + nExitCode ; + md.put("depend-ret-code", "" + nExitCode ); + sout.setMetadata(md); + log.emitLog( strInvRes ); + } + catch( Exception e ) + { + System.err.print( "Exception: " + e.getMessage() ); + log.emitLog( "Exception: " + e.getMessage() ); + throw new StorletException( e.getMessage() ); + } + finally + { + try + { + if (sinob != null) + sinob.getStream().close(); + if (sout !=null ) + { + sout.getStream().close(); + sout.getMDStream().close(); + } + + } + catch (IOException e) {} + } + } +} diff --git a/StorletSamples/IdentityStorlet/build.xml b/StorletSamples/IdentityStorlet/build.xml new file mode 100644 index 00000000..04772c71 --- /dev/null +++ b/StorletSamples/IdentityStorlet/build.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/StorletSamples/IdentityStorlet/src/com/ibm/storlet/identity/IdentityStorlet.java b/StorletSamples/IdentityStorlet/src/com/ibm/storlet/identity/IdentityStorlet.java new file mode 100644 index 00000000..557cd8b9 --- /dev/null +++ b/StorletSamples/IdentityStorlet/src/com/ibm/storlet/identity/IdentityStorlet.java @@ -0,0 +1,202 @@ +/*---------------------------------------------------------------------------- + * Copyright IBM Corp. 2015, 2015 All Rights Reserved + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + * --------------------------------------------------------------------------- + */ + +/*============================================================================ + 22-Sep-2014 eranr Initial implementation. + ===========================================================================*/ + +package com.ibm.storlet.identity; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Date; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.io.InputStream; +import java.io.OutputStream; + +import com.ibm.storlet.common.IStorlet; +import com.ibm.storlet.common.StorletException; +import com.ibm.storlet.common.StorletInputStream; +import com.ibm.storlet.common.StorletLogger; +import com.ibm.storlet.common.StorletObjectOutputStream; +import com.ibm.storlet.common.StorletContainerHandle; +import com.ibm.storlet.common.StorletOutputStream; +import com.ibm.storlet.common.StorletUtils; + +public class IdentityStorlet implements IStorlet +{ + @Override + public void invoke( ArrayList inputStreams, + ArrayList outputStreams, + Map parameters, + StorletLogger log ) + throws StorletException { + log.emitLog("IdentityStorlet Invoked"); + + /* + * Copy metadata into out md + */ + HashMap md = new HashMap(); + HashMap object_md; + Iterator it; + StorletInputStream sis = inputStreams.get(0); + object_md = sis.getMetadata(); + it = object_md.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pairs = (Map.Entry)it.next(); + log.emitLog("Putting metadata " + (String)pairs.getKey() + "=" + (String)pairs.getValue()); + md.put((String)pairs.getKey(), (String)pairs.getValue()); + } + + /* + * Get optional execute flag + */ + String strExecute = new String("false"); + if (parameters.get("execute") != null) { + strExecute = parameters.get("execute"); + } + boolean bExecute = Boolean.parseBoolean(strExecute); + int nExitCode = -1; + /* + * Execute + */ + if (bExecute == true) { + String strJarPath = StorletUtils.getClassFolder(this.getClass()); + + // Combine the invocation string + String strExec = strJarPath + java.io.File.separator + "get42"; + log.emitLog( "Exec = " + strExec ); + try { + // Start process, wait for it to finish, get the exit code + Process ExecProc = new ProcessBuilder( strExec ).start(); + nExitCode = ExecProc.waitFor(); + log.emitLog( "Exit code = " + nExitCode ); + } catch (Exception e) { + log.emitLog( "Execution failed. Got Exception " + e.getMessage() ); + } + } + + /* + * Get optional chunk size + */ + String strChunkSize = "1024"; + if (parameters.get("chunk_size") != null) { + strChunkSize = parameters.get("chunk_size"); + } + int iChunkSize; + try { + iChunkSize = Integer.parseInt(strChunkSize); + } catch (NumberFormatException e) { + log.emitLog("The chunk_size parameter is not an integer"); + throw new StorletException("The chunk_size parameter is not an integer"); + } + + /* + * 1) If the output stream is StorletObjectOutputStream + * we are in a GET or PUT scenario where we copy the + * data and metadata into it. + * 2) If the output stream is StorletContainerHandle + * we are in a Storlet batch scenario where we first ask + * for a StorletObjectOutputStream, and then do the copy. + */ + StorletObjectOutputStream storletObjectOutputStream; + StorletOutputStream storletOutputStream = outputStreams.get(0); + if (storletOutputStream instanceof StorletContainerHandle) { + log.emitLog("Requesting for output object"); + StorletContainerHandle storletContainerHandle = + (StorletContainerHandle) storletOutputStream; + String objectName = new String(storletContainerHandle.getName()+"/copy_target"); + storletObjectOutputStream = storletContainerHandle.getObjectOutputStream(objectName); + storletContainerHandle.close(); + } else { + storletObjectOutputStream = (StorletObjectOutputStream)outputStreams.get(0); + } + + /* + * add execution invocation result to out md + */ + if (bExecute == true) { + md.put("Execution result", Integer.toString(nExitCode)); + } + /* + * Copy parameters into out md + */ + it = parameters.entrySet().iterator(); + while (it.hasNext()) { + Map.Entry pairs = (Map.Entry)it.next(); + log.emitLog("Putting parameter " + (String)pairs.getKey() + "=" + (String)pairs.getValue()); + md.put("Parameter-" + (String)pairs.getKey(), (String)pairs.getValue()); + } + + /* + * Now set the output metadata + */ + log.emitLog("Setting metadata"); + storletObjectOutputStream.setMetadata(md); + + /* + * Get optional double flag + */ + String strDouble = new String("false"); + if (parameters.get("double") != null) { + strDouble = parameters.get("double"); + } + boolean bDouble = Boolean.parseBoolean(strDouble); + log.emitLog("bDouble is " + bDouble); + + /* + * Copy data from input stream to output stream + */ + log.emitLog("Copying data"); + StorletInputStream psis = (StorletInputStream)inputStreams.get(0); + InputStream is; + is = psis.getStream(); + + OutputStream os = storletObjectOutputStream.getStream(); + final byte[] buffer = new byte[iChunkSize]; + String readString = null; + try { + log.emitLog(new Date().toString() + "About to read from input"); + for (int bytes_read = is.read(buffer); bytes_read >= 0; bytes_read = is.read(buffer)) { + log.emitLog(new Date().toString() + "read from input " + bytes_read + "bytes"); + readString = new String(buffer); + readString = readString.replaceAll("\0", ""); + log.emitLog(new Date().toString() + "Writing to output " + bytes_read + "bytes"); + os.write(readString.getBytes()); + if (bDouble == true) { + log.emitLog("bDouble == true writing again"); + log.emitLog(new Date().toString() + "Writing to output " + bytes_read + "bytes"); + //os.write(buffer); + os.write(readString.getBytes()); + } + log.emitLog("About to read from input"); + } + os.close(); + } catch (Exception e) { + log.emitLog("Copying data from inut stream to output stream failed: " + e.getMessage()); + throw new StorletException("Copying data from inut stream to output stream failed: " + + e.getMessage()); + } finally { + try { + is.close(); + os.close(); + } catch (IOException e) { } + } + log.emitLog("IdentityStorlet Invocation done"); + } +} \ No newline at end of file diff --git a/StorletSamples/TestMetadataStorlet/build.xml b/StorletSamples/TestMetadataStorlet/build.xml new file mode 100644 index 00000000..ede5adaa --- /dev/null +++ b/StorletSamples/TestMetadataStorlet/build.xml @@ -0,0 +1,46 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/StorletSamples/TestMetadataStorlet/src/com/ibm/storlet/testmetadatastorlet/TestMetadataStorlet.java b/StorletSamples/TestMetadataStorlet/src/com/ibm/storlet/testmetadatastorlet/TestMetadataStorlet.java new file mode 100644 index 00000000..165e203e --- /dev/null +++ b/StorletSamples/TestMetadataStorlet/src/com/ibm/storlet/testmetadatastorlet/TestMetadataStorlet.java @@ -0,0 +1,75 @@ +/*---------------------------------------------------------------------------- + * Copyright IBM Corp. 2015, 2015 All Rights Reserved + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + * --------------------------------------------------------------------------- + */ + +package com.ibm.storlet.testmetadatastorlet; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.io.InputStream; +import java.io.OutputStream; + +import com.ibm.storlet.common.IStorlet; +import com.ibm.storlet.common.StorletException; +import com.ibm.storlet.common.StorletInputStream; +import com.ibm.storlet.common.StorletLogger; +import com.ibm.storlet.common.StorletObjectOutputStream; +import com.ibm.storlet.common.StorletOutputStream; + +public class TestMetadataStorlet implements IStorlet +{ + @Override + public void invoke( ArrayList inputStreams, + ArrayList outputStreams, + Map parameters, + StorletLogger log ) + throws StorletException { + log.emitLog("Test Metadata Storlet Invoked"); + final InputStream inputStream = inputStreams.get(0).getStream(); + final HashMap metadata = inputStreams.get(0).getMetadata(); + final StorletObjectOutputStream storletObjectOutputStream = (StorletObjectOutputStream) outputStreams.get(0); + Iterator it = metadata.entrySet().iterator(); + log.emitLog("Printing the input metadata"); + while (it.hasNext()) { + Map.Entry pairs = (Map.Entry)it.next(); + log.emitLog((String)pairs.getKey() + " : "+ (String)pairs.getValue()); + } + + metadata.put("override_key", "new_value"); + it = metadata.entrySet().iterator(); + log.emitLog("Printing the input metadata"); + while (it.hasNext()) { + Map.Entry pairs = (Map.Entry)it.next(); + log.emitLog((String)pairs.getKey() + " : "+ (String)pairs.getValue()); + } + storletObjectOutputStream.setMetadata(metadata); + + OutputStream outputStream = storletObjectOutputStream.getStream(); + try { + byte[] bytearray = new byte[100]; + inputStream.read(bytearray ,0,100); + + outputStream.write("1234567890".getBytes()); + inputStream.close(); + outputStream.close(); + } catch (IOException ex) { + log.emitLog(ex.getMessage()); + throw new StorletException(ex.getMessage()); + } + } +} diff --git a/StorletSamples/TestStorlet/build.xml b/StorletSamples/TestStorlet/build.xml new file mode 100644 index 00000000..e3c4fed8 --- /dev/null +++ b/StorletSamples/TestStorlet/build.xml @@ -0,0 +1,42 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/StorletSamples/TestStorlet/src/com/ibm/storlet/test/test1.java b/StorletSamples/TestStorlet/src/com/ibm/storlet/test/test1.java new file mode 100644 index 00000000..0086c095 --- /dev/null +++ b/StorletSamples/TestStorlet/src/com/ibm/storlet/test/test1.java @@ -0,0 +1,108 @@ +/*---------------------------------------------------------------------------- + * Copyright IBM Corp. 2015, 2015 All Rights Reserved + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * Limitations under the License. + * --------------------------------------------------------------------------- + */ + +/* + * Author: eranr + */ +package com.ibm.storlet.test; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Map; + +import com.ibm.storlet.common.IStorlet; +import com.ibm.storlet.common.StorletException; +import com.ibm.storlet.common.StorletInputStream; +import com.ibm.storlet.common.StorletLogger; +import com.ibm.storlet.common.StorletOutputStream; +import com.ibm.storlet.common.StorletObjectOutputStream; + +public class test1 implements IStorlet{ + /*** + * Storlet invoke method. + * @throws InterruptedException + */ + @Override + public void invoke(ArrayList inputStreams, + ArrayList outputStreams, + Map params, + StorletLogger logger) + throws StorletException + { + try{ + logger.emitLog("In test invoke!"); + logger.emitLog("Iterating over params"); + for (Map.Entry entry : params.entrySet()) + { + String key = entry.getKey(); + String value = entry.getValue(); + logger.emitLog(key+ ":"+value); + } + StorletInputStream sins = inputStreams.get(0); + HashMap md = sins.getMetadata(); + StorletObjectOutputStream outStream = (StorletObjectOutputStream) + outputStreams.get(0); + outStream.setMetadata( md ); + OutputStream stream = outStream.getStream(); + logger.emitLog("About to get param op"); + String op = params.get("op"); + if (op == null) { + logger.emitLog("No op raising..."); + throw new StorletException("no op in params"); + } + logger.emitLog("Got op " + op); + if (op.equals("print")) { + logger.emitLog("op = print"); + String key; + String value; + String s; + for (Map.Entry entry : params.entrySet()) { + key = entry.getKey(); + stream.write(key.getBytes()); + s = " "; + stream.write(s.getBytes()); + value = entry.getValue(); + stream.write(value.getBytes()); + s="\n"; + stream.write(s.getBytes()); + } + stream.close(); + return; + } + + if (op.equals("crash")) { + InputStream a = null; + a.close(); + return; + } + + if (op.equals("hold")) { + Thread.sleep(100000); + } + outStream.getStream().close(); + + } catch (IOException e) { + logger.emitLog(e.getMessage()); + throw new StorletException(e.getMessage()); + } catch (InterruptedException e) { + logger.emitLog(e.getMessage()); + throw new StorletException(e.getMessage()); + } + } +}