Index: GridbusBroker/src/org/gridbus/broker/farming/.cvsignore diff -u GridbusBroker/src/org/gridbus/broker/farming/.cvsignore:1.1.2.1 GridbusBroker/src/org/gridbus/broker/farming/.cvsignore:removed --- GridbusBroker/src/org/gridbus/broker/farming/.cvsignore:1.1.2.1 Mon Oct 25 19:50:41 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/.cvsignore Thu Nov 25 18:48:50 2004 @@ -1 +0,0 @@ -unicore Index: GridbusBroker/src/org/gridbus/broker/farming/common/DataHost.java diff -u GridbusBroker/src/org/gridbus/broker/farming/common/DataHost.java:1.6.2.2 GridbusBroker/src/org/gridbus/broker/farming/common/DataHost.java:1.8 --- GridbusBroker/src/org/gridbus/broker/farming/common/DataHost.java:1.6.2.2 Tue Nov 16 11:38:10 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/common/DataHost.java Sat Nov 6 19:59:40 2004 @@ -111,24 +111,6 @@ } /** - * Returns the full path of the file (including the filename) on this host specified by the filename - * @param filename - * @return string filename - */ - public String getFullFilePath(String filename){ - int x=0; - while(x !=-1){ - // Remove all those '\''s in the filename - if((x=filename.indexOf('\\',x))!=-1){ - String pre=filename.substring(0,x); - String suff=filename.substring(x+1); - filename=pre+suff; - } - } - return (String)FileList.get(filename) + filename; - } - - /** * Returns the hostname. * @return String */ Index: GridbusBroker/src/org/gridbus/broker/farming/common/GridbusFarmingEngine.java diff -u GridbusBroker/src/org/gridbus/broker/farming/common/GridbusFarmingEngine.java:1.6.2.3 GridbusBroker/src/org/gridbus/broker/farming/common/GridbusFarmingEngine.java:1.13 --- GridbusBroker/src/org/gridbus/broker/farming/common/GridbusFarmingEngine.java:1.6.2.3 Tue Nov 23 20:04:42 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/common/GridbusFarmingEngine.java Sat Nov 6 19:59:40 2004 @@ -62,7 +62,8 @@ private Scheduler scheduler=null; private boolean InputDataFilePresent=false; private boolean OutputDataFilePresent=false; - + private String paramName=""; + private float deadline=0; private float budget=0; private boolean feasible=false; @@ -95,13 +96,13 @@ //srikumar - 8/3/04 - Added new property 'ENV' (for environment) to DB.properties //if environment is command-line then generate proxy , if environment is a //Web-server environment like Tomcat then let it set the proxy in the Broker properties - if(BrokerProperties.getProperty("ENV","cli").equalsIgnoreCase("cli")){ - logger.debug("GridbusFarmingEngine() - Generating proxy"); + if(BrokerProperties.getProperty("ENV","web").equals("cli")){ + logger.debug("GridbusFarmingEngine() - web environment, generating proxy"); try { BrokerProperties.setProxy(genProxy()); - logger.debug("GridbusFarmingEngine() - Proxy generated"); + logger.debug("GridbusFarmingEngine() - web environment, proxy generated"); } catch (GSSException e) { - logger.error("GridbusFarmingEngine() - Generating proxy failed",e); + logger.error("GridbusFarmingEngine() - generating proxy failed",e); throw new GridBrokerException("Proxy not initialised : run grid-proxy-init"); } } @@ -348,6 +349,7 @@ BrokerProperties.setProperty("LOCALDIR",val); } + //TODO will be impl later... generate a proxy /** * To be implemented in a future version. * @return GSSCredential proxy Index: GridbusBroker/src/org/gridbus/broker/farming/common/Job.java diff -u GridbusBroker/src/org/gridbus/broker/farming/common/Job.java:1.5.2.1 GridbusBroker/src/org/gridbus/broker/farming/common/Job.java:1.9 --- GridbusBroker/src/org/gridbus/broker/farming/common/Job.java:1.5.2.1 Tue Nov 16 11:38:11 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/common/Job.java Mon Nov 1 13:02:55 2004 @@ -406,12 +406,11 @@ * FileOpsUtil.replaceString(destination,key,(String)variableTable.get(key)); * } * } -* String cwd = BrokerProperties.getProperty("LOCALDIR",System.getProperty("user.dir")); * UrlCopyUtil.copy("gsiftp://" * +server.getHostname()+"/~/" * +source, * "file:///" -* +cwd +* +System.getProperty("user.dir") * +"/"+destination); * System.out.println("got "+source+" from "+server.getHostname()); * } Index: GridbusBroker/src/org/gridbus/broker/farming/common/SrbDataHost.java diff -u GridbusBroker/src/org/gridbus/broker/farming/common/SrbDataHost.java:1.1.2.1 GridbusBroker/src/org/gridbus/broker/farming/common/SrbDataHost.java:1.1 --- GridbusBroker/src/org/gridbus/broker/farming/common/SrbDataHost.java:1.1.2.1 Tue Nov 16 11:38:10 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/common/SrbDataHost.java Mon Nov 1 13:02:55 2004 @@ -24,10 +24,6 @@ * This class contains the methods to work with an SRB Data Host. * @author Krishna Nadiminti (kna@unimelb.edu.au) */ -/* - * SRB URL protocol: - * srb:// [ userName . domainHome [ : password ] @ ] host [ : port ][ / path ] - */ public class SrbDataHost extends DataHost { /** * Logger for this class @@ -37,11 +33,8 @@ private String username=null; private String password=null; private String domain=null; - private String collection; private String srbHost=null; - private String resource=null; private int srbPort=0; - private String auth_scheme; /** * @see DataHost#DataHost(String) @@ -52,6 +45,26 @@ } /** + * Returns the path of the file on this host specified by the filename + * @param filename + * @return string filename + */ + public String getFilePath(String filename){ + String fn = (String)FileList.get(filename); + /* + * SRB URL protocol: + * srb:// [ userName . domainHome [ : password ] @ ] host [ : port ][ / path ] + */ + //not needed now? simple logical container path is enough? + //fn = "srb://"+username+"."+domain+":"+password+"@"+srbHost+":"+srbPort+fn+"/"+filename; + + //this gives the full path from logical SRB Home (root) to the file itself. + //so the /.MdasEnv mdasHome should be set to "/" + fn=fn+"/"+filename; + return fn; + } + + /** * @return Returns the domain. */ public String getDomain() { @@ -133,58 +146,4 @@ this.password = password; } - /** - * @see org.gridbus.broker.farming.common.DataHost#getFullFilePath(java.lang.String) - */ - public String getFullFilePath(String filename){ - //to differentiate between srb filenames and other ones in the JobWrapper - return "srb://"+(String)FileList.get(filename) + filename; - } - - /** - * Gets the resource name for the SRB connection - * @return - */ - public String getResourceName() { - return resource; - } - - /** - * Sets the Resource Name for the SRB Connection - * @param resourceName - */ - public void setResourceName(String resourceName){ - resource = resourceName; - } - - /** - * Gets the collection name for the SRB connection to this host - * @return - */ - public String getCollectionName() { - return collection; - } - - /** - * Sets the collection name for the SRB connection to this host - * @param collectionName - */ - public void setCollection(String collectionName){ - collection = collectionName; - } - - /** - * Gets the Authentication Scheme used with SRB connection - * @return - */ - public String getAuthScheme() { - return auth_scheme; - } - - /** - * @param auth_scheme The auth_scheme to set. - */ - public void setAuthScheme(String auth_scheme) { - this.auth_scheme = auth_scheme; - } } Index: GridbusBroker/src/org/gridbus/broker/farming/common/scrap diff -u GridbusBroker/src/org/gridbus/broker/farming/common/scrap:1.1.2.1 GridbusBroker/src/org/gridbus/broker/farming/common/scrap:removed --- GridbusBroker/src/org/gridbus/broker/farming/common/scrap:1.1.2.1 Mon Oct 25 19:50:40 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/common/scrap Thu Nov 25 18:48:50 2004 @@ -1,3 +0,0 @@ -costFil=new File(filename); -costBuf=new BufferedReader(new FileReader(costFil)); - \ No newline at end of file Index: GridbusBroker/src/org/gridbus/broker/farming/globus/GlobusJobWrapper.java diff -u GridbusBroker/src/org/gridbus/broker/farming/globus/GlobusJobWrapper.java:1.6.2.3 GridbusBroker/src/org/gridbus/broker/farming/globus/GlobusJobWrapper.java:1.10 --- GridbusBroker/src/org/gridbus/broker/farming/globus/GlobusJobWrapper.java:1.6.2.3 Tue Nov 16 17:41:51 2004 +++ GridbusBroker/src/org/gridbus/broker/farming/globus/GlobusJobWrapper.java Sun Nov 7 17:34:22 2004 @@ -1,572 +1,443 @@ -/* - * Title : JobWrapper.java - * Package : gridbus.broker.farming.execution - * Project : HEPscheduler - * Description : Handles the creation of a Globus Job and its submission. Registers a Job Monitor for the Job. - * Created : - * Modified on : Sept. 12 2002 - Added delegation of proxy.. - Srikumar - * Sept. 13 2002 - Added staging in of executable through GASS+RSL - Srikumar. - * Author : Jia Yu (jiayu@cs.mu.oz.au), Srikumar(srikumar@cs.mu.oz.au) - * Copyright : (c) 2003, Grid Computing and Distributed Systems Laboratory, - * Dept. of Computer Science and Software Engineering, - * University of Melbourne, Australia. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any later version. - * See the GNU General Public License (http://www.gnu.org/copyleft/gpl.html)for more details. - * - */ -package org.gridbus.broker.farming.globus; - -import org.apache.log4j.Logger; - -import java.io.File; -import java.io.FileWriter; -import java.io.IOException; -import java.util.Enumeration; -import java.util.Hashtable; -import java.util.Vector; - -import org.globus.gram.Gram; -import org.globus.gram.GramException; -import org.globus.gram.GramJob; -import org.gridbus.broker.farming.common.BrokerProperties; -import org.gridbus.broker.farming.common.Job; -import org.gridbus.broker.farming.common.JobWrapper; -import org.gridbus.broker.farming.common.SrbDataHost; -import org.gridbus.broker.runfile.CopyCommand; -import org.gridbus.broker.runfile.ExecuteCommand; -import org.gridbus.broker.runfile.MCopyCommand; -import org.gridbus.broker.runfile.GCopyCommand; -import org.gridbus.broker.runfile.SingleVariable; -import org.gridbus.broker.runfile.SubstituteCommand; -import org.gridbus.broker.runfile.TaskCommand; -import org.gridbus.broker.scheduler.DataComputePair; -import org.gridbus.broker.util.FileOpsUtil; -import org.gridbus.broker.util.GassServerUtil; -import org.gridbus.broker.util.RSLUtil; -import org.gridforum.jgss.ExtendedGSSManager; -import org.ietf.jgss.GSSCredential; -import org.ietf.jgss.GSSException; - -/** - * This class interacts with the Globus2.4.2 Gatekeeper at the remote host. It creates RSL for the job, - * submits RSL to the remote host and spawns off a GlobusJobMonitor thread to monitor the job. The actual job - * submission is done in a separate thread. - * - * @author Jia Yu (jiayu@cs.mu.oz.au) - * @author Srikumar Venugopal (srikumar@cs.mu.oz.au) - * @author Brett Beeson (bbeeson@ph.unimelb.edu.au) - * @see JobWrapper - */ -public class GlobusJobWrapper extends JobWrapper { - /** - * Logger for this class - */ - private static final Logger logger = Logger.getLogger(GlobusJobWrapper.class); - - private GramJob gramJob; - private GSSCredential proxy; - //private GlobusJobMonitor jobMonitor; - private Thread jobMonitor; - private Thread t; - private GassServerUtil gassServer; - private String jobUrl; - private Job job; - - /** - * Constructor - * @param proxy - * - */ - public GlobusJobWrapper(GSSCredential proxy){ - - this.proxy=proxy; - ExtendedGSSManager manager; - - try { - //cog 1.0 - //proxy=GlobusProxy.getDefaultUserProxy(true); - //cog 1.1 - //if no proxy is specified, then we - if(proxy==null){ - manager=(ExtendedGSSManager) ExtendedGSSManager.getInstance(); - proxy=manager.createCredential(GSSCredential.INITIATE_AND_ACCEPT); - } - } catch (GSSException e) { - logger.error("GlobusJobWrapper(GSSCredential) - Valid proxy missing: " + e.getMessage(), e); - /*Put in code for refreshing an expired proxy - * using the username and the password. - */ - } - } - - /** - * @see org.gridbus.broker.farming.common.JobWrapper#execute(org.gridbus.broker.farming.common.Job) - */ - public void execute(Job job){ - this.job=job; - GramJob gramJob=null; - String errorCodeString="GBX=`echo $?`\n"; - String errorCodeString2="echo \"GBX:$GBX\"\n"; - String errorCheckString="if [ $GBX -ne 0 ]\n"; - String errorCheckString2="then exit $GBX\n"; - String errorCheckString3="fi\n"; - String errorCheckString4="echo *done \n"; - String rDir=null; - String lDir=null; //srikumar - 10/02 - added variable to specify local directory - String shellFileName="";//srikumar - 11/02 - made shellfile reference here . - String srbScript=""; //krishna - 15-Nov-04 - added this to accomodate srb - - this.setGassServer(new GassServerUtil()); - this.getGassServer().setProxy(proxy); - this.getGassServer().start(); - - GlobusComputeServer cs=(GlobusComputeServer)job.getServer(); - job.addVariable(new SingleVariable("$OS", "string", cs.getOS())); - - //srikumar - added LocalDir checking... - try{ - if((lDir=BrokerProperties.getProperty("LOCALDIR"))==null){ - lDir=""; - } - else { - lDir=lDir+"/"; //added a trailing / for directory separation - } - }catch(Exception e){lDir=""; } - - shellFileName=lDir+job.getJobID()+".sh"; - rDir ="DB"+System.currentTimeMillis(); //remote temp directory - - File shellFile=new File(shellFileName); - FileWriter out=null; - - try { - out = new FileWriter(shellFile); - } catch (IOException e1) { - logger.error("execute(Job)", e1); - } - - StringBuffer fileOutputBuf=new StringBuffer(); - // bbeeson : remove -f since 'sh -c "help set"' says - // "-f Disable file name generation (globbing)." - fileOutputBuf.append("#!/bin/sh\n"); - - fileOutputBuf.append("echo *create directory \n"); - fileOutputBuf.append("mkdir "+rDir+"\n"); - fileOutputBuf.append("echo *enter the directory \n"); - fileOutputBuf.append("cd "+rDir+"\n"); - fileOutputBuf.append("source ~/.bashrc\n"); - fileOutputBuf.append("export TMPDIR=`pwd`\n"); - - //check for SRB and put in its init code only once. krishna : 15-Nov-04 - //this will check the job variables and do Sinit stuff if needed - srbScript=genSRBInitScript(job); - if (srbScript.length()>0){ - fileOutputBuf.append(srbScript); - srbScript=""; //reset it to be safe, we'll use it later on - } - - Vector commands=null; - //get a clone of the Vector, since we dont need to change the original one - //especially for the Substitute command, we do a "setDesintation" which changes things for - //all jobs, which is not desirable! - commands=(Vector)job.getTask().getCommands().clone(); - TaskCommand tc; - - String source=null; - String destination=null; - Hashtable variables= job.getVariableValueTable(); - //TODO: put in check if variables are null. - - /* - * Added a few more extra variables to support GCopy. --chd-- - */ - String sourceFile=null; - String sourceNode=null; - String destinationFile=null; - String destinationNode=null; - - for (int i = 0; i < commands.size(); i++) { - // 10/11 -Sri -- Putting in the substitute thingy - if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.SUBSTITUTE_CMD)){ - source=((SubstituteCommand)tc).getSource(); - source=substituteVariables(source,variables); - //srikumar - 11/02-added localDir here for source and destination - //so that substitution happens on files within the localDir - ((SubstituteCommand)tc).setSource(lDir+source); - destination=((SubstituteCommand)tc).getDestination(); - destination=substituteVariables(destination,variables); - ((SubstituteCommand)tc).setDestination(lDir+destination); - ((SubstituteCommand)tc).doSubstitution(variables); - continue; - } - if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.MCOPY_CMD)) { - /* - MULTIPLE FILE COPY - - globus-url-copy only support filename to filename copy - - multiple file copy using /bin/sh commands - - use ls to glob the pattern - - use [ -f ] to remove non-files (eg directories) - - use for ... in to loop over each file and call globus-url-copy - - destination MUST be a existing directory - - source MUST be local - */ - source=((MCopyCommand)tc).getSource(); - destination=((MCopyCommand)tc).getDestination(); - source=substituteVariables(source,variables); - destination=substituteVariables(destination,variables); - - fileOutputBuf.append("echo '*copying "+source+" to "+destination+"'\n"); - fileOutputBuf.append("for file in `ls " + source + "` ; do\n"); - fileOutputBuf.append("if [ -f $file ] ; then\n"); - fileOutputBuf.append("basefile=`basename $file`\n"); - fileOutputBuf.append("$GLOBUS_LOCATION/bin/globus-url-copy "+" "); - - // Local (within for loop) - fileOutputBuf.append("file:$TMPDIR/$file "); - - // Remote (directory) - if(!((MCopyCommand)tc).isDestinationRemote()){ - fileOutputBuf.append(this.getGassServer().getURL()+"/"+lDir); //srikumar - added lDir - }else{ - fileOutputBuf.append("file:$TMPDIR/"); - } - // Write to same filename. - fileOutputBuf.append(destination+"/$basefile\n"); - fileOutputBuf.append(errorCodeString+errorCodeString2); - fileOutputBuf.append(errorCheckString+errorCheckString2+errorCheckString3); - - fileOutputBuf.append("fi\n"); // [ -f $file ] - fileOutputBuf.append("done\n"); // for file in - - }else if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.GCOPY_CMD)) { - - sourceNode=((GCopyCommand)tc).getSourceNode(); - sourceFile=((GCopyCommand)tc).getSourceFile(); - destinationNode=((GCopyCommand)tc).getDestinationNode(); - destinationFile=((GCopyCommand)tc).getDestinationFile(); - source=substituteVariables(sourceNode,variables); - destination=substituteVariables(destinationNode,variables); - - fileOutputBuf.append("echo '*copying from "+sourceNode+" to "+destinationNode+"'\n"); - fileOutputBuf.append("$GLOBUS_LOCATION/bin/globus-url-copy "+" "); - - //TODO need to remove this hardcoding, to gsiftp , to support SRB GCopy - if(sourceNode.equals("localhost")){ - fileOutputBuf.append("file://"); - }else{ - fileOutputBuf.append("gsiftp://"+sourceNode+"/"); - } - fileOutputBuf.append(sourceFile+" "); - - if(destinationNode.equals("localhost")){ - fileOutputBuf.append("file://"); - }else{ - fileOutputBuf.append("gsiftp://"+destinationNode+"/"); - } - fileOutputBuf.append(destinationFile+"\n"); - - }else if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.COPY_CMD)) { - source=((CopyCommand)tc).getSource(); - destination=((CopyCommand)tc).getDestination(); - source=substituteVariables(source,variables); - destination=substituteVariables(destination,variables); - - fileOutputBuf.append("echo *copying "+source+" to "+destination+"\n"); - fileOutputBuf.append("$GLOBUS_LOCATION/bin/globus-url-copy "+" "); - if(!((CopyCommand)tc).isSourceRemote()){ - fileOutputBuf.append(this.getGassServer().getURL()+"/"+lDir); //srikumar - added lDir - }else{ - fileOutputBuf.append("file:$TMPDIR/"); - } - fileOutputBuf.append(source+" "); - - if(!((CopyCommand)tc).isDestinationRemote()){ - fileOutputBuf.append(this.getGassServer().getURL()+"/"+lDir); //srikumar - added lDir - }else{ - fileOutputBuf.append("file:$TMPDIR/"); - } - fileOutputBuf.append(destination+"\n"); - - fileOutputBuf.append(errorCodeString+errorCodeString2); - fileOutputBuf.append(errorCheckString+errorCheckString2+errorCheckString3+errorCheckString4); - - }else{ //we assume it has to be an execute command - String ex=((ExecuteCommand)tc).getExecutable(); - ex=substituteVariables(ex,variables); - - //build the command line - String tempCLI="", tempArg=""; - tempCLI=ex; - Vector args=((ExecuteCommand)tc).getArguments(); - for (int ai = 0; ai < args.size(); ai++) { - String a=(String)args.get(ai); - String value=job.getVariableValue(a); - if(value==null){ - a=substituteVariables(a,variables); - tempArg=a; - }else{ - tempArg=value; - } - //check if it is an SRB file - if (tempArg.startsWith("srb")){ - srbScript=srbScript + genSRBGetScript(tempArg); - tempArg=tempArg.substring(tempArg.lastIndexOf('/')+1); - } - tempCLI=tempCLI+" "+tempArg; - } - - //check if there is any srb script - if (srbScript.length()>0){ - fileOutputBuf.append(srbScript); - srbScript=""; - } - - if(ex.startsWith("./")){ // 8/11 - Sri - if executable has been passed to the remote host then set its permissions - fileOutputBuf.append("echo *setting permissions for executable \n"); - fileOutputBuf.append("chmod u+x "+ex+"\n"); - } - fileOutputBuf.append("echo *executing "+ex+"\n"); - fileOutputBuf.append(tempCLI).append("\n"); - fileOutputBuf.append(errorCodeString+errorCodeString2); - fileOutputBuf.append(errorCheckString+errorCheckString2+errorCheckString3+errorCheckString4); - } - } - - //check if there is any srb script - //no need to do this - /*srbScript=genSRBExitScript(job); - if (srbScript.length()>0){ - fileOutputBuf.append(srbScript); - srbScript=""; - }*/ - - fileOutputBuf.append("cd .. \n"); - // bbeeson: debug by leaving directory - fileOutputBuf.append("rm -rf "+rDir+"\n"); - fileOutputBuf.append("echo *deleted the directory "+"\n"); - - try { - out.write(fileOutputBuf.toString()); - out.close(); - } catch (IOException e) { - logger.error("execute(Job)", e); - } - - logger.warn("JOB SH OUTPUTTED"); - - RSLUtil jobRsl=new RSLUtil(); - String finalRsl=""; jobRsl.setExecutable(shellFileName); - jobRsl.setGassServer(gassServer); - finalRsl=jobRsl.rslOutputSubst(true); - // 9/11 - Sri - Made this change so that the job output is stored on the remote side instead of local.. - finalRsl = finalRsl + "(stdout=x-gass-cache://$(GLOBUS_GRAM_JOB_CONTACT)stdout anExtraTag)" - + "(stderr=x-gass-cache://$(GLOBUS_GRAM_JOB_CONTACT)stderr anExtraTag)"; - - gramJob=new GramJob(finalRsl); - gramJob.setCredentials(proxy); - //gramJob.addListener(jobMonitor); - this.gramJob=gramJob; - - jobRun(cs.getContactString(),job); - // if jobRun returns successfully, make the job status submitted - job.setJobStatus(Job.SUBMITTED); - job.setJobSubmittedTimestamp(); - jobMonitor= new Thread(new GlobusJobMonitor(job), job.getJobID()); - jobMonitor.start(); - } - - private String substituteVariables(String st, Hashtable variables){ - Enumeration e=variables.keys(); - while(e.hasMoreElements()){ - String key=(String)e.nextElement(); - if(st.indexOf(key)!=-1){ - st=FileOpsUtil.replaceString(st,key,(String)variables.get(key)); - } - } - return st; - } - - /** - * @see org.gridbus.broker.farming.common.JobWrapper#terminate() - */ - public void terminate(){ - try{ - this.getGassServer().shutdown(); - this.setGassServer(null); - File shellFile=new File(job.getJobID()+".sh"); - shellFile.delete(); - if(gramJob.getStatus()==GramJob.STATUS_ACTIVE){ - Gram.cancel(gramJob); - //logger.log("JobID:"+jobID+" Job cancel successful"); - //System.out.println("JobID:"+jobID+" Job cancel successful"); - } - if(jobMonitor!=null && jobMonitor.isAlive()){ - jobMonitor.interrupt(); - } - this.jobMonitor=null; - } - catch(Exception e){ - if(e instanceof java.net.ConnectException){ - logger.error("terminate() - cancelJob: failed to connect to jobmanager",e); - }else{ - logger.error("terminate()", e); - logger.error("terminate() - cancelJob: Unknown exception", e); - } - } - } - - /** - * Returns the GSSCredential proxy - * @return the user proxy - */ - public GSSCredential getProxy() { - return proxy; - } - - /** - * Sets the GSS credential - * @param credential - the user credential - */ - public void setProxy(GSSCredential credential) { - proxy = credential; - } - - /** - * Returns the gass server object - * @return gass server - */ - public GassServerUtil getGassServer() { - return gassServer; - } - - /** - * Sets the gass server - * @param util - */ - public void setGassServer(GassServerUtil util) { - gassServer = util; - } - - /** - * Returns the job URL - * @return job url - */ - public String getJobUrl() { - return jobUrl; - } - - /** - * Sets the job url - * @param string - */ - public void setJobUrl(String string) { - jobUrl = string; - } - - private void jobRun(String resourceManagerContact,Job j){ - try{ - String subjectName; - logger.info("jobRun(String, Job) - Submitting job " - + job.getJobID() + " to server " - + resourceManagerContact); - //Gram.request(resourceManagerContact,gramJob); // - submits as an ineractive job - Gram.request(resourceManagerContact,gramJob,true); - j.setJobHandle(gramJob.getIDAsString()); - - logger.info("jobRun(String, Job)" + gramJob.getIDAsString()); - - } catch (GSSException gpe) { - logger.error("jobRun(String, Job) - Error:" + gpe.getMessage(), gpe); - }catch(org.globus.gram.GramException ge){ - logger.error("jobRun(String, Job)" + GramException.getMessage(ge.getErrorCode())); - return; - } - } - - private String genSRBInitScript(Job j){ - StringBuffer script = new StringBuffer(); - try{ - Enumeration enum=job.getVariableTable().elements(); - while(enum.hasMoreElements()){ - SingleVariable var=(SingleVariable)enum.nextElement(); - if((var.getType()!=null)&&(var.getType().equals("Gridfile")))//has data file - { - if (var.getValue().startsWith("srb")){ - DataComputePair dcp= (DataComputePair) job.getSubmitRecord(); - SrbDataHost sdh = (SrbDataHost)dcp.getDataHost(); - script.append("echo *Setting SRB Env variables\n"); - script.append("export srbUser='"+sdh.getUsername()+"'\n"); - script.append("export mdasDomainName='"+sdh.getDomain()+"'\n"); - script.append("export srbHost="+sdh.getSrbHost()+"\n"); - script.append("export srbPort="+sdh.getSrbPort()+"\n"); - script.append("export mdasResourceName='"+sdh.getResourceName()+"'\n"); - script.append("export mdasCollectionName='"+sdh.getCollectionName()+"'\n"); - script.append("export srbAuth='"+sdh.getPassword()+"'\n"); - script.append("export AUTH_SCHEME='ENCRYPT1'\n"); - //script.append("echo *Starting SRB Env - Sinit\n"); - //script.append("Sinit\n"); - } - } - } - }catch (Exception e){ - logger.error("genSRBInitScript(Job j)"+e.getMessage()); - } - - return script.toString(); - } - - /** - * Checks if the fileName passed in is an SRB file and if so, adds an Sget script - * otherwise the filename is returned as it is. - * @param filName - * @return - */ - private String genSRBGetScript(String filName){ - StringBuffer script = new StringBuffer(); - String errorCodeString="GBX=`echo $?`\n"; - String errorCodeString2="echo \"GBX:$GBX\"\n"; - String errorCheckString="if [ $GBX -ne 0 ]\n"; - String errorCheckString2="then exit $GBX\n"; - String errorCheckString3="fi\n"; - String errorCheckString4="echo *Sget done \n"; - - if (filName.startsWith("srb")){ - filName=filName.substring(6); // "srb://" is 6 characters long, we just skip that - script.append("echo *executing Sget "+filName+"\n"); - script.append("Sget "+filName+" .\n"); - script.append(errorCodeString); - script.append(errorCodeString2); - script.append(errorCheckString); - script.append(errorCheckString2); - script.append(errorCheckString3); - script.append(errorCheckString4); - }else{ - script.append(filName); - } - return script.toString(); - } - - private String genSRBExitScript(Job j){ - StringBuffer script=new StringBuffer(); - - try{ - Enumeration enum=job.getVariableTable().elements(); - while(enum.hasMoreElements()){ - SingleVariable var=(SingleVariable)enum.nextElement(); - if((var.getType()!=null)&&(var.getType().equals("Gridfile")))//has data file - { - if (var.getValue().startsWith("srb")){ //if any of the vars is an srb file - //script.append("Sexit\n"); - break; - } - } - } - }catch (Exception e){ - logger.error("genSRBExitScript(Job j)"+e.getMessage()); - } - return script.toString(); - } -} +/* + * Title : JobWrapper.java + * Package : gridbus.broker.farming.execution + * Project : HEPscheduler + * Description : Handles the creation of a Globus Job and its submission. Registers a Job Monitor for the Job. + * Created : + * Modified on : Sept. 12 2002 - Added delegation of proxy.. - Srikumar + * Sept. 13 2002 - Added staging in of executable through GASS+RSL - Srikumar. + * Author : Jia Yu (jiayu@cs.mu.oz.au), Srikumar(srikumar@cs.mu.oz.au) + * Copyright : (c) 2003, Grid Computing and Distributed Systems Laboratory, + * Dept. of Computer Science and Software Engineering, + * University of Melbourne, Australia. + * + * This program is free software; you can redistribute it and/or modify it under + * the terms of the GNU General Public License as published by the Free Software + * Foundation; either version 2 of the License, or (at your option) any later version. + * See the GNU General Public License (http://www.gnu.org/copyleft/gpl.html)for more details. + * + */ +package org.gridbus.broker.farming.globus; + +import org.apache.log4j.Logger; + +import java.io.File; +import java.io.FileWriter; +import java.io.IOException; +import java.util.Enumeration; +import java.util.Hashtable; +import java.util.Vector; + +import org.globus.gram.Gram; +import org.globus.gram.GramException; +import org.globus.gram.GramJob; +import org.gridbus.broker.farming.common.BrokerProperties; +import org.gridbus.broker.farming.common.Job; +import org.gridbus.broker.farming.common.JobWrapper; +import org.gridbus.broker.runfile.CopyCommand; +import org.gridbus.broker.runfile.ExecuteCommand; +import org.gridbus.broker.runfile.MCopyCommand; +import org.gridbus.broker.runfile.GCopyCommand; +import org.gridbus.broker.runfile.SingleVariable; +import org.gridbus.broker.runfile.SubstituteCommand; +import org.gridbus.broker.runfile.TaskCommand; +import org.gridbus.broker.util.FileOpsUtil; +import org.gridbus.broker.util.GassServerUtil; +import org.gridbus.broker.util.RSLUtil; +import org.gridforum.jgss.ExtendedGSSManager; +import org.ietf.jgss.GSSCredential; +import org.ietf.jgss.GSSException; + +/** + * This class interacts with the Globus2.4.2 Gatekeeper at the remote host. It creates RSL for the job, + * submits RSL to the remote host and spawns off a GlobusJobMonitor thread to monitor the job. The actual job + * submission is done in a separate thread. + * + * @author Jia Yu (jiayu@cs.mu.oz.au) + * @author Srikumar Venugopal (srikumar@cs.mu.oz.au) + * @author Brett Beeson (bbeeson@ph.unimelb.edu.au) + * @see JobWrapper + */ +public class GlobusJobWrapper extends JobWrapper { + /** + * Logger for this class + */ + private static final Logger logger = Logger.getLogger(GlobusJobWrapper.class); + + private GramJob gramJob; + private GSSCredential proxy; + //private GlobusJobMonitor jobMonitor; + private Thread jobMonitor; + private Thread t; + private GassServerUtil gassServer; + private String jobUrl; + private Job job; + + /** + * Constructor + * @param proxy + * + */ + public GlobusJobWrapper(GSSCredential proxy){ + + this.proxy=proxy; + ExtendedGSSManager manager; + + try { + //cog 1.0 + //proxy=GlobusProxy.getDefaultUserProxy(true); + //cog 1.1 + //if no proxy is specified, then we + if(proxy==null){ + manager=(ExtendedGSSManager) ExtendedGSSManager.getInstance(); + proxy=manager.createCredential(GSSCredential.INITIATE_AND_ACCEPT); + } + } catch (GSSException e) { + logger.error("GlobusJobWrapper(GSSCredential) - Valid proxy missing: " + e.getMessage(), e); + /*Put in code for refreshing an expired proxy + * using the username and the password. + */ + } + } + + /** + * @see org.gridbus.broker.farming.common.JobWrapper#execute(org.gridbus.broker.farming.common.Job) + */ + public void execute(Job job){ + this.job=job; + GramJob gramJob=null; + String errorCodeString="GBX=`echo $?`\n"; + String errorCodeString2="echo \"GBX:$GBX\"\n"; + String errorCheckString="if [ $GBX -ne 0 ]\n"; + String errorCheckString2="then exit $GBX\n"; + String errorCheckString3="fi\n"; + String errorCheckString4="echo *done \n"; + String rDir=null; + String lDir=null; //srikumar - 10/02 - added variable to specify local directory + String shellFileName="";//srikumar - 11/02 - made shellfile reference here . + + this.setGassServer(new GassServerUtil()); + this.getGassServer().setProxy(proxy); + this.getGassServer().start(); + + GlobusComputeServer cs=(GlobusComputeServer)job.getServer(); + job.addVariable(new SingleVariable("$OS", "string", cs.getOS())); + + //srikumar - added LocalDir checking... + try{ + if((lDir=BrokerProperties.getProperty("LOCALDIR"))==null){ + lDir=""; + } + else { + lDir=lDir+"/"; //added a trailing / for directory separation + } + }catch(Exception e){lDir=""; } + + shellFileName=lDir+job.getJobID()+".sh"; + rDir ="DB"+System.currentTimeMillis(); + + File shellFile=new File(shellFileName); + FileWriter out=null; + + try { + out = new FileWriter(shellFile); + } catch (IOException e1) { + logger.error("execute(Job)", e1); + } + StringBuffer fileOutputBuf=new StringBuffer(); + // bbeeson : remove -f since 'sh -c "help set"' says + // "-f Disable file name generation (globbing)." + fileOutputBuf.append("#!/bin/sh\n"); + + fileOutputBuf.append("echo *create directory \n"); + fileOutputBuf.append("mkdir "+rDir+"\n"); + fileOutputBuf.append("echo *enter the directory \n"); + fileOutputBuf.append("cd "+rDir+"\n"); + fileOutputBuf.append("export TMPDIR=`pwd`\n"); + //check for SRB and put in its init code. + Vector commands=null; + commands=job.getTask().getCommands(); + TaskCommand tc; + + String source=null; + String destination=null; + Hashtable variables= job.getVariableValueTable(); + //TODO: put in check if variables are null. + + /* + * Added a few more extra variables to support GCopy. --chd-- + */ + String sourceFile=null; + String sourceNode=null; + String destinationFile=null; + String destinationNode=null; + + for (int i = 0; i < commands.size(); i++) { + // 10/11 -Sri -- Putting in the substitute thingy + if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.SUBSTITUTE_CMD)){ + source=((SubstituteCommand)tc).getSource(); + source=substituteVariables(source,variables); + //srikumar - 11/02-added localDir here for source and destination + //so that substitution happens on files within the localDir + ((SubstituteCommand)tc).setSource(lDir+source); + destination=((SubstituteCommand)tc).getDestination(); + destination=substituteVariables(destination,variables); + ((SubstituteCommand)tc).setDestination(lDir+destination); + ((SubstituteCommand)tc).doSubstitution(variables); + continue; + } + if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.MCOPY_CMD)) { + /* + MULTIPLE FILE COPY + - globus-url-copy only support filename to filename copy + - multiple file copy using /bin/sh commands + - use ls to glob the pattern + - use [ -f ] to remove non-files (eg directories) + - use for ... in to loop over each file and call globus-url-copy + - destination MUST be a existing directory + - source MUST be local + */ + source=((MCopyCommand)tc).getSource(); + destination=((MCopyCommand)tc).getDestination(); + source=substituteVariables(source,variables); + destination=substituteVariables(destination,variables); + + fileOutputBuf.append("echo '*copying "+source+" to "+destination+"'\n"); + fileOutputBuf.append("for file in `ls " + source + "` ; do\n"); + fileOutputBuf.append("if [ -f $file ] ; then\n"); + fileOutputBuf.append("basefile=`basename $file`\n"); + fileOutputBuf.append("$GLOBUS_LOCATION/bin/globus-url-copy "+" "); + + // Local (within for loop) + fileOutputBuf.append("file:$TMPDIR/$file "); + + // Remote (directory) + if(!((MCopyCommand)tc).isDestinationRemote()){ + fileOutputBuf.append(this.getGassServer().getURL()+"/"+lDir); //srikumar - added lDir + }else{ + fileOutputBuf.append("file:$TMPDIR/"); + } + // Write to same filename. + fileOutputBuf.append(destination+"/$basefile\n"); + fileOutputBuf.append(errorCodeString+errorCodeString2); + fileOutputBuf.append(errorCheckString+errorCheckString2+errorCheckString3); + + fileOutputBuf.append("fi\n"); // [ -f $file ] + fileOutputBuf.append("done\n"); // for file in + }else if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.GCOPY_CMD)) { + + sourceNode=((GCopyCommand)tc).getSourceNode(); + sourceFile=((GCopyCommand)tc).getSourceFile(); + destinationNode=((GCopyCommand)tc).getDestinationNode(); + destinationFile=((GCopyCommand)tc).getDestinationFile(); + source=substituteVariables(sourceNode,variables); + destination=substituteVariables(destinationNode,variables); + + fileOutputBuf.append("echo '*copying from "+sourceNode+" to "+destinationNode+"'\n"); + fileOutputBuf.append("$GLOBUS_LOCATION/bin/globus-url-copy "+" "); + + //TODO need to remove this hardcoding, to gsiftp , to support SRB GCopy + if(sourceNode.equals("localhost")){ + fileOutputBuf.append("file://"); + }else{ + fileOutputBuf.append("gsiftp://"+sourceNode+"/"); + } + fileOutputBuf.append(sourceFile+" "); + + if(destinationNode.equals("localhost")){ + fileOutputBuf.append("file://"); + }else{ + fileOutputBuf.append("gsiftp://"+destinationNode+"/"); + } + fileOutputBuf.append(destinationFile+"\n"); + + }else if ((tc = (TaskCommand) commands.elementAt(i)).getType().equals(TaskCommand.COPY_CMD)) { + source=((CopyCommand)tc).getSource(); + destination=((CopyCommand)tc).getDestination(); + source=substituteVariables(source,variables); + destination=substituteVariables(destination,variables); + + fileOutputBuf.append("echo *copying "+source+" to "+destination+"\n"); + fileOutputBuf.append("$GLOBUS_LOCATION/bin/globus-url-copy "+" "); + if(!((CopyCommand)tc).isSourceRemote()){ + fileOutputBuf.append(this.getGassServer().getURL()+"/"+lDir); //srikumar - added lDir + }else{ + fileOutputBuf.append("file:$TMPDIR/"); + } + fileOutputBuf.append(source+" "); + + if(!((CopyCommand)tc).isDestinationRemote()){ + fileOutputBuf.append(this.getGassServer().getURL()+"/"+lDir); //srikumar - added lDir + }else{ + fileOutputBuf.append("file:$TMPDIR/"); + } + fileOutputBuf.append(destination+"\n"); + + fileOutputBuf.append(errorCodeString+errorCodeString2); + fileOutputBuf.append(errorCheckString+errorCheckString2+errorCheckString3+errorCheckString4); + }else{ + String ex=((ExecuteCommand)tc).getExecutable(); + ex=substituteVariables(ex,variables); + fileOutputBuf.append("echo *executing "+ex+"\n"); + if(ex.startsWith("./")){ // 8/11 - Sri - if executable has been passed to the remote host then set its permissions + fileOutputBuf.append("echo *setting permissions for executable \n"); + fileOutputBuf.append("chmod u+x "+ex+"\n"); + } + fileOutputBuf.append(ex); + Vector args=((ExecuteCommand)tc).getArguments(); + for (int ai = 0; ai < args.size(); ai++) { + String a=(String)args.get(ai); + String value=job.getVariableValue(a); + if(value==null){ + a=substituteVariables(a,variables); + fileOutputBuf.append(" "+a); + }else{ + fileOutputBuf.append(" "+value); + } + } + fileOutputBuf.append("\n"); + fileOutputBuf.append(errorCodeString+errorCodeString2); + fileOutputBuf.append(errorCheckString+errorCheckString2+errorCheckString3+errorCheckString4); + } + } + fileOutputBuf.append("cd .. \n"); + // bbeeson: debug by leaving directory + fileOutputBuf.append("rm -rf "+rDir+"\n"); + fileOutputBuf.append("echo *deleted the directory "+"\n"); + + try { + out.write(fileOutputBuf.toString()); + out.close(); + } catch (IOException e) { + logger.error("execute(Job)", e); + } + RSLUtil jobRsl=new RSLUtil(); + String finalRsl=""; jobRsl.setExecutable(shellFileName); + jobRsl.setGassServer(gassServer); + finalRsl=jobRsl.rslOutputSubst(true); + // 9/11 - Sri - Made this change so that the job output is stored on the remote side instead of local.. + finalRsl = finalRsl + "(stdout=x-gass-cache://$(GLOBUS_GRAM_JOB_CONTACT)stdout anExtraTag)" + + "(stderr=x-gass-cache://$(GLOBUS_GRAM_JOB_CONTACT)stderr anExtraTag)"; + + gramJob=new GramJob(finalRsl); + gramJob.setCredentials(proxy); + //gramJob.addListener(jobMonitor); + this.gramJob=gramJob; + + jobRun(cs.getContactString(),job); + // if jobRun returns successfully, make the job status submitted + job.setJobStatus(Job.SUBMITTED); + job.setJobSubmittedTimestamp(); + jobMonitor= new Thread(new GlobusJobMonitor(job), job.getJobID()); + jobMonitor.start(); + } + + private String substituteVariables(String st, Hashtable variables){ + Enumeration e=variables.keys(); + while(e.hasMoreElements()){ + String key=(String)e.nextElement(); + if(st.indexOf(key)!=-1){ + st=FileOpsUtil.replaceString(st,key,(String)variables.get(key)); + } + } + return st; + } + + /** + * @see org.gridbus.broker.farming.common.JobWrapper#terminate() + */ + public void terminate(){ + try{ + this.getGassServer().shutdown(); + this.setGassServer(null); + File shellFile=new File(job.getJobID()+".sh"); + shellFile.delete(); + if(gramJob.getStatus()==GramJob.STATUS_ACTIVE){ + Gram.cancel(gramJob); + //logger.log("JobID:"+jobID+" Job cancel successful"); + //System.out.println("JobID:"+jobID+" Job cancel successful"); + } + if(jobMonitor!=null && jobMonitor.isAlive()){ + jobMonitor.interrupt(); + } + this.jobMonitor=null; + } + catch(Exception e){ + if(e instanceof java.net.ConnectException){ + logger.error("terminate() - cancelJob: failed to connect to jobmanager",e); + }else{ + logger.error("terminate()", e); + logger.error("terminate() - cancelJob: Unknown exception", e); + } + } + } + + /** + * Returns the GSSCredential proxy + * @return the user proxy + */ + public GSSCredential getProxy() { + return proxy; + } + + /** + * Sets the GSS credential + * @param credential - the user credential + */ + public void setProxy(GSSCredential credential) { + proxy = credential; + } + + /** + * Returns the gass server object + * @return gass server + */ + public GassServerUtil getGassServer() { + return gassServer; + } + + /** + * Sets the gass server + * @param util + */ + public void setGassServer(GassServerUtil util) { + gassServer = util; + } + + /** + * Returns the job URL + * @return job url + */ + public String getJobUrl() { + return jobUrl; + } + + /** + * Sets the job url + * @param string + */ + public void setJobUrl(String string) { + jobUrl = string; + } + + private void jobRun(String resourceManagerContact,Job j){ + try{ + String subjectName; + logger.info("jobRun(String, Job) - Submitting job " + + job.getJobID() + " to server " + + resourceManagerContact); + //Gram.request(resourceManagerContact,gramJob); // - submits as an ineractive job + Gram.request(resourceManagerContact,gramJob,true); + j.setJobHandle(gramJob.getIDAsString()); + + logger.info("jobRun(String, Job)" + gramJob.getIDAsString()); + + } catch (GSSException gpe) { + logger.error("jobRun(String, Job) - Error:" + gpe.getMessage(), gpe); + }catch(org.globus.gram.GramException ge){ + logger.error("jobRun(String, Job)" + GramException.getMessage(ge.getErrorCode())); + return; + } + } + +} Index: GridbusBroker/src/org/gridbus/broker/initializer/Initialization.java diff -u GridbusBroker/src/org/gridbus/broker/initializer/Initialization.java:1.6.2.2 GridbusBroker/src/org/gridbus/broker/initializer/Initialization.java:1.16 --- GridbusBroker/src/org/gridbus/broker/initializer/Initialization.java:1.6.2.2 Tue Nov 16 11:42:41 2004 +++ GridbusBroker/src/org/gridbus/broker/initializer/Initialization.java Sun Nov 7 17:34:22 2004 @@ -31,7 +31,6 @@ import org.gridbus.broker.GridBrokerException; import org.gridbus.broker.farming.common.BrokerProperties; import org.gridbus.broker.farming.common.CommonObjects; -import org.gridbus.broker.farming.common.DataFile; import org.gridbus.broker.farming.common.DataHost; import org.gridbus.broker.gatekeeper.Gatekeeper; import org.gridbus.broker.plan.InputReader; @@ -160,7 +159,7 @@ throw new GridBrokerException("Plan file format not known"); } - logger.info("setUp() - " + jobs.size() + " Jobs created ..."); + logger.info("setUp() - Jobs created ..."); dataFileList=(Vector) CommonObjects.getObject("DataFileList"); datahosts=(Hashtable) CommonObjects.getObject("DataHostTable"); @@ -174,7 +173,7 @@ if (this.InputDataFilePresent) { DataHost dh; - logger.info("setUp() - Got " + (this.dataFileList.size()) + " input data files.."); + logger.info("setUp() - Got " + (this.dataFileList.size() - 1) + " input data files.."); Collection c= (Collection) datahosts.values(); Vector dataHostList=new Vector(c); String filename="", coststr=""; @@ -215,7 +214,7 @@ Vector cmds = task.getCommands(); TaskCommand cmd; CopyCommand cpcmd; - String cwd = BrokerProperties.getProperty("LOCALDIR",System.getProperty("user.dir")); + for (int i = 0; i < cmds.size(); i++) { cmd = (TaskCommand) cmds.elementAt(i); if (cmd.getType().equals(TaskCommand.COPY_CMD)) { @@ -224,7 +223,7 @@ while(it.hasNext()){ GenericServer gs=(GenericServer)it.next(); UrlCopyUtil.copy("file:///" - +cwd + +System.getProperty("user.dir") +"/"+cpcmd.getSource(), "gsiftp://" +gs.getHostname()+"/~/" Index: GridbusBroker/src/org/gridbus/broker/initializer/StartRun.java diff -u GridbusBroker/src/org/gridbus/broker/initializer/StartRun.java:1.2.2.3 GridbusBroker/src/org/gridbus/broker/initializer/StartRun.java:1.8 --- GridbusBroker/src/org/gridbus/broker/initializer/StartRun.java:1.2.2.3 Tue Nov 23 20:04:42 2004 +++ GridbusBroker/src/org/gridbus/broker/initializer/StartRun.java Sun Nov 7 12:17:56 2004 @@ -95,11 +95,7 @@ }else if (args[i].trim().startsWith("--logp=")){ //set the log4j properties file logPropsFile=args[i].substring(7); - }else if (i == 0){ - planFile=args[i]; - }else if (i == 1){ - gatekeeperFile=args[i]; - } + } } try { Index: GridbusBroker/src/org/gridbus/broker/plan/FileResolver.java diff -u GridbusBroker/src/org/gridbus/broker/plan/FileResolver.java:1.1.2.3 GridbusBroker/src/org/gridbus/broker/plan/FileResolver.java:1.5 --- GridbusBroker/src/org/gridbus/broker/plan/FileResolver.java:1.1.2.3 Tue Nov 23 20:04:41 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/FileResolver.java Mon Nov 1 13:02:54 2004 @@ -31,7 +31,6 @@ Hashtable dataHostTable; Collection dataFileList; - protected int limit = 0; /** * Extracts the physical locations of the files from the logical file names @@ -70,28 +69,5 @@ */ public void setDataFileList(Collection dataFileList){ this.dataFileList=dataFileList; - } - - /** - * Limits the number of query results - * @param fileLimit - */ //this is an integer param since it is called by reflection - public void setLimit(Integer fileLimit) { - limit = fileLimit.intValue(); - } - - /** - * Limits the number of query results - * @param fileLimit - */ - public void setLimit(int fileLimit){ - setLimit(new Integer(fileLimit)); - } - /** - * Returns the upper limit of the # of files returned by the SRB query - * @return - */ - public int getLimit() { - return limit; } } Index: GridbusBroker/src/org/gridbus/broker/plan/GridfileProcessor.java diff -u GridbusBroker/src/org/gridbus/broker/plan/GridfileProcessor.java:1.1.2.3 GridbusBroker/src/org/gridbus/broker/plan/GridfileProcessor.java:removed --- GridbusBroker/src/org/gridbus/broker/plan/GridfileProcessor.java:1.1.2.3 Wed Nov 24 20:09:05 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/GridfileProcessor.java Thu Nov 25 18:48:51 2004 @@ -1,188 +0,0 @@ -/* - * Title : GridfileProcessor.java - * Package : org.gridbus.broker.plan - * Project : GridbusBroker - * Description : This class extends Exception and is used in the Gridbus Farming Engine - * Created on : 22/11/2004 - * Author : Krishna Nadiminti (kna@unimelb.edu.au) - * Copyright : (c) 2004, Grid Computing and Distributed Systems Laboratory, - * Dept. of Computer Science and Software Engineering, - * University of Melbourne, Australia. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any later version. - * See the GNU General Public License (http://www.gnu.org/copyleft/gpl.html)for more details. - * - */ - -package org.gridbus.broker.plan; - -import java.lang.reflect.Constructor; -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Collection; -import java.util.Hashtable; -import java.util.Iterator; -import java.util.Vector; - -import org.apache.log4j.Logger; -import org.dom4j.Element; -import org.gridbus.broker.farming.common.BrokerProperties; -import org.gridbus.broker.farming.common.CommonObjects; -import org.gridbus.broker.farming.common.DataFile; -import org.gridbus.broker.runfile.ListVariable; - -/** - * @author krishna - * - */ -public class GridfileProcessor extends ParameterProcessor{ - /** - * Logger for this class - */ - private static final Logger logger = Logger.getLogger(GridfileProcessor.class); - - Hashtable dataHostTable; - Collection dataFileList; - - /** - * - */ - public GridfileProcessor() { - super(); - } - - /** - * Sets up the gridfile processor with the input element object - * @param fragmentElement - */ - public GridfileProcessor(Element fragmentElement){ - super(fragmentElement); - } - - /** - * Extracts and returns the parameters in the xml file as objects - * - * This function is meant for Dynamic Parameters. - * Users will have to write their own class, that extends this ParameterProcessor - * class. This function will use reflection to invoke that class based on the - * variable name. - * A standard method (i.e. fixed method name) must be defined in that class - * to do the processing of the Gridfile parameter. - * @return ListVariable - */ - public Object extractGridfile(){ - - logger.debug("extractGridfile() - varName= "+varName); - logger.debug("extractGridfile() - varType= "+varType); - logger.debug("extractGridfile() - domain= "+domainType); - - String url = null; - Class gridfileClass; - Object gridfileRetobj, dataFileListObj, dataHostTableObj; - Object gridfileConsObj; - ListVariable lv=null; - - //fileLimit is an Integer because setLimit needs to be called by reflection - Integer fileLimit= Integer.valueOf(BrokerProperties.getProperty("MAX_NUM_FILES","0")); - - /* - * Reference to dataHostTable and dataFileList so that whatever changes made to - * these two objects will be reflected upon the original objects in the - * GridbusFarmingEngine. These two objects will be passed over to the classes - * that process the urls. - */ - - try{ - //Krishna, put this in to allow for adding to the file,host lists instead of creating them each time. - dataFileList = (Vector)CommonObjects.getObject("DataFileList"); - dataHostTable = (Hashtable)CommonObjects.getObject("DataHostTable"); - if (dataHostTable==null) dataHostTable = new Hashtable(); - if (dataFileList==null) dataFileList = new Vector(); - - Iterator elmit = fragmentElement.elementIterator("file"); - while (elmit.hasNext()){ - Element gridfile=(Element)elmit.next(); - url = gridfile.attributeValue("url"); - String protocol=gridfile.attributeValue("protocol"); - if (protocol!=null){ - protocol = Character.toUpperCase(protocol.charAt(0))+protocol.substring(1); //to convert the first char into uppercase for reflection - }else{ - protocol="file";//default - } - //get the appropriate class based on the domain type . - gridfileClass = Class.forName("org.gridbus.broker.plan."+protocol+"FileResolver"); - //get the default constructor for that class. - logger.debug("Gridfile Processor Calling : " + gridfileClass); - Constructor cons = gridfileClass.getConstructor(new Class[]{Element.class,Hashtable.class,Collection.class}); - //create an object of that class. - gridfileConsObj = cons.newInstance(new Object[]{fragmentElement,dataHostTable,dataFileList}); - - //call the method to set a limit on the data files returned. - Method mthd=gridfileClass.getMethod("setLimit",new Class[]{Integer.class}); - gridfileRetobj=mthd.invoke(gridfileConsObj,new Object[]{fileLimit}); - //call the method to extract files - mthd=gridfileClass.getMethod("extractFiles",new Class[]{String.class, String.class}); - gridfileRetobj=mthd.invoke(gridfileConsObj,new Object[]{varName, url}); - - }//end while - - CommonObjects.setObject("DataFileList", dataFileList); - CommonObjects.setObject("DataHostTable", dataHostTable); - - int count=0; - String[] fileValues=new String[dataFileList.size()]; - for (Iterator it=dataFileList.iterator(); - it.hasNext(); fileValues[count++]=((DataFile)it.next()).getFilename()); - lv=new ListVariable(varName,fileValues); - lv.setType(varType); - - } catch (SecurityException se) { - logger.error("processGridfile(Element, String, String) - Security Exception Occurred: " + se, se); - } catch (NoSuchMethodException nsme) { - logger.error("processGridfile(Element, String, String) - No Such Method Exception Occurred: " + nsme, nsme); - } catch (IllegalArgumentException iae) { - logger.error("processGridfile(Element, String, String) - Illegal Argument Exception Occurred: "+ iae, iae); - } catch (IllegalAccessException iace) { - logger.error("processGridfile(Element, String, String) - Illegal Access Exception Occurred: "+ iace, iace); - } catch (InvocationTargetException ite) { - logger.error("processGridfile(Element, String, String) - Invocation Target Exception Occurred: "+ ite, ite); - } catch (ClassNotFoundException cnfe) { - logger.error("processGridfile(Element, String, String) - Class Not Found Exception Occurred: " + cnfe, cnfe); - } catch (InstantiationException ie) { - logger.error("processGridfile(Element, String, String) - Instantiation Exception Occurred: "+ ie, ie); - } catch (Exception e){ - logger.error("processGridfile(Element, String, String) - Exception Occurred: "+ e, e); - } - - return lv; - } - - /** - * @return Returns the dataFileList. - */ - public Collection getDataFileList() { - return dataFileList; - } - /** - * @param dataFileList The dataFileList to set. - */ - public void setDataFileList(Collection dataFileList) { - this.dataFileList = dataFileList; - } - - /** - * @return Returns the dataHostTable. - */ - public Hashtable getDataHostTable() { - return dataHostTable; - } - /** - * @param dataHostTable The dataHostTable to set. - */ - public void setDataHostTable(Hashtable dataHostTable) { - this.dataHostTable = dataHostTable; - } - -} Index: GridbusBroker/src/org/gridbus/broker/plan/InputReader.java diff -u GridbusBroker/src/org/gridbus/broker/plan/InputReader.java:1.2.2.1 GridbusBroker/src/org/gridbus/broker/plan/InputReader.java:1.2 --- GridbusBroker/src/org/gridbus/broker/plan/InputReader.java:1.2.2.1 Tue Nov 16 11:41:52 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/InputReader.java Mon Nov 1 13:02:54 2004 @@ -22,7 +22,6 @@ import java.util.Vector; import org.dom4j.DocumentException; -import org.gridbus.broker.farming.common.BrokerProperties; import org.jaxen.JaxenException; /** @@ -47,16 +46,25 @@ //this is to convert the plan file into an XML file Plan2XML con = new Plan2XML(); - String cwd = BrokerProperties.getProperty("LOCALDIR",System.getProperty("user.dir")); - con.setSourceFile(cwd + System.getProperty("file.separator")+planFile); - con.setDestinationFile(cwd +System.getProperty("file.separator")+ XMLfile); - + //commented because the files are not to be placed into separate folers + //con.setSourceFile(System.getProperty("user.dir") + "\\files/plan/"+newPlanFile); + //con.setDestinationFile(System.getProperty("user.dir") + "\\files/xml/"+ XMLfile); + + con.setSourceFile(System.getProperty("user.dir") + System.getProperty("file.separator")+planFile); + con.setDestinationFile(System.getProperty("user.dir") +System.getProperty("file.separator")+ XMLfile); + con.readPlan(); //this is to convert the XML file into a run file XMLReader convert = new XMLReader(); - - convert.setSourceFile(cwd + System.getProperty("file.separator")+XMLfile); + + //commented because the files are not to be placed into separate folers + //convert.setSourceFile(System.getProperty("user.dir") + "\\files/xml/"+XMLfile); + //convert.setDestinationFile(System.getProperty("user.dir") + "\\files/run/"+runFileName); + + convert.setSourceFile(System.getProperty("user.dir") + System.getProperty("file.separator")+XMLfile); + /*convert.setDestinationFile(System.getProperty("user.dir") + System.getProperty("file.separator")+"whatever); + */ return convert.parseXML(); } Index: GridbusBroker/src/org/gridbus/broker/plan/IntegerProcessor.java diff -u GridbusBroker/src/org/gridbus/broker/plan/IntegerProcessor.java:1.1.2.1 GridbusBroker/src/org/gridbus/broker/plan/IntegerProcessor.java:removed --- GridbusBroker/src/org/gridbus/broker/plan/IntegerProcessor.java:1.1.2.1 Tue Nov 23 12:32:03 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/IntegerProcessor.java Thu Nov 25 18:48:51 2004 @@ -1,370 +0,0 @@ -/* - * Title : IntegerProcessor.java - * Package : org.gridbus.broker.plan - * Project : GridbusBroker - * Description : This class extends Exception and is used in the Gridbus Farming Engine - * Created on : 22/11/2004 - * Author : Krishna Nadiminti (kna@unimelb.edu.au) - * Copyright : (c) 2004, Grid Computing and Distributed Systems Laboratory, - * Dept. of Computer Science and Software Engineering, - * University of Melbourne, Australia. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any later version. - * See the GNU General Public License (http://www.gnu.org/copyleft/gpl.html)for more details. - * - */ - -package org.gridbus.broker.plan; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; -import java.util.Iterator; -import java.util.List; -import java.util.Random; -import java.util.StringTokenizer; - -import org.apache.log4j.Logger; -import org.dom4j.DocumentHelper; -import org.dom4j.Element; -import org.dom4j.XPath; -import org.gridbus.broker.runfile.ListVariable; -import org.gridbus.broker.runfile.SingleVariable; - -/** - * @author krishna - */ -public class IntegerProcessor extends ParameterProcessor{ - - /** - * Logger for this class - */ - private static final Logger logger = Logger.getLogger(IntegerProcessor.class); - - /** - * - */ - public IntegerProcessor() { - super(); - } - - /** - * Sets up the integer processor with the input element object - * @param fragmentElement - */ - public IntegerProcessor(Element fragmentElement){ - super(fragmentElement); - } - - /** - * Extracts and returns the parameters in the xml file as objects - * @return parameters - */ - public Object extractInteger(){ - - try { - - logger.debug("extractInteger() - varName= "+varName); - logger.debug("extractInteger() - varType= "+varType); - logger.debug("extractInteger() - domain= "+domainType); - - Method mthd = this.getClass().getMethod("process"+domainType, new Class[]{Element.class, String.class, String.class}); - retobj=mthd.invoke(this,new Object[]{fragmentElement, varName, "integer"}); - - } catch (SecurityException se) { - logger.error("extractInteger() - Security Exception Occurred: " + se, se); - } catch (NoSuchMethodException nsme) { - logger.error("extractInteger() - No Such Method Exception Occurred: " + nsme, nsme); - } catch (IllegalArgumentException iae) { - logger.error("extractInteger() - Illegal Argument Exception Occurred: " + iae, iae); - } catch (IllegalAccessException iace) { - logger.error("extractInteger() - Illegal Access Exception Occurred: " + iace, iace); - } catch (InvocationTargetException ite) { - logger.error("extractInteger() - Invocation Target Exception Occurred: " + ite, ite); - } catch (Exception e){ - logger.error("extractInteger() - Exception occurred: " + e,e); - } - - return retobj; - } - - /** - * Processes a singlevariable tag and returns a SingleVariable object - * @param fragmentElement - * @param varName - * @param singleVarType - * @return single variable object - */ - public SingleVariable processSingle(Element fragmentElement, String varName, String singleVarType){ - String singleValue = null; - SingleVariable singleVar; - Element single = null; - - single = fragmentElement.element("single"); - singleValue = single.attributeValue("value"); - singleValue = singleValue.replaceAll("\n",""); - singleValue = singleValue.trim(); - - //wrap everything in a SingleVariable object and pass it back. - singleVar = new SingleVariable(varName, singleVarType, singleValue); - - logger.debug("singleValue= "+singleValue); - - return singleVar; - } - - /** - * Returns a string listing out the job parameters in case of a range within an interval - * @param from - * @param to - * @param points - * @return a string listing out the job parameters - */ - private String pointsInteger(String from, String to, String points){ - String stepList = new String(); - int fromValue = Integer.parseInt(from.trim()); - int toValue = Integer.parseInt(to.trim()); - int pointsValue = Integer.parseInt(points.trim()); - int currentValue = 0; - int interval; - - interval = toValue/pointsValue; - //stepList = "\""+Integer.toString(currentValue)+"\" "; - - while((currentValue + interval) < toValue){ - currentValue = currentValue + interval; - stepList = stepList + "\""+Integer.toString(currentValue)+"\" "; - } - - return stepList; - } - - /** - * Processes a listvariable tag and returns a ListVariable object - * @param fragmentElement - * @param varName - * @param rangeVarType - * @return variable list - */ - public ListVariable processRange(Element fragmentElement, String varName, String rangeVarType){ - ListVariable listVar; - String from = null; - String to = null; - String stepOrPoints = null; - String intervalType = null; - String numberList = null; - Element range=null; - - range=fragmentElement.element("range"); - from=range.attributeValue("from"); - to=range.attributeValue("to"); - intervalType=range.attributeValue("type"); - stepOrPoints=range.attributeValue("interval"); - if(intervalType.equals("step")){ - numberList = stepInteger(from, to, stepOrPoints); - }else if(intervalType.equals("points")){ - numberList = pointsInteger(from, to, stepOrPoints); - } - /* - XPath xpathSelector = DocumentHelper.createXPath("domain/range/*"); - - List results = xpathSelector.selectNodes(fragmentElement); - for(Iterator iter = results.iterator(); iter.hasNext();){ - Element element = (Element) iter.next(); - //System.out.println("range2**= "+element.getName()); //temp - - if(element.getName().equals("value")){ - from = element.attributeValue("from"); - to = element.attributeValue("to"); - //from = element.getText(); - //from = from.substring(1); //to remove the leading '\n' character. - }else if(element.getName().equals("interval")){ - intervalType = element.attributeValue("type"); - stepOrPoints = element.getText(); - - if(intervalType.equals("step")){ - numberList = stepInteger(from, to, stepOrPoints); - break; - }else if(intervalType.equals("points")){ - numberList = pointsInteger(from, to, stepOrPoints); - break; - } - } - } - */ - - listVar = new ListVariable(varName); - listVar.setValues(numberList); - listVar.setType("Integer"); - - System.out.println("GOT IN ==> Range: "+numberList); //temp - - return listVar; - } - - /** - * Returns a string listing out the job parameters, in case of a range with a - * step between each value - * @param from - * @param to - * @param step - * @return a string listing out the job parameters - */ - private String stepInteger(String from, String to, String step){ - String stepList = null; - int fromValue = Integer.parseInt(from.trim()); - int toValue = Integer.parseInt(to.trim()); - int stepValue = Integer.parseInt(step.trim()); - int currentValue = fromValue; - - stepList = "\""+Integer.toString(fromValue)+"\" "; - - while(currentValue < toValue){ - currentValue = currentValue + stepValue; - stepList = stepList + "\""+Integer.toString(currentValue)+"\" "; - } - - return stepList; - } - - /** - * Processes a 'any-of' tag and returns a ListVariable object - * @param fragmentElement - * @param varName - * @param listType - * @return variable list - */ - private ListVariable processSelect_anyof(Element fragmentElement, String varName, String listType){ - ListVariable listVar; - Element elementType; - String valueList = null; - String defaultList = null; - String tempToken = null; - String formatedList = null; - boolean defaultExist = false; //to indicate if default list has been provided - String numberList = null; - StringTokenizer tk; - Random random = new Random(); - - XPath xpathSelector = DocumentHelper.createXPath("domain/select_anyof/*"); - //elementType = (Element)xpathSelector.evaluate(fragmentElement); - //variableType = elementType.getName(); - //xpathSelector = DocumentHelper.createXPath("domain/select_anyof/"+variableType+"/*"); - - List results = xpathSelector.selectNodes(fragmentElement); - for(Iterator iter = results.iterator(); iter.hasNext();){ - Element element = (Element) iter.next(); - //System.out.println("anyof%%%= "+element.getName()); //temp - - if(element.getName().equals("value")){ - valueList = element.getText(); - }else if(element.getName().equals("default")){ - defaultList = element.getText(); - defaultExist = true; - } - } - - /* - * Generating the list of values. If the default values are provided, then - * take those values instead. -taken from the XMLtoRunConverter class. - */ - if(defaultExist){ - tk = new StringTokenizer(defaultList); - //text values are already with quotation marks. - //so, don't need to add them. - if(listType.equals("text")){ - formatedList = tk.nextToken()+" "; - }else{ - formatedList = "\""+tk.nextToken()+"\" "; - } - - while(tk.hasMoreTokens()){ - tempToken = tk.nextToken(); - if(listType.equals("text")){ - formatedList = formatedList + tempToken + " "; - }else{ - formatedList = formatedList + "\""+tempToken+"\" "; - } - } - }else{ - tk = new StringTokenizer(valueList); - //text values are already with quotation marks. - //so, don't need to add them. - if(listType.equals("text")){ - formatedList = tk.nextToken()+" "; - }else{ - formatedList = "\""+tk.nextToken()+"\" "; - } - - //have to randomly pick from the given list of values, unless - //the default value has already been given. - while(tk.hasMoreTokens()){ - tempToken = tk.nextToken(); - if(random.nextBoolean()){ //random true or false - if(listType.equals("text")){ - formatedList = formatedList + tempToken + " "; - }else{ - formatedList = formatedList + "\""+tempToken+"\" "; - } - } - } - } - - //System.out.println("valueList: "+valueList); //temp - //System.out.println("defaultList: "+defaultList); //temp - //System.out.println("formatedList: "+formatedList); //temp - //System.out.println("GOT IN ==> Anyof"); //temp - - listVar = new ListVariable(varName); - listVar.setValues(formatedList); - listVar.setType("Integer"); - - return listVar; - } - - /* - * Not implemented - * Processes a 'select-one-of' tag and returns a ListVariable object - * @param fragmentElement - * @param varName - * @param variableType - * @return - */ - //TODO not impl. processSelect_oneof impl later...then public it... - private String processSelect_oneof(Element fragmentElement, String varName, String variableType){ - //System.out.println("GOT IN ==> Oneof"); //temp - - return "temporarily put as String"; - } - - /* - * Not implemented - * Processes a 'select-one-of' tag and returns a ListVariable object - * @param fragmentElement - * @param varName - * @param variableType - * @return - - //TODO not impl. processRandom - private String processRandom(Element fragmentElement, String varName, String variableType){ - //System.out.println("GOT IN ==> Random"); //temp - - return "temporarily put as String"; - }*/ - - /* - * Not implemented - * Processes a 'select-one-of' tag and returns a ListVariable object - * @param fragmentElement - * @param varName - * @param variableType - * @return - * - //TODO not impl. processCompute, impl later and then public it - private String processCompute(Element fragmentElement, String varName, String variableType){ - //System.out.println("GOT IN ==> Compute"); //temp - - return "temporarily put as String"; - }*/ -} Index: GridbusBroker/src/org/gridbus/broker/plan/LfnFileResolver.java diff -u GridbusBroker/src/org/gridbus/broker/plan/LfnFileResolver.java:1.3.2.3 GridbusBroker/src/org/gridbus/broker/plan/LfnFileResolver.java:1.8 --- GridbusBroker/src/org/gridbus/broker/plan/LfnFileResolver.java:1.3.2.3 Wed Nov 24 20:09:05 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/LfnFileResolver.java Mon Nov 1 13:02:54 2004 @@ -69,6 +69,8 @@ this.dataHostTable = dataHostTable; this.dataFileList = dataFileList; + //this.datahosts = new Hashtable(); //temp + /* * This part is taken from the constructor of Initialization class. */ @@ -109,12 +111,7 @@ prelimDataFileList=new Vector(); try{ - - //first check for limit, since this may not be the first in the fileResolvers to be called - //limit to the specified # of files - if ((limit>0) && (dataFileList.size()>=limit)) - return; - + /* * The Gridfile param has already been provided. Together with the url and everything. */ @@ -176,10 +173,6 @@ } for(int i=0;i0) && (dataFileList.size()>=limit)) - break; - temp=(Vector)result.elementAt(i); Iterator ti=temp.iterator(); while(ti.hasNext()){ @@ -199,10 +192,10 @@ df.setSizeInBytes(filesize); System.out.print(" ..."); //this needs to remain as it, it is like a progress bar fetchDataServers(df); - dataFileList.add(df); + dataFileList.add(df); } System.out.println(); //this needs to remain as it, it is for the progress bar - logger.info("Retrieved "+dataFileList.size()+" files in total"); + logger.info("retrieved "+dataFileList.size()+" files"); } private void fetchDataServers(DataFile df){ int x=0; Index: GridbusBroker/src/org/gridbus/broker/plan/ParameterProcessor.java diff -u GridbusBroker/src/org/gridbus/broker/plan/ParameterProcessor.java:1.2.2.3 GridbusBroker/src/org/gridbus/broker/plan/ParameterProcessor.java:1.8 --- GridbusBroker/src/org/gridbus/broker/plan/ParameterProcessor.java:1.2.2.3 Tue Nov 23 12:32:03 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/ParameterProcessor.java Mon Nov 1 13:02:54 2004 @@ -17,13 +17,26 @@ */ package org.gridbus.broker.plan; - import org.apache.log4j.Logger; + import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; +import java.util.Collection; +import java.util.Hashtable; import java.util.Iterator; +import java.util.List; +import java.util.Random; +import java.util.StringTokenizer; +import java.util.Vector; + +import org.dom4j.DocumentHelper; import org.dom4j.Element; +import org.dom4j.XPath; +import org.gridbus.broker.farming.common.CommonObjects; +import org.gridbus.broker.farming.common.DataFile; +import org.gridbus.broker.runfile.ListVariable; +import org.gridbus.broker.runfile.SingleVariable; /** * This class will extract all the information regarding Parameters from the @@ -43,16 +56,17 @@ Iterator elementIterator; Object retobj; Object consObj; - String elementName = null; - String varName = null; - String varType = null; - String domainType = null; - + String elementName = null; + ParameterProcessor pp; //object of this class which will be used to invoke the + //methods in this class later on through reflection. + Hashtable dataHostTable; + Collection dataFileList; + /** * Default constructor for this class */ public ParameterProcessor(){ - //default constructor. + //default constructor. } /** @@ -61,14 +75,7 @@ */ public ParameterProcessor(Element fragmentElement){ this.fragmentElement = fragmentElement; - if (fragmentElement!=null){ - varName = fragmentElement.attributeValue("name"); - varType = fragmentElement.attributeValue("type"); - domainType= fragmentElement.attributeValue("domain"); - - varType = Character.toUpperCase(varType.charAt(0))+varType.substring(1); //to convert the first char into uppercase for reflection - domainType = Character.toUpperCase(domainType.charAt(0))+domainType.substring(1); //to convert the first char into uppercase for reflection - } + this.pp = new ParameterProcessor(); } /** @@ -76,25 +83,39 @@ * @return parameters */ public Object extractParameter(){ - - logger.debug("extractParameter() - varName= "+varName); - logger.debug("extractParameter() - varType= "+varType); - logger.debug("extractParameter() - domain= "+domainType); + String varName = null; + String varType = null; + String domainType = null; + String domainTypeUpper = null; + + XPath xpathSelector = DocumentHelper.createXPath("domain/*"); + /* + * The commented part is for the old XML Schema, where the name of the variable is stored + * as an element instead of an attribute. + */ + //varName = fragmentElement.valueOf("name"); + //varName = varName.substring(1); //to remove the leading '\n' character. + //I suspect its a bug in the XPath valueOf() method. + + varName = fragmentElement.attributeValue("name"); + varType = fragmentElement.attributeValue("type"); + //System.out.println("varName= "+varName); //temp + //System.out.println("varType= "+varType); //temp + Element domain = (Element)xpathSelector.selectSingleNode(fragmentElement); + domainType = domain.getName(); + domainTypeUpper = Character.toUpperCase(domainType.charAt(0))+domainType.substring(1); + + clas=pp.getClass(); try { /* * Use reflection to call the required methods. * Removes the need to use if-else statements. More dynamic!, but slower */ - clas = Class.forName("org.gridbus.broker.plan."+varType+"Processor"); - - //get the default constructor for that class. - Constructor cons = clas.getConstructor(new Class[] {Element.class}); - //create an object of that class. - consObj = cons.newInstance(new Object[] {fragmentElement}); - - Method mthd = clas.getMethod("extract"+varType, null); - retobj=mthd.invoke(consObj,null); + //Method mthd = clas.getMethod("process" + domainTypeUpper, null); + Method mthd = clas.getMethod("process" + domainTypeUpper, new Class[]{Element.class, String.class, String.class}); + //retobj=mthd.invoke(pp,null); + retobj=mthd.invoke(pp,new Object[]{fragmentElement, varName, varType}); } catch (SecurityException se) { logger.error("extractParameter() - Security Exception Occurred: " + se, se); @@ -106,13 +127,415 @@ logger.error("extractParameter() - Illegal Access Exception Occurred: " + iace, iace); } catch (InvocationTargetException ite) { logger.error("extractParameter() - Invocation Target Exception Occurred: " + ite, ite); - } catch (ClassNotFoundException cnfe) { - logger.error("extractParameter() - Class Not Found Exception Occurred: " + cnfe, cnfe); - } catch (InstantiationException ie) { - logger.error("extractParameter() - Instantiation Exception Occurred: " + ie, ie); - } catch (Exception e){ - e.printStackTrace(); } return retobj; + } + + /* + * This function was taken from the XMLtoRunConverter class. + */ + /** + * Returns a string listing out the job parameters, in case of a range with a + * step between each value + * @param from + * @param to + * @param step + * @return a string listing out the job parameters + */ + public String stepInteger(String from, String to, String step){ + String stepList = null; + int fromValue = Integer.parseInt(from.trim()); + int toValue = Integer.parseInt(to.trim()); + int stepValue = Integer.parseInt(step.trim()); + int currentValue = fromValue; + + stepList = "\""+Integer.toString(fromValue)+"\" "; + + while(currentValue < toValue){ + currentValue = currentValue + stepValue; + stepList = stepList + "\""+Integer.toString(currentValue)+"\" "; + } + + return stepList; + } + + /* + * This function was taken from the XMLtoRunConverter class. + */ + /** + * Returns a string listing out the job parameters in case of a range within an interval + * @param from + * @param to + * @param points + * @return a string listing out the job parameters + */ + public String pointsInteger(String from, String to, String points){ + String stepList = new String(); + int fromValue = Integer.parseInt(from.trim()); + int toValue = Integer.parseInt(to.trim()); + int pointsValue = Integer.parseInt(points.trim()); + int currentValue = 0; + int interval; + + interval = toValue/pointsValue; + //stepList = "\""+Integer.toString(currentValue)+"\" "; + + while((currentValue + interval) < toValue){ + currentValue = currentValue + interval; + stepList = stepList + "\""+Integer.toString(currentValue)+"\" "; + } + + return stepList; + } + + /** + * Processes a singlevariable tag and returns a SingleVariable object + * @param fragmentElement + * @param varName + * @param singleVarType + * @return single variable object + */ + public SingleVariable processSingle(Element fragmentElement, String varName, String singleVarType){ + //String singleVarType = null; + String singleValue = null; + Element elementType; + SingleVariable singleVar; + + //XPath xpathSelector = DocumentHelper.createXPath("domain/single/*"); + //elementType = (Element)xpathSelector.evaluate(fragmentElement); + //singleVarType = elementType.getName(); + singleValue = fragmentElement.valueOf("domain/single/value"); + if (singleValue.startsWith("\n")) + singleValue=singleValue.substring(1); + if (singleValue.startsWith(" ")) + singleValue=singleValue.substring(1); + // singleValue.trim(); trim does not work + //wrap everything in a SingleVariable object and pass it back. + singleVar = new SingleVariable(varName, singleVarType, singleValue); + + //System.out.println("singleValue= "+singleValue); //temp + //System.out.println("GOT IN *** ==> Single"); //temp + + return singleVar; + } + + /** + * Processes a listvariable tag and returns a ListVariable object + * @param fragmentElement + * @param varName + * @param rangeVarType + * @return variable list + */ + public ListVariable processRange(Element fragmentElement, String varName, String rangeVarType){ + ListVariable listVar; + String from = null; + String to = null; + String stepOrPoints = null; + String intervalType = null; + String numberList = null; + + XPath xpathSelector = DocumentHelper.createXPath("domain/range/*"); + + List results = xpathSelector.selectNodes(fragmentElement); + for(Iterator iter = results.iterator(); iter.hasNext();){ + Element element = (Element) iter.next(); + //System.out.println("range2**= "+element.getName()); //temp + + if(element.getName().equals("value")){ + from = element.attributeValue("from"); + to = element.attributeValue("to"); + //from = element.getText(); + //from = from.substring(1); //to remove the leading '\n' character. + }else if(element.getName().equals("interval")){ + intervalType = element.attributeValue("type"); + stepOrPoints = element.getText(); + + if(intervalType.equals("step")){ + numberList = stepInteger(from, to, stepOrPoints); + break; + }else if(intervalType.equals("points")){ + numberList = pointsInteger(from, to, stepOrPoints); + break; + } + } + } + + listVar = new ListVariable(varName); + listVar.setValues(numberList); + listVar.setType("Integer"); + + //System.out.println("GOT IN ==> Range: "+numberList); //temp + + return listVar; + } + + /** + * Processes a 'any-of' tag and returns a ListVariable object + * @param fragmentElement + * @param varName + * @param listType + * @return variable list + */ + public ListVariable processSelect_anyof(Element fragmentElement, String varName, String listType){ + ListVariable listVar; + Element elementType; + String valueList = null; + String defaultList = null; + String tempToken = null; + String formatedList = null; + boolean defaultExist = false; //to indicate if default list has been provided + String numberList = null; + StringTokenizer tk; + Random random = new Random(); + + XPath xpathSelector = DocumentHelper.createXPath("domain/select_anyof/*"); + //elementType = (Element)xpathSelector.evaluate(fragmentElement); + //variableType = elementType.getName(); + //xpathSelector = DocumentHelper.createXPath("domain/select_anyof/"+variableType+"/*"); + + List results = xpathSelector.selectNodes(fragmentElement); + for(Iterator iter = results.iterator(); iter.hasNext();){ + Element element = (Element) iter.next(); + //System.out.println("anyof%%%= "+element.getName()); //temp + + if(element.getName().equals("value")){ + valueList = element.getText(); + }else if(element.getName().equals("default")){ + defaultList = element.getText(); + defaultExist = true; + } + } + + /* + * Generating the list of values. If the default values are provided, then + * take those values instead. -taken from the XMLtoRunConverter class. + */ + if(defaultExist){ + tk = new StringTokenizer(defaultList); + //text values are already with quotation marks. + //so, don't need to add them. + if(listType.equals("text")){ + formatedList = tk.nextToken()+" "; + }else{ + formatedList = "\""+tk.nextToken()+"\" "; + } + + while(tk.hasMoreTokens()){ + tempToken = tk.nextToken(); + if(listType.equals("text")){ + formatedList = formatedList + tempToken + " "; + }else{ + formatedList = formatedList + "\""+tempToken+"\" "; + } + } + }else{ + tk = new StringTokenizer(valueList); + //text values are already with quotation marks. + //so, don't need to add them. + if(listType.equals("text")){ + formatedList = tk.nextToken()+" "; + }else{ + formatedList = "\""+tk.nextToken()+"\" "; + } + + //have to randomly pick from the given list of values, unless + //the default value has already been given. + while(tk.hasMoreTokens()){ + tempToken = tk.nextToken(); + if(random.nextBoolean()){ //random true or false + if(listType.equals("text")){ + formatedList = formatedList + tempToken + " "; + }else{ + formatedList = formatedList + "\""+tempToken+"\" "; + } + } + } + } + + //System.out.println("valueList: "+valueList); //temp + //System.out.println("defaultList: "+defaultList); //temp + //System.out.println("formatedList: "+formatedList); //temp + //System.out.println("GOT IN ==> Anyof"); //temp + + listVar = new ListVariable(varName); + listVar.setValues(formatedList); + listVar.setType("Integer"); + + return listVar; + } + + /* + * Not implemented + * Processes a 'select-one-of' tag and returns a ListVariable object + * @param fragmentElement + * @param varName + * @param variableType + * @return + */ + //TODO not impl. processSelect_oneof impl later...then public it... + private String processSelect_oneof(Element fragmentElement, String varName, String variableType){ + //System.out.println("GOT IN ==> Oneof"); //temp + + return "temporarily put as String"; + } + + /* + * Not implemented + * Processes a 'select-one-of' tag and returns a ListVariable object + * @param fragmentElement + * @param varName + * @param variableType + * @return + */ + //TODO not impl. processRandom + private String processRandom(Element fragmentElement, String varName, String variableType){ + //System.out.println("GOT IN ==> Random"); //temp + + return "temporarily put as String"; + } + + /* + * Not implemented + * Processes a 'select-one-of' tag and returns a ListVariable object + * @param fragmentElement + * @param varName + * @param variableType + * @return + */ + //TODO not impl. processCompute, impl later and then public it + private String processCompute(Element fragmentElement, String varName, String variableType){ + //System.out.println("GOT IN ==> Compute"); //temp + + return "temporarily put as String"; + } + + /** + * This function is meant for Dynamic Parameters. + * Users will have to write their own class, that extends this ParameterProcessor + * class. This function will use reflection to invoke that class based on the + * variable name. + * A standard method (i.e. fixed method name) must be defined in that class + * to do the processing of the Gridfile parameter. + * @param fragmentElement + * @param varName + * @param variableType + * @return ListVariable + */ + public ListVariable processGridfile_list(Element fragmentElement, String varName, String variableType){ + String elementName = null; + String url = null; + Class gridfileClass; + Object gridfileRetobj, dataFileListObj, dataHostTableObj; + Object gridfileConsObj; + + //System.out.println("GOT IN ==> Gridfile"); //temp + + /* + * Reference to dataHostTable and dataFileList so that whatever changes made to + * these two objects will be reflected upon the original objects in the + * GridbusFarmingEngine. These two objects will be passed over to the classes + * that process the urls. + */ + //temporarily commented because GridbusFarmingEngine is not initialized yet. + //dataHostTable = gbfe.getDataHostTable(); + //dataFileList = gbfe.getDataFileList(); + + //Krishna, put this in to allow for adding to the file,host lists instead of creating them each time. + dataFileList = (Vector)CommonObjects.getObject("DataFileList"); + dataHostTable = (Hashtable)CommonObjects.getObject("DataHostTable"); + if (dataHostTable==null) dataHostTable = new Hashtable(); + if (dataFileList==null) dataFileList = new Vector(); + + XPath xpathSelector = DocumentHelper.createXPath("domain/gridfile_list/*"); + + List results = xpathSelector.selectNodes(fragmentElement); + for(Iterator iter = results.iterator(); iter.hasNext();){ + Element element = (Element) iter.next(); + elementName = element.getName(); + elementName = Character.toUpperCase(elementName.charAt(0))+elementName.substring(1); + url = element.attributeValue("url"); + + //System.out.println("gridfile_list= "+element.getName()); //temp + + try{ + //get the appropriate class based on the element name. + gridfileClass = Class.forName("org.gridbus.broker.plan."+elementName+"FileResolver"); + //get the default constructor for that class. + + logger.debug("Parameter Processor Calling : " + gridfileClass); + + Constructor cons = gridfileClass.getConstructor(new Class[]{Element.class,Hashtable.class,Collection.class}); //use this only if u want to pass the whole document through the constructor. + //create an object of that class. + /* + * Actually it is no longer necessary to pass the element fragment over. + * Can just extract the name and other info and pass it on. + */ + gridfileConsObj = cons.newInstance(new Object[]{fragmentElement,dataHostTable,dataFileList}); + + //call the method. + Method mthd=gridfileClass.getMethod("extractFiles",new Class[]{String.class, String.class}); + gridfileRetobj=mthd.invoke(gridfileConsObj,new Object[]{varName, url}); + /* + //Get the DataFileList and the dataHostTable from the resovler + mthd=gridfileClass.getMethod("getDataFileList", null); + dataFileList=(Vector)mthd.invoke(gridfileConsObj, null); +// Get the DataFileList and the dataHostTable from the resovler + mthd=gridfileClass.getMethod("getDataHostTable", null); + dataHostTableObj=(Hashtable)mthd.invoke(gridfileConsObj, null); + */ + + CommonObjects.setObject("DataFileList", dataFileList); + CommonObjects.setObject("DataHostTable", dataHostTable); + + } catch (SecurityException se) { + logger.error("processGridfile_list(Element, String, String) - Security Exception Occurred: " + se, se); + } catch (NoSuchMethodException nsme) { + logger.error("processGridfile_list(Element, String, String) - No Such Method Exception Occurred: " + nsme, nsme); + } catch (IllegalArgumentException iae) { + logger.error("processGridfile_list(Element, String, String) - Illegal Argument Exception Occurred: "+ iae, iae); + } catch (IllegalAccessException iace) { + logger.error("processGridfile_list(Element, String, String) - Illegal Access Exception Occurred: "+ iace, iace); + } catch (InvocationTargetException ite) { + logger.error("processGridfile_list(Element, String, String) - Invocation Target Exception Occurred: "+ ite, ite); + } catch (ClassNotFoundException cnfe) { + logger.error("processGridfile_list(Element, String, String) - Class Not Found Exception Occurred: " + cnfe, cnfe); + } catch (InstantiationException ie) { + logger.error("processGridfile_list(Element, String, String) - Instantiation Exception Occurred: "+ ie, ie); + } + } + int count=0; + String[] fileValues=new String[dataFileList.size()]; + for (Iterator it=dataFileList.iterator(); + it.hasNext(); fileValues[count++]=((DataFile)it.next()).getFilename()); + ListVariable lv=new ListVariable(varName,fileValues); + + lv.setType("Gridfile"); + return lv; + } + + /** + * @return Returns the dataFileList. + */ + public Collection getDataFileList() { + return dataFileList; + } + /** + * @param dataFileList The dataFileList to set. + */ + public void setDataFileList(Collection dataFileList) { + this.dataFileList = dataFileList; + } + + /** + * @return Returns the dataHostTable. + */ + public Hashtable getDataHostTable() { + return dataHostTable; + } + /** + * @param dataHostTable The dataHostTable to set. + */ + public void setDataHostTable(Hashtable dataHostTable) { + this.dataHostTable = dataHostTable; } } Index: GridbusBroker/src/org/gridbus/broker/plan/Plan2XML.java diff -u GridbusBroker/src/org/gridbus/broker/plan/Plan2XML.java:1.2.2.2 GridbusBroker/src/org/gridbus/broker/plan/Plan2XML.java:1.2 --- GridbusBroker/src/org/gridbus/broker/plan/Plan2XML.java:1.2.2.2 Tue Nov 23 20:04:42 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/Plan2XML.java Mon Nov 1 13:02:54 2004 @@ -51,15 +51,23 @@ //Krishna changed these fields to private private XMLStringWriter xsw= XMLStringWriter.create(); private String[] xmlHeader_attrNames = {"xmlns:xsi", "xsi:noNamespaceSchemaLocation"}; - private String[] xmlHeader_attrValues = {"http://www.w3.org/2001/XMLSchema-instance", "xml/XMLInputSchema.xsd"}; - private String[] parameter_attrb = {"name", "type", "domain"}; + private String[] xmlHeader_attrValues = {"http://www.w3.org/2001/XMLSchema-instance", "xml/plan_xmlSchema.xsd"}; + private String[] parameter_attrb = {"name", "type"}; /** * Sets the source file location for this substitute command. * @param source */ public void setSourceFile(String source){ + if (logger.isDebugEnabled()) { + logger.debug("setSourceFile(String) - start"); + } + this.source = source; + + if (logger.isDebugEnabled()) { + logger.debug("setSourceFile(String) - end"); + } } /** @@ -67,7 +75,15 @@ * @param destination */ public void setDestinationFile(String destination){ + if (logger.isDebugEnabled()) { + logger.debug("setDestinationFile(String) - start"); + } + this.destination = destination; + + if (logger.isDebugEnabled()) { + logger.debug("setDestinationFile(String) - end"); + } } /** @@ -76,8 +92,15 @@ */ public void intializexmlwriter() { - xsw.setPrettyPrintMode(true); + if (logger.isDebugEnabled()) { + logger.debug("intializexmlwriter() - start"); + } + xsw.writeStartTag(XMLStringWriter.createStartTag("plan", xmlHeader_attrNames, xmlHeader_attrValues)); + + if (logger.isDebugEnabled()) { + logger.debug("intializexmlwriter() - end"); + } } /** @@ -102,11 +125,10 @@ * Write a parameter xml tag with the given attributes name,type * @param varName * @param varType - * @param domain */ - public void openParameterXmlTag(String varName, String varType, String domain) + public void openParameterXmlTag(String varName, String varType) { - xsw.writeStartTag(XMLStringWriter.createStartTag("parameter", parameter_attrb, new String[]{varName, varType, domain})); + xsw.writeStartTag(XMLStringWriter.createStartTag("parameter", parameter_attrb, new String[]{varName, varType})); } /** @@ -150,9 +172,14 @@ * @param value * @param type */ - public void writeSingleXmlTag(String value, String type){ - //create an empty "single" tag with attribute "value" - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("single","value",value)); + public void writeSingleXmlTag(String value, String type){ + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("single")); + xsw.writeStartTag(XMLStringWriter.createStartTag("value")); + xsw.writePCData(value); + xsw.writeEndTag(XMLStringWriter.createEndTag("value")); + xsw.writeEndTag(XMLStringWriter.createEndTag("single")); + xsw.writeStartTag(XMLStringWriter.createEndTag("domain")); } /** @@ -163,11 +190,24 @@ * @param value */ public void writeRangeXmlTag(String from, String to, String pointOrStep, String value){ - if ((from!=null) && (to!=null)){ - if (pointOrStep==null) pointOrStep="step"; //default - if (value==null) value="1"; //default - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("range", new String []{"from", "to", "type", "interval"}, new String []{from, to, pointOrStep, value})); + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("range")); + if(from != null){ + xsw.writeStartTag(XMLStringWriter.createEmptyTag("value", new String []{"from", "to"}, new String []{from, to})); } + if(pointOrStep != null){ + + if(pointOrStep.equals("step")){ + xsw.writeStartTag(XMLStringWriter.createStartTag("interval", new String[] {"type"}, new String[] {"step"})); + xsw.writePCData(value); + }else if(pointOrStep.equals("points")){ + xsw.writeStartTag(XMLStringWriter.createStartTag("interval", new String[] {"type"}, new String[] {"points"})); + xsw.writePCData(value); + } + xsw.writeEndTag(XMLStringWriter.createEndTag("interval")); + } + xsw.writeEndTag(XMLStringWriter.createEndTag("range")); + xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); } /** @@ -177,7 +217,7 @@ * @param varType */ public void writeAnyofXmlTag(String varList, String defaultList, String varType){ - //xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); xsw.writeStartTag(XMLStringWriter.createStartTag("select_anyof")); xsw.writeStartTag(XMLStringWriter.createStartTag("value")); @@ -189,7 +229,7 @@ xsw.writeEndTag(XMLStringWriter.createEndTag("default")); } xsw.writeEndTag(XMLStringWriter.createEndTag("select_anyof")); - //xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); + xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); } /** @@ -199,7 +239,7 @@ * @param varType */ public void writeOneofXmlTag(String varList, String defaultList, String varType){ - //xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); xsw.writeStartTag(XMLStringWriter.createStartTag("select_oneof")); xsw.writeStartTag(XMLStringWriter.createStartTag("value")); @@ -211,7 +251,7 @@ xsw.writeEndTag(XMLStringWriter.createEndTag("default")); } xsw.writeEndTag(XMLStringWriter.createEndTag("select_oneof")); - //xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); + xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); } /** @@ -221,10 +261,19 @@ * @param points */ public void writeRandomXmlTag(String from, String to, String points){ - if ((from!=null) && (to!=null)){ - if (points==null) points="points"; //default - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("random", new String []{"from", "to", "type"}, new String []{from, to, points})); + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("random")); + if(from != null){ + xsw.writeStartTag(XMLStringWriter.createEmptyTag("value", new String []{"from", "to"}, new String []{from, to})); + } + + if(points != null){ + xsw.writeStartTag(XMLStringWriter.createStartTag("interval", new String[] {"type"}, new String[] {"points"})); + xsw.writePCData(points); + xsw.writeEndTag(XMLStringWriter.createEndTag("interval")); } + xsw.writeEndTag(XMLStringWriter.createEndTag("random")); + xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); } /** @@ -232,8 +281,15 @@ * @param expression */ public void writeComputeXmlTag(String expression){ - //create an empty "compute" tag with attribute "expression" - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("compute","expression",expression)); + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("compute")); + + xsw.writeStartTag(XMLStringWriter.createStartTag("value")); + xsw.writePCData(expression); + xsw.writeEndTag(XMLStringWriter.createEndTag("value")); + + xsw.writeEndTag(XMLStringWriter.createEndTag("compute")); + xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); } /** @@ -282,14 +338,11 @@ * @param executeCommand */ public void writeExecuteXmlTag(String location, String executeCommand){ - StringTokenizer args = new StringTokenizer(executeCommand," "); - String command = args.nextToken(); - xsw.writeStartTag(XMLStringWriter.createStartTag("execute", new String[]{"location"}, new String[]{location})); - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("command","value",command)); - while (args.hasMoreTokens()){ - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("arg","value",args.nextToken())); - } + + xsw.writeStartTag(XMLStringWriter.createStartTag("command")); + xsw.writePCData(executeCommand); + xsw.writeEndTag(XMLStringWriter.createEndTag("command")); xsw.writeEndTag(XMLStringWriter.createEndTag("execute")); } @@ -339,33 +392,34 @@ /** * Writes the GridFile parameter tags - * @param gridFile + * @param gridFile_List */ - public void writeGridfileXmlTag(String gridFile){ - //Krishna changed this to include only one protocol type per gridfile param - //and also generate XML according to new schema: 23-Nov-04 - //parameter INFILE Gridfile srb:/L0sim*.bbs; - - String url=gridFile.trim(); - String protocol="file"; //default; - String mode="stream"; //default - - if(url.regionMatches(0,"lfn",0,3)){ - protocol = "lfn"; - }else if(url.regionMatches(0,"gsiftp",0,6)){ - protocol = "gsiftp"; - }else if(url.regionMatches(0,"srb",0,3)){ //Krishna added SRB tags 29-10-2004 - protocol = "srb"; - mode="block"; - }else if(url.regionMatches(0,"ftp",0,3)){ - protocol = "ftp"; - }else if(url.regionMatches(0,"http",0,4)){ - protocol = "http"; - }else if(url.regionMatches(0,"https",0,5)){ - protocol = "https"; + public void writeGridfileXmlTag(String gridFile_List){ + //Krishna changed this : 29-10-2004, to allow multiple gridfiles in the + //same gridfile_list i.e under the same parameter name + //the token seperator was changed from a space " " to comma "," + //this assumes there are no "," commas in the url itself!! + StringTokenizer strTok = new StringTokenizer(gridFile_List, ","); + String url = null; + + xsw.writeStartTag(XMLStringWriter.createStartTag("domain")); + xsw.writeStartTag(XMLStringWriter.createStartTag("gridfile_list")); + + while(strTok.hasMoreTokens()){ + url = strTok.nextToken().trim(); + //logger.debug("url="+url); + if(url.regionMatches(0,"lfn",0,3)){ + xsw.writeStartTag(XMLStringWriter.createEmptyTag("lfn", new String[]{"url"}, new String[]{url})); + }else if(url.regionMatches(0,"gsiftp",0,6)){ + xsw.writeStartTag(XMLStringWriter.createEmptyTag("gsiftp", new String[]{"url"}, new String[]{url})); + }else if(url.regionMatches(0,"srb",0,3)){ //Krishna added SRB tags 29-10-2004 + xsw.writeStartTag(XMLStringWriter.createEmptyTag("srb", new String[]{"url"}, new String[]{url})); + } + } - - xsw.writeEmptyElement(XMLStringWriter.createEmptyTag("file", new String[]{"protocol","mode","url"}, new String[]{protocol,mode,url})); + + xsw.writeEndTag(XMLStringWriter.createEndTag("gridfile_list")); + xsw.writeEndTag(XMLStringWriter.createEndTag("domain")); } /** @@ -377,7 +431,6 @@ String temp = null; String varLabel = null; String varType = null; - String varDomain = null; strTok.nextToken(); //do not take into account the 'parameter' keyword. String varName = strTok.nextToken(); @@ -388,134 +441,142 @@ }else{ varType = temp; } - //to make sure the varType always starts with a lower case letter. String varTypeLower = Character.toLowerCase(varType.charAt(0))+varType.substring(1); - - //hack! for gridfile, since we dont have a domain defined for it in the plan file - if (varType.equalsIgnoreCase("gridfile")){ - varDomain = "file"; + if (logger.isDebugEnabled()) { + logger.debug("parseParameter(String) - varTypeLower= " + + varTypeLower); + } //temp + + this.openParameterXmlTag(varName, varTypeLower); + + if(varType.equals("Gridfile")){ + String gridFile_List = strTok.nextToken(";"); + this.writeGridfileXmlTag(gridFile_List); + /* + * This should be in the function that does the writing of the xml tags. + */ + StringTokenizer strTok_gridfile = new StringTokenizer(gridFile_List, " "); + if (logger.isDebugEnabled()) { + logger.debug("parseParameter(String) - gridFile_List" + + gridFile_List); + } }else{ - varDomain = strTok.nextToken(); - } - - logger.debug("parseParameter(String) - varName= " + varName); - logger.debug("parseParameter(String) - varType= " + varType); - logger.debug("parseParameter(String) - varDomain= " + varDomain); - - if (varDomain.equalsIgnoreCase("default")){ - varDomain="single"; - } - - this.openParameterXmlTag(varName, varTypeLower, varDomain); + + String varDomain = strTok.nextToken(); - if(varType.equalsIgnoreCase("gridfile")){ - String gridfile = strTok.nextToken(";"); - this.writeGridfileXmlTag(gridfile); - }else{ //it is an integer, float or string - - if(varDomain.equals("single")){ - String value = strTok.nextToken(";"); - this.writeSingleXmlTag(value, varType); - }else if(varDomain.equals("range")){ - String from = null; - String to = null; - String pointOrStep = null; - String value = null; - - if(strTok.hasMoreTokens()){ - if((strTok.nextToken()).equals("from")){ //skip the 'from' keyword - from = strTok.nextToken(); //System.out.println("from="+ from); //temp - strTok.nextToken(); //skip the 'to' keyword - to = strTok.nextToken(); - } - } - if(strTok.hasMoreTokens()){ - pointOrStep = strTok.nextToken(); //skip the 'points' keyword - value = strTok.nextToken(";"); - } - this.writeRangeXmlTag(from, to, pointOrStep, value); //if point/step is specified - - }else if(varDomain.equals("select")){ - String varDomainNext = strTok.nextToken(); - - if(varDomainNext.equals("anyof")){ - String varList = strTok.nextToken(); - String varListTemp = varList; - String defaultList = null; - - while(strTok.hasMoreTokens()){ - varListTemp = strTok.nextToken(" ;"); - if(varListTemp.equals("default")){ - defaultList = strTok.nextToken(); - String defaultListTemp = defaultList; - - while(strTok.hasMoreTokens()){ - defaultListTemp = strTok.nextToken(" ;"); - defaultList = defaultList + " " + defaultListTemp; - }//inner-while-loop - break; - }//if-statement - varList = varList + " " + varListTemp; - }//outer-while-loop - //System.out.println("varList="+ varList); //temp - //System.out.println("defaultList="+ defaultList); //temp - this.writeAnyofXmlTag(varList, defaultList, varType); - - }else if(varDomainNext.equals("oneof")){ - String varList = strTok.nextToken(); - String varListTemp = varList; - String defaultList = null; - - while(strTok.hasMoreTokens()){ - varListTemp = strTok.nextToken(" ;"); - if(varListTemp.equals("default")){ - defaultList = strTok.nextToken(); - String defaultListTemp = defaultList; - - while(strTok.hasMoreTokens()){ - defaultListTemp = strTok.nextToken(" ;"); - defaultList = defaultList + " " + defaultListTemp; - }//inner-while-loop - break; - }//if-statement - varList = varList + " " + varListTemp; - }//outer-while-loop - //System.out.println("varList="+ varList); //temp - //System.out.println("defaultList="+ defaultList); //temp - this.writeOneofXmlTag(varList, defaultList, varType); + if (logger.isDebugEnabled()) { + logger.debug("parseParameter(String) - varName= " + varName); + } //temp + if (logger.isDebugEnabled()) { + logger.debug("parseParameter(String) - varType= " + varType); + } //temp + if (logger.isDebugEnabled()) { + logger.debug("parseParameter(String) - varDomain= " + varDomain); + } //temp + + if(varDomain.equals("default")){ + String value = strTok.nextToken(";"); + this.writeSingleXmlTag(value, varType); + }else if(varDomain.equals("range")){ + String from = null; + String to = null; + String pointOrStep = null; + String value = null; + + if(strTok.hasMoreTokens()){ + if((strTok.nextToken()).equals("from")){ //skip the 'from' keyword + from = strTok.nextToken(); //System.out.println("from="+ from); //temp + strTok.nextToken(); //skip the 'to' keyword + to = strTok.nextToken(); } - }else if(varDomain.equals("random")){ - String from = null; - String to = null; - String points = null; + } + if(strTok.hasMoreTokens()){ + pointOrStep = strTok.nextToken(); //skip the 'points' keyword + value = strTok.nextToken(";"); + } + this.writeRangeXmlTag(from, to, pointOrStep, value); //if point/step is specified + + }else if(varDomain.equals("select")){ + String varDomainNext = strTok.nextToken(); + + if(varDomainNext.equals("anyof")){ + String varList = strTok.nextToken(); + String varListTemp = varList; + String defaultList = null; - if(strTok.hasMoreTokens()){ - if((strTok.nextToken()).equals("from")){ //skip the 'from' keyword - from = strTok.nextToken(); - strTok.nextToken(); //skip the 'to' keyword - to = strTok.nextToken(); - } - } - if(strTok.hasMoreTokens()){ - strTok.nextToken(); //skip the 'points' keyword - points = strTok.nextToken(";"); - }else{ - //if no points specified, then remove the semicolon from the'to' - to = to.substring(0, to.length()-1); - } - this.writeRandomXmlTag(from, to, points); + while(strTok.hasMoreTokens()){ + varListTemp = strTok.nextToken(" ;"); + if(varListTemp.equals("default")){ + defaultList = strTok.nextToken(); + String defaultListTemp = defaultList; + + while(strTok.hasMoreTokens()){ + defaultListTemp = strTok.nextToken(" ;"); + defaultList = defaultList + " " + defaultListTemp; + }//inner-while-loop + break; + }//if-statement + varList = varList + " " + varListTemp; + }//outer-while-loop + //System.out.println("varList="+ varList); //temp + //System.out.println("defaultList="+ defaultList); //temp + this.writeAnyofXmlTag(varList, defaultList, varType); - }else if(varDomain.equals("compute")){ - String expression = strTok.nextToken(";"); ; - String expressionTemp = expression; + }else if(varDomainNext.equals("oneof")){ + String varList = strTok.nextToken(); + String varListTemp = varList; + String defaultList = null; while(strTok.hasMoreTokens()){ - expressionTemp = strTok.nextToken(";"); - expression = expression + " " + expressionTemp; - } - this.writeComputeXmlTag(expression); + varListTemp = strTok.nextToken(" ;"); + if(varListTemp.equals("default")){ + defaultList = strTok.nextToken(); + String defaultListTemp = defaultList; + + while(strTok.hasMoreTokens()){ + defaultListTemp = strTok.nextToken(" ;"); + defaultList = defaultList + " " + defaultListTemp; + }//inner-while-loop + break; + }//if-statement + varList = varList + " " + varListTemp; + }//outer-while-loop + //System.out.println("varList="+ varList); //temp + //System.out.println("defaultList="+ defaultList); //temp + this.writeOneofXmlTag(varList, defaultList, varType); + } + }else if(varDomain.equals("random")){ + String from = null; + String to = null; + String points = null; + + if(strTok.hasMoreTokens()){ + if((strTok.nextToken()).equals("from")){ //skip the 'from' keyword + from = strTok.nextToken(); + strTok.nextToken(); //skip the 'to' keyword + to = strTok.nextToken(); + } + } + if(strTok.hasMoreTokens()){ + strTok.nextToken(); //skip the 'points' keyword + points = strTok.nextToken(";"); + }else{ + //if no points specified, then remove the semicolon from the'to' + to = to.substring(0, to.length()-1); + } + this.writeRandomXmlTag(from, to, points); + + }else if(varDomain.equals("compute")){ + String expression = strTok.nextToken(";"); ; + String expressionTemp = expression; + + while(strTok.hasMoreTokens()){ + expressionTemp = strTok.nextToken(";"); + expression = expression + " " + expressionTemp; } + this.writeComputeXmlTag(expression); + } } this.closeParameterXmlTag(); } Index: GridbusBroker/src/org/gridbus/broker/plan/PlanInterpreter.java diff -u GridbusBroker/src/org/gridbus/broker/plan/PlanInterpreter.java:1.2.2.4 GridbusBroker/src/org/gridbus/broker/plan/PlanInterpreter.java:1.8 --- GridbusBroker/src/org/gridbus/broker/plan/PlanInterpreter.java:1.2.2.4 Tue Nov 23 20:04:41 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/PlanInterpreter.java Mon Nov 1 13:02:54 2004 @@ -183,8 +183,6 @@ //get the appropriate class based on the element name. clas = Class.forName("org.gridbus.broker.plan."+elementName+"Processor"); - - logger.debug("Class called: "+clas); //get the default constructor for that class. Constructor cons = clas.getConstructor(new Class[]{Element.class}); //use this only if u want to pass the whole document through the constructor. //Constructor cons = clas.getConstructor(null); @@ -385,8 +383,6 @@ for (int i = 0; i < commands.length; i++) { - logger.debug("createTask() commands:"+commands[i]); - //make taskcommand classes OR just remove abstract for now. strTok = new StringTokenizer(commands[i]); String cmdType = TaskCommand.findType(strTok.nextToken(" ")); Index: GridbusBroker/src/org/gridbus/broker/plan/SrbFileResolver.java diff -u GridbusBroker/src/org/gridbus/broker/plan/SrbFileResolver.java:1.1.2.4 GridbusBroker/src/org/gridbus/broker/plan/SrbFileResolver.java:1.1 --- GridbusBroker/src/org/gridbus/broker/plan/SrbFileResolver.java:1.1.2.4 Thu Nov 25 13:57:35 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/SrbFileResolver.java Mon Nov 1 13:02:54 2004 @@ -34,7 +34,6 @@ import edu.sdsc.grid.io.MetaDataRecordList; import edu.sdsc.grid.io.MetaDataSelect; import edu.sdsc.grid.io.MetaDataSet; -import edu.sdsc.grid.io.UserMetaData; import edu.sdsc.grid.io.srb.SRBAccount; import edu.sdsc.grid.io.srb.SRBFileSystem; import edu.sdsc.grid.io.srb.SRBMetaDataSet; @@ -53,6 +52,9 @@ private SRBAccount ac = null; private SRBFileSystem fs = null; + private long totalFiles=0; + private int limit=100; //limit the # of datafiles + Element fragmentElement; /** @@ -83,19 +85,24 @@ //if (this.dataFileList == null) this.dataFileList = new Vector(); try { + logger.debug("Connecting to SRB..."); + srbHost = BrokerProperties.getProperty("SRB_HOST"); try{ srbPort = Integer.parseInt(BrokerProperties.getProperty("SRB_PORT")); - srbUser = BrokerProperties.getProperty("SRB_USERNAME"); - srbPwd = BrokerProperties.getProperty("SRB_PWD"); - srbHomeDir = BrokerProperties.getProperty("SRB_HOMEDIR"); - srbMDASDomain = BrokerProperties.getProperty("SRB_MDAS_DOMAIN"); - srbDefResource = BrokerProperties.getProperty("SRB_DEF_RESOURCE"); + limit = Integer.parseInt(BrokerProperties.getProperty("SRB_QUERY_LIMIT")); }catch(Exception e){ + limit = 100; //reset to default. } - + + srbUser = BrokerProperties.getProperty("SRB_USERNAME"); + srbPwd = BrokerProperties.getProperty("SRB_PWD"); + srbHomeDir = BrokerProperties.getProperty("SRB_HOMEDIR"); + srbMDASDomain = BrokerProperties.getProperty("SRB_MDAS_DOMAIN"); + srbDefResource = BrokerProperties.getProperty("SRB_DEF_RESOURCE"); + if ((srbHost==null) || (srbUser==null) || (srbPwd==null) || (srbHomeDir==null) || (srbMDASDomain==null) || (srbDefResource==null) ){ //gets connection info from the user home dir SRB Env files : default ac = new SRBAccount(); @@ -123,11 +130,6 @@ public void extractFiles(String varName, String srchFilName) { - //first check for limit, since this may not be the first in the fileResolvers to be called - //limit to the specified # of files - if ((limit>0) && (dataFileList.size()>=limit)) - return; - logger.debug("extractFiles: " + varName + "," + srchFilName); MetaDataRecordList[] rl = null; @@ -143,27 +145,24 @@ } logger.debug("extractFiles: " + varName + "," + srchFilName); - + + //conditions + MetaDataCondition cond[] = { + MetaDataSet.newCondition(SRBMetaDataSet.FILE_NAME, + MetaDataCondition.LIKE, srchFilName) + }; + //fields returned in the results String[] selFields = { SRBMetaDataSet.FILE_NAME, SRBMetaDataSet.DIRECTORY_NAME, //not used currently - SRBMetaDataSet.SIZE, - UserMetaData.USER_NAME, - SRBMetaDataSet.ACCESS_CONSTRAINT + SRBMetaDataSet.SIZE }; + MetaDataSelect sel[] = MetaDataSet.newSelection( selFields ); + try { - //conditions - MetaDataCondition cond[] = { - MetaDataSet.newCondition(SRBMetaDataSet.FILE_NAME, - MetaDataCondition.LIKE, srchFilName), - MetaDataSet.newCondition(UserMetaData.USER_NAME, - MetaDataCondition.LIKE,ac.getUserName()) - }; - - MetaDataSelect sel[] = MetaDataSet.newSelection( selFields ); /* * public edu.sdsc.grid.io.MetaDataRecordList[] query(edu.sdsc.grid.io.MetaDataCondition[] conditions, edu.sdsc.grid.io.MetaDataSelect[] selects, @@ -172,15 +171,10 @@ boolean nonDistinct) throws java.io.IOException */ - if (limit>0){ - rl = fs.query(cond,sel,limit,false,false); //this returns the first 300 results by default. - }else{ - rl = fs.query(cond,sel); - } - - processFileQueryResults(rl); - logger.debug("Retrieved " + dataFileList.size() +" files in total..."); + rl = fs.query(cond,sel,limit,false,false); //this returns the first 300 results by default. + processFileQueryResults(rl); + logger.debug("extractFiles() - SRB - Done."); } @@ -192,32 +186,38 @@ private void processFileQueryResults(MetaDataRecordList[] rl) throws IOException { - //this is to double-check to limit the query results. - if ((limit>0) && (dataFileList.size() >= limit)) return; - - if (rl != null) + if (rl != null) { + totalFiles+=rl.length; + logger.debug(totalFiles +" records found so far..."); for (int i = 0; i < rl.length ; i++) { - String fn = rl[i].getValue(SRBMetaDataSet.FILE_NAME).toString(); - //logger.debug(i+". file="+fn+",DIRECTORY_NAME="+rl[i].getValue(SRBMetaDataSet.DIRECTORY_NAME)+", USER_NAME="+rl[i].getValue(UserMetaData.USER_NAME) + ", ACCESS_CONSTRAINT="+ rl[i].getValue(SRBMetaDataSet.ACCESS_CONSTRAINT)); - System.out.print("... "); + String fn = rl[i].getValue(SRBMetaDataSet.FILE_NAME).toString(); + + //logger.debug(i+". container = " +rl[i].getValue(SRBMetaDataSet.DIRECTORY_NAME)); + //logger.debug(i + "." + fn); + System.out.print("."); + // get data file list DataFile df = new DataFile(fn); df.setSizeInBytes(Long.parseLong(rl[i].getValue(SRBMetaDataSet.SIZE).toString())); fetchDataServers(df); dataFileList.add(df); } + + //this is to double-check to limit the query results. + if (totalFiles >= limit) return; if (rl.length > 0){ if (!rl[0].isQueryComplete()){ - logger.debug("Getting the next set of files..."); + logger.debug("Getting the next set of records..."); rl = rl[0].getMoreResults(); processFileQueryResults(rl); } } } + } private void fetchDataServers(DataFile df) @@ -286,21 +286,15 @@ if (temp!=null) host = temp[0]; } - path=path+"/";//java.io.File.separator; this is commented because it causes problems when - //we run the broker on windows and submit to a unix node - if(!dataHostTable.containsKey(host)) { dh = new SrbDataHost(host); dh.setDomain(ac.getDomainName()); - dh.setResourceName(host); - dh.setAuthScheme("ENCRYPT1"); - dh.setCollection(ac.getHomeDirectory()); dh.setUsername(ac.getUserName()); dh.setPassword(ac.getPassword()); dh.setSrbHost(ac.getHost()); dh.setSrbPort(ac.getPort()); - dh.addFilePath(df.getFilename(),path); + dh.addFilePath(df.getFilename(), path); dataHostTable.put(host,dh); } else @@ -419,6 +413,22 @@ */ public SRBAccount getSRBAccount(){ return ac; + } + + /** + * Limits the number of query results + * @param fileLimit + */ + public void setLimit(int fileLimit){ + limit = fileLimit; + } + + /** + * Returns the upper limit of the # of files returned by the SRB query + * @return + */ + public int getLimit(){ + return limit; } } Index: GridbusBroker/src/org/gridbus/broker/plan/StringProcessor.java diff -u GridbusBroker/src/org/gridbus/broker/plan/StringProcessor.java:1.1.2.2 GridbusBroker/src/org/gridbus/broker/plan/StringProcessor.java:removed --- GridbusBroker/src/org/gridbus/broker/plan/StringProcessor.java:1.1.2.2 Tue Nov 23 20:04:41 2004 +++ GridbusBroker/src/org/gridbus/broker/plan/StringProcessor.java Thu Nov 25 18:48:51 2004 @@ -1,108 +0,0 @@ -/* - * Title : StringProcessor.java - * Package : org.gridbus.broker.plan - * Project : GridbusBroker - * Description : This class extends Exception and is used in the Gridbus Farming Engine - * Created on : 22/11/2004 - * Author : Krishna Nadiminti (kna@unimelb.edu.au) - * Copyright : (c) 2004, Grid Computing and Distributed Systems Laboratory, - * Dept. of Computer Science and Software Engineering, - * University of Melbourne, Australia. - * - * This program is free software; you can redistribute it and/or modify it under - * the terms of the GNU General Public License as published by the Free Software - * Foundation; either version 2 of the License, or (at your option) any later version. - * See the GNU General Public License (http://www.gnu.org/copyleft/gpl.html)for more details. - * - */ - -package org.gridbus.broker.plan; - -import java.lang.reflect.InvocationTargetException; -import java.lang.reflect.Method; - -import org.apache.log4j.Logger; -import org.dom4j.Element; -import org.gridbus.broker.runfile.SingleVariable; - -/** - * @author krishna - */ -public class StringProcessor extends ParameterProcessor{ - /** - * Logger for this class - */ - private static final Logger logger = Logger.getLogger(StringProcessor.class); - - /** - * - */ - public StringProcessor() { - super(); - } - - /** - * Sets up the text processor with the input element object - * @param fragmentElement - */ - public StringProcessor(Element fragmentElement){ - super(fragmentElement); - } - - /** - * Extracts and returns the parameters in the xml file as objects - * @return parameters - */ - public Object extractString(){ - - try { - - logger.debug("extractString() - varName= "+varName); - logger.debug("extractString() - varType= "+varType); - logger.debug("extractString() - domain= "+domainType); - - Method mthd = this.getClass().getMethod("process"+domainType, new Class[]{Element.class, String.class, String.class}); - retobj=mthd.invoke(this,new Object[]{fragmentElement, varName, "string"}); - - } catch (SecurityException se) { - logger.error("extractString() - Security Exception Occurred: " + se, se); - } catch (NoSuchMethodException nsme) { - logger.error("extractString() - No Such Method Exception Occurred: " + nsme, nsme); - } catch (IllegalArgumentException iae) { - logger.error("extractString() - Illegal Argument Exception Occurred: " + iae, iae); - } catch (IllegalAccessException iace) { - logger.error("extractString() - Illegal Access Exception Oc