Steps to start ApplicationMaster by Client

7309 단어 application

1. Apply for Application
1) Connect to ResourceManager
		YarnConfiguration yarnConf = new YarnConfiguration(conf);
		InetSocketAddress rmAddress = NetUtils.createSocketAddr(yarnConf.get(
			YarnConfiguration.RM_ADDRESS,
			YarnConfiguration.DEFAULT_RM_ADDRESS));		
		LOG.info("Connecting to ResourceManager at "  rmAddress);
		ClientRMProtocol applicationsManager = ((ClientRMProtocol) rpc.getProxy(
			ClientRMProtocol.class, rmAddress, conf));
 
2) Apply for an Application to ResourceManager
		    GetNewApplicationRequest request = Records.newRecord(GetNewApplicationRequest.class);		
			GetNewApplicationResponse response = applicationsManager.getNewApplication(request);
			LOG.info("Got new application id="  response.getApplicationId());
 
2. Initialize the context of the ApplicationMaster (
ApplicationId
ApplicationName
Queue: The queue to which the Application will be submitted
Priority: The priority of the Application
User: the user running the Application
AMContainerSpec: Information about the Container running the ApplicationMaster
)
1) Set the Application property
		ApplicationId appId = response.getApplicationId();
		ApplicationSubmissionContext appContext = Records.newRecord(ApplicationSubmissionContext.class);
		appContext.setApplicationId(appId);
		appContext.setApplicationName(appName);
 
2) Set the context of the Container running the ApplicationMaster
		ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class);
		
		//  ApplicationMaster jar 
		FileSystem fs = FileSystem.get(conf);
		Path src = new Path(appMasterJar);
		String pathSuffix = appName  "/"  appId.getId()  "/AppMaster.jar";	    
		Path dst = new Path(fs.getHomeDirectory(), pathSuffix);
		fs.copyFromLocalFile(false, true, src, dst);
		FileStatus destStatus = fs.getFileStatus(dst);
		
		//  ApplicationMaster log4j 
		Path log4jSrc = new Path(log4jPropFile);
		Path log4jDst = new Path(fs.getHomeDirectory(), "log4j.props");
		fs.copyFromLocalFile(false, true, log4jSrc, log4jDst);
		FileStatus log4jFileStatus = fs.getFileStatus(log4jDst);
		
		//  LocalResources
		LocalResource amJarRsrc = Records.newRecord(LocalResource.class);
		amJarRsrc.setType(LocalResourceType.FILE);
		amJarRsrc.setVisibility(LocalResourceVisibility.APPLICATION);	   
		amJarRsrc.setResource(ConverterUtils.getYarnUrlFromPath(dst)); 
		amJarRsrc.setTimestamp(destStatus.getModificationTime());
		amJarRsrc.setSize(destStatus.getLen());
		Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
		localResources.put("AppMaster.jar",  amJarRsrc);
		
		LocalResource log4jRsrc = Records.newRecord(LocalResource.class);
		log4jRsrc.setType(LocalResourceType.FILE);
		log4jRsrc.setVisibility(LocalResourceVisibility.APPLICATION);	   
		log4jRsrc.setResource(ConverterUtils.getYarnUrlFromURI(log4jDst.toUri()));
		log4jRsrc.setTimestamp(log4jFileStatus.getModificationTime());
		log4jRsrc.setSize(log4jFileStatus.getLen());
		localResources.put("log4j.properties", log4jRsrc);
		
		amContainer.setLocalResources(localResources);
		
		//  ApplicationMaster 
		StringBuilder classPathEnv = new StringBuilder("${CLASSPATH}:./*");  
		for (String c : conf.get(YarnConfiguration.YARN_APPLICATION_CLASSPATH).split(",")) {  
			classPathEnv.append(':');  
			classPathEnv.append(c.trim());  
		}  
		classPathEnv.append(":./log4j.properties");
		Map<String, String> env = new HashMap<String, String>();
		env.put("CLASSPATH", classPathEnv.toString());
		amContainer.setEnvironment(env);
		
		//  ApplicationMaster 
		Vector<CharSequence> vargs = new Vector<CharSequence>(30);
		vargs.add("${JAVA_HOME}" + "/bin/java");
		vargs.add("-Xmx" + amMemory + "m");
		vargs.add(appMasterMainClass);
		vargs.add(some args);
		vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout");
		vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr");
		StringBuilder command = new StringBuilder();
		for (CharSequence str : vargs) {
		  command.append(str).append(" ");
		} 
		List<String> commands = new ArrayList<String>();
		commands.add(command.toString());		
		amContainer.setCommands(commands);
		
		//  ApplicationMaster 
		Resource capability = Records.newRecord(Resource.class);
		capability.setMemory(amMemory);
		amContainer.setResource(capability);
		
		appContext.setAMContainerSpec(amContainer);
 
3) Set priorities, queues, users, etc.
		Priority pri = Records.newRecord(Priority.class);
		pri.setPriority(amPriority);
		appContext.setPriority(pri);
		appContext.setQueue(amQueue);
		appContext.setUser(amUser);
 
3. Create a request to run the application and submit it
	SubmitApplicationRequest appRequest = Records.newRecord(SubmitApplicationRequest.class);
    appRequest.setApplicationSubmissionContext(appContext);
	applicationsManager.submitApplication(appRequest);
 
4. Rotate the status of the Application to the ResourceManager
1) Build a request to get the Application status and submit it
		GetApplicationReportRequest reportRequest = Records.newRecord(GetApplicationReportRequest.class);
		reportRequest.setApplicationId(appId);
		GetApplicationReportResponse reportResponse = applicationsManager.getApplicationReport(reportRequest);
		ApplicationReport report = reportResponse.getApplicationReport();
 
2) Get the relevant information of the Application
		LOG.info("Got application report from ASM for"
			+ ", appId=" + appId.getId()
			+ ", clientToken=" + report.getClientToken()
			+ ", appDiagnostics=" + report.getDiagnostics()
			+ ", appMasterHost=" + report.getHost()
			+ ", appQueue=" + report.getQueue()
			+ ", appMasterRpcPort=" + report.getRpcPort()
			+ ", appStartTime=" + report.getStartTime()
			+ ", yarnAppState=" + report.getYarnApplicationState().toString()
			+ ", distributedFinalState=" + report.getFinalApplicationStatus().toString()
			+ ", appTrackingUrl=" + report.getTrackingUrl()
			+ ", appUser=" + report.getUser());
		YarnApplicationState state = report.getYarnApplicationState();
		FinalApplicationStatus dsStatus = report.getFinalApplicationStatus();

좋은 웹페이지 즐겨찾기