C# Hadoop
阿新 • • 發佈：2019-01-02
- using System;
- using System.Collections.Generic;
- using System.Linq;
- using System.Text;
- using System.Threading.Tasks;
- using Microsoft.Hadoop;
- using Microsoft.Hadoop.MapReduce;
- using Microsoft.Hadoop.WebClient.WebHCatClient;
- using System.Diagnostics;
- using System.IO;
- using System.IO.MemoryMappedFiles;
- namespace HadoopConsol
- {
- class Program
- {
- static void Main(string[] args)
- {
- Stopwatch sw = new Stopwatch();
- long hadoopTime=0;
- long normalTime=0;
- sw.Start();
- //start hadoop
- Console.WriteLine(" Hadoop Process Strating ....");
- #region Hadoop time
- #region hadoopconnet
- Console.WriteLine(" Hadoop Connect Strating ....");
- //establish job configuration
- HadoopJobConfiguration myConfig = new HadoopJobConfiguration();
- myConfig.InputPath = "/demo/simple/in";
- myConfig.OutputFolder = "/demo/simple/out";
- //connect to cluster
- Uri myUri = new Uri("http://localhost");
- string userName = "hadoop";
- string passWord = null;
- IHadoop myCluster = Hadoop.Connect(myUri, userName, passWord);
- hadoopTime += sw.ElapsedMilliseconds;
- Console.WriteLine(" Hadoop Connect End.");
- Console.WriteLine(" Hadoop Connect time:" + sw.ElapsedMilliseconds);
- #endregion
- #region hadoopmapreduce
- sw.Reset();
- sw.Start();
- Console.WriteLine(" Hadoop MapReduce Strating ....");
- //execute mapreduce job
- MapReduceResult jobResult =
- myCluster.MapReduceJob.Execute<MySimpleMapper, MySimpleReducer>(myConfig);
- hadoopTime += sw.ElapsedMilliseconds;
- Console.WriteLine(" Hadoop MapReduce End.");
- Console.WriteLine(" Hadoop MapReduce Time:"+sw.ElapsedMilliseconds);
- #endregion
- #region Hadoop End
- sw.Reset();
- sw.Start();
- Console.WriteLine(" Hadoop Endprocess Strating ....");
- //write job result to console
- int exitCode = jobResult.Info.ExitCode;
- string exitStatus = "Failure";
- if (exitCode == 0) exitStatus = "Success";
- exitStatus = exitCode + " (" + exitStatus + ")";
- Console.WriteLine();
- Console.Write("Exit Code = " + exitStatus);
- Console.WriteLine(" Hadoop Endprocess End.");
- hadoopTime += sw.ElapsedMilliseconds;
- Console.WriteLine(" Hadoop Exit Time:" + sw.ElapsedMilliseconds);
- Console.WriteLine(" Hadoop Process All Time:" + hadoopTime);
- #endregion
- #endregion
- #region Normal time
- //start Normal
- Console.WriteLine(" Normal Process Strating ....");
- sw.Reset();
- sw.Start();
- //normal process
- #region Normal Process
- int myevenCount = 0;
- int myeventSum = 0;
- int myoddCount = 0;
- int myoddSum = 0;
- StreamReader fs = new StreamReader(@"c:\TEMP\integers.txt");
- while (fs.Peek() >= 0)
- {
- string strTemp = fs.ReadLine();
- if (Int32.Parse(strTemp) % 2 == 0)
- {
- myevenCount++;
- myeventSum += Int32.Parse(strTemp);
- }
- else
- {
- myoddCount++;
- myoddSum += Int32.Parse(strTemp);
- }
- }
- //MemoryMappedFile m = MemoryMappedFile.
- Console.WriteLine("even:" + "\t" + myevenCount + "\t" + myeventSum);
- Console.WriteLine("odd:" + "\t" + myoddCount + "\t" + myoddSum);
- #endregion
- Console.WriteLine(" Normal Process End.");
- normalTime += sw.ElapsedMilliseconds;
- Console.WriteLine(" Normal Exit Time:" + sw.ElapsedMilliseconds);
- Console.WriteLine(" Normal Process All Time:" + normalTime);
- #endregion
- sw.Stop();
- Console.Read();
- }
- }
- public class MySimpleMapper : MapperBase
- {
- public override void Map(string inputLine, MapperContext context)
- {
- //interpret the incoming line as an integer value
- int value = int.Parse(inputLine);
- //determine whether value is even or odd
- string key = (value % 2 == 0) ? "even" : "odd";
- //output key assignment with value
- context.EmitKeyValue(key, value.ToString());
- }
- }
- public class MySimpleReducer : ReducerCombinerBase
- {
- public override void Reduce(
- string key, IEnumerable<string> values, ReducerCombinerContext context
- )
- {
- //initialize counters
- int myCount = 0;
- int mySum = 0;
- //count and sum incoming values
- foreach (string value in values)
- {
- mySum += int.Parse(value);
- myCount++;
- }
- //output results
- context.EmitKeyValue(key, myCount + "\t" + mySum);
- }
- }
- }