异步tcp通讯——APM.Console德姆o

澳门美高梅手机网站 1澳门美高梅手机网站 2

</property>

 

        <name>yarn.nodemanager.aux-services</name>

异步tcp通信——APM.Core 解包

30275 Master

View Code

 

APM测试

<configuration>

异步tcp通讯——APM.Core 服务端概述

mac 启动sshd

测试demo:

     解压安装包即可

异步tcp通信——APM.ConsoleDemo

转发请标明本文来源:http://www.cnblogs.com/yswenli/
越来越多内容欢迎star我的github:https://github.com/yswenli/APM
假诺发现本文有啥难题和其他提出,也随时欢迎沟通~

 

    <value>1</value>

 

export
HADOOP_CONF_DIR=/Users/ysisl/app/hadoop/hadoop-2.6.4/etc/hadoop

  github地址:https://github.com/yswenli/APM

export
SCALA_HOME=/Users/ysisl/app/spark/scala-2.10.4

异步tcp通讯——APM.Server 音讯推送服务的贯彻

 

  1 using System;
  2 using System.Collections.Generic;
  3 using System.Linq;
  4 using System.Net;
  5 using System.Text;
  6 using System.Threading;
  7 using System.Threading.Tasks;
  8 
  9 namespace APM.ConsoleDemo
 10 {
 11     class Program
 12     {
 13 
 14         static void Main(string[] args)
 15         {
 16             Console.Title = "APM.Core test";
 17 
 18             Console.WriteLine("输入1测试APM tcp通讯");
 19 
 20             Console.WriteLine("输入2测试APM转发");
 21 
 22             var str = Console.ReadLine();
 23 
 24             if (!string.IsNullOrEmpty(str) && str != "1")
 25             {
 26                 APMServer();
 27                 APMClient();
 28                 APMClient();
 29             }
 30             else
 31             {
 32                 Task.Factory.StartNew(() =>
 33                 {
 34                     Thread.Sleep(2000);
 35                     ClientProcess();
 36                 });
 37                 ServerProcess(true);
 38             }
 39             Console.ReadLine();
 40         }
 41 
 42         private static APM.Core.Server server;
 43         static void ServerProcess(bool falg = false)
 44         {
 45             Console.WriteLine("server test");
 46             server = new APM.Core.Server(8889, 500);
 47             server.OnAccepted += Server_OnAccepted;
 48             server.OnMessage += Server_OnReceived;
 49             server.OnDisConnected += Server_OnDisConnected;
 50             server.OnOnError += Server_OnOnError;
 51             server.Start();
 52             Console.WriteLine("server is running...");
 53             if (falg)
 54             {
 55                 while (true)
 56                 {
 57                     Console.ReadLine();
 58                     Console.WriteLine(string.Format("serverinfo[ClientCount:{0},ReceiveCount:{1},SendCount:{2}]", server.ClientCount, server.ReceiveCount, server.SendCount));
 59                 }
 60             }
 61 
 62         }
 63 
 64 
 65         private static APM.Core.Client client;
 66         static void ClientProcess()
 67         {
 68 
 69             Console.WriteLine("Client test");
 70 
 71             var localIP = GetLocalIp() + ":8889";
 72 
 73             Console.WriteLine("Client send test");
 74             client = new APM.Core.Client(Guid.NewGuid().ToString("N"), 10, localIP);
 75             client.OnConnected += Client_OnConnected;
 76             client.OnMessage += Client_OnMessage;
 77             client.OnDisConnected += Client_OnDisConnected;
 78             client.OnError += Client_OnError;
 79             client.Connect();
 80 
 81             Console.Title = "APM Server & Client";
 82             Console.WriteLine("MutiClients test");
 83             for (int i = 0; i < 10000; i++)
 84             {
 85                 new APM.Core.Client(Guid.NewGuid().ToString("N"), 10, localIP).Connect();
 86             }
 87 
 88         }
 89 
 90         #region server events
 91         private static void Server_OnAccepted(APM.Core.UserToken remote)
 92         {
 93             Console.WriteLine("收到客户端连接:" + remote.Client.RemoteEndPoint);
 94 
 95         }
 96         private static void Server_OnReceived(APM.Core.UserToken remote, byte[] data)
 97         {
 98             if (server.ClientCount <= 10)
 99             {
100                 Console.WriteLine("收到客户端消息:" + remote.Client.RemoteEndPoint + "  " + Encoding.UTF8.GetString(data));
101             }
102             server.SendMsg(remote, "server:hello   " + DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff"));
103         }
104 
105         private static void Server_OnDisConnected(APM.Core.UserToken remote, Exception ex)
106         {
107             Console.WriteLine(string.Format("客户端{0}已断开连接,断开消息:{1}", remote.ID, ex.Message));
108         }
109 
110         private static void Server_OnOnError(APM.Core.UserToken remote, Exception ex)
111         {
112             Console.WriteLine(string.Format("操作客户端{0}异常,断开消息:{1}", remote.ID, ex.Message));
113         }
114 
115 
116         #endregion
117 
118         #region client events
119 
120         static string msg = "hellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohellohelloh";
121         private static void Client_OnConnected(APM.Core.Client c)
122         {
123             if (c != null)
124                 c.SendMsg(string.Format("client:{0}    {1}", msg, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff")));
125 
126         }
127 
128         private static void Client_OnMessage(byte[] data)
129         {
130             Console.WriteLine("收到服务器信息:" + Encoding.UTF8.GetString(data));
131             if (client != null)
132                 client.SendMsg(string.Format("client:{0}    {1}", msg, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff")));
133         }
134 
135         private static void Client_OnDisConnected(Exception ex)
136         {
137             Console.WriteLine(string.Format("客户端断开连接,断开消息:{0}", ex.Message));
138         }
139 
140         private static void Client_OnError(Exception ex)
141         {
142             Console.WriteLine(string.Format("客户端异常,异常消息:{0}", ex.Message));
143         }
144 
145 
146         #endregion
147 
148         #region MyRegion
149         static string GetLocalIp()
150         {
151             string hostname = Dns.GetHostName();//得到本机名    
152             //IPHostEntry localhost = Dns.GetHostByName(hostname);//方法已过期,只得到IPv4的地址     
153             IPHostEntry localhost = Dns.GetHostEntry(hostname);
154             IPAddress localaddr = localhost.AddressList.Where(b => b.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork).Last();
155             return localaddr.ToString();
156         }
157         #endregion
158 
159 
160         #region APM转发
161         public static void APMServer()
162         {
163             Console.WriteLine("APMServer test");
164             APM.Server.APMServer server = new Server.APMServer(8890, 1024);
165             server.OnAccepted += Server_OnAccepted;
166             server.OnDisConnected += Server_OnDisConnected;
167             server.OnError += Server_OnOnError;
168             server.OnMessage += Server_OnMessage;
169             server.Start();
170             Console.WriteLine("APMServer 已启动...");
171         }
172 
173         private static void Server_OnMessage(Core.Extention.Message msg)
174         {
175             Console.WriteLine("APMServer 收到并转发消息:ID {0},Sender {1},SessionID {2},SendTick {3}", msg.ID, msg.Sender, msg.SessionID, msg.SendTick);
176         }
177 
178         public static void APMClient()
179         {
180             Console.WriteLine("APMClient test");
181             var userID = "张三" + new Random((int)DateTime.Now.Ticks).Next(10000, 99999);
182             APM.Client.APMClient apmClient = new Client.APMClient(userID, GetLocalIp(), 8890);
183             apmClient.OnConnected += ApmClient_OnConnected;
184             apmClient.OnDisConnected += Client_OnDisConnected;
185             apmClient.OnError += Client_OnError;
186             apmClient.OnMessage += APMClient_OnMessage;
187             apmClient.Connect();
188             Task.Factory.StartNew(() =>
189             {
190                 while (true)
191                 {
192                     if (apmClient.Connected)
193                     {
194                         apmClient.SendChannelMsg("all", string.Format("client:{0}    {1}", msg, DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss.fff")));
195                     }
196                     Thread.Sleep(0);
197                 }
198             });
199             Console.WriteLine("APMClient:{0} 已连接到服务器", userID);
200         }
201 
202         private static void ApmClient_OnConnected(Client.APMClient c)
203         {
204             c.Subscribe("all");            
205         }
206 
207         private static void APMClient_OnMessage(Core.Extention.Message msg)
208         {
209             Console.WriteLine("APMClient 收到消息:ID {0},Sender {1},SessionID {2},SendTick {3}", msg.ID, msg.Sender, msg.SessionID, msg.SendTick);
210         }
211         #endregion
212     }
213 }

$HADOOP_HOME/bin/hdfs namenode -format 

  俗话说麻雀虽小,五脏俱全。apm就算不难,可是能够完毕单机高品质消息推送(能够采取redis、kafka等改造成大型分布式音讯推送服务器)。

export SPARK_WORKER_MEMORY=2g

    <name>dfs.replication</name>

    scala
> val file=sc.textFile(“hdfs:/test/test.csv”)

 

mkdir tmp

29970 SecondaryNameNode

vi spar-env.sh

cp spark-env.sh.template spark-env.sh

30407 Worker

  1. 进入sbin/ 执行 start-all.sh

  2. 推行jps 查看是还是不是正规运营

 

 

29638 NameNode

<configuration>

  ssh-keygen -t dsa -P ” -f ~/.ssh/id_dsa

① 、 下载资料

<property>

21472

</configuration>

</property>

vi ~/.bashrc

 

配置hadoop,都在$HADOOP_HOME/etc/hadoop 下

29793 DataNode

  1. 安装JDK

     a. localhost:8080  ,查看spark 集群运市场价格况。
此端口一般与任何端口冲突

 

export SPARK_MASTER_IP=localhost

输出- 0 com.openssh.sshd
表示运行成功

    <value>/Users/ysisl/app/hadoop/workspace/hdfs/data</value>

贰 、预先安装

    <name>dfs.name.dir</name>

mkdir -pv hdfs/data

sudo launchctl list | grep ssh

30586 Jps

  1. 格式化HDFS文件系统

 

  1. 安插scala 、spark 、 hadoop 环境变量参与PATH
    ,方便执行

<property>

 

<property>

</property>

        <value>mapreduce_shuffle</value>

        <value>localhost:8099</value>

 1. 备选贰个csv文件,路径  /Users/ysisl/app/hadoop/test.csv 

 

 3. 新建目录 , hadoop fs -mkdir /test

sudo launchctl unload -w
/System/Library/LaunchDaemons/ssh.plist

30256 Jps

           在spark-env.sh 中加入 export
SPARK_MASTER_WEBUI_PORT=98080 来指定

30070 ResourceManager

        <value>yarn</value>

</configuration>

30231 NodeManager

 

<configuration>

</property>

    <description>HDFS
URI</description>

    <description>副本个数,配置暗中认可是3,应低于datanode机器数量</description>

</property>

  1. 进入环境变量

 

暗中同意slaves今后就主机一台

    <description>namenode temp
dir</description>

 

 cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys

  1. copy mapred-site.xml.template to
    mapred-site.xml

HADOOP_HOME=/Users/ysisl/app/hadoop/hadoop-2.6.4

<property>

    scala
> val count=file.flatMap(line=>line.split(” “)).map(word=>(word,1)).reduceByKey(_+_)

sudo launchctl load -w
/System/Library/LaunchDaemons/ssh.plist

四、安装Spark

 

export
SPARK_HOME=/Users/ysisl/app/spark/spark-1.6.1-bin-hadoop2.6

30231 NodeManager

</property>

  1. 设置配置文件

   
<value>hdfs://localhost:9000</value>

vi ~/.bashrc

29793 DataNode

  1. core-site.xml

   
<value>/Users/ysisl/app/hadoop/workspace/tmp</value>

 6. 执行 spark-shell 

停止sshd服务

</configuration>

export
HADOOP_HOME=/Users/ysisl/app/hadoop/hadoop-2.6.4

 4. 上传文件到目录, hadoop fs -put
/Users/ysisl/app/hadoop/test.csv /test/

  1. yarn-site.xml

 

mkdir -pv hdfs/name

29638 NameNode

SCALA_HOME=/Users/ysisl/app/spark/scala-2.10.4

 

  1. JDK 1.6 + 

  2. Scala 2.10.4

  3. Hadoop 2.6.4 

  4. Spark 1.6 

jps 查看到多出二个Master,worker进度

添加hadoop目录环境变量

 

    <name>dfs.data.dir</name>

SPARK_HOME=/Users/ysisl/app/spark/spark-1.6.1-bin-hadoop2.6

 

 

    <value>/Users/ysisl/app/hadoop/workspace/hdfs/name</value>

    <name>fs.default.name</name>

  1. 打开 http://localhost:50070/explorer.html 网页查看hadoop目录结构,表达安装成功

2. hdfs-site.xml

2. 安装Scala 2.10.4

        <name>yarn.resourcemanager.webapp.address</name>

  1. 解压spark压缩包

export
PATH=”${HADOOP_HOME}/bin:${SCALA_HOME}/bin:${SPARK_HOME}/bin:$PATH”

cp slaves.template slaves

29970 SecondaryNameNode

     c. http://localhost:50070/ hadoop集群运市场价格况

30468 SparkSubmit

<property>

 7. 查看执市场价格况

 tar xvzf spark.1.6.tar.gz 

<configuration>

 5. hadoop fs -lsr / 查看已创立的目录文件

五. 测试运维 

     b. http://localhost:4040/jobs/ ,查看
spark task job运转境况

增加如下内容

21472

  1. 启动 sbin/start-all.sh
  1. 开创Hadoop文件系统目录

三、安装Hadoop 

 

30070 ResourceManager

export
JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.8.0_25.jdk/Contents/Home

查阅运行

 

 

<property>

转载:http://www.cnblogs.com/ysisl/p/5979268.html

        <name>mapreduce.framework.name</name>

export
SCALA_HOME=/Users/ysisl/app/spark/scala-2.10.4

<property>

    <description>namenode上囤积hdfs名字空间元数据 </description> 

cd hadoop/workspace

</configuration>

 2. 翻看DFS文件系统结构, 执行 hadoop fs -lsr /

</property>

vi ~/.bashrc

    <description>datanode上数据块的大体存储位置</description>

<property>

 

</property>

cd spar-1.6.1-bin-hadoop2.6/conf

export SPARK_WORKER_CORES=2

 

  1. 澳门美高梅手机网站,配置sshd 

    <name>hadoop.tmp.dir</name>

    scala > count.collect

mkdir -pv  hadoop/workspace

发表评论

电子邮件地址不会被公开。 必填项已用*标注