一、Mycat分片路由原理
我们先来看下面的一个SQL在Mycat里面是如何执行的:
travelrecord id (,);
有3个分片dn1,dn2,dn3,id=5000001
查询时可能有出现的问题:
dn1 dn2 dn3
2)只扫描某个片。漏掉数据的情况。
那么Mycat是如何解决上面的问题的呢?
Mycat解析的过程主要有Visitor和Statement两个阶段
1)Visitor过程,解析出如下属性:
哪一张表
字段列表
条件信息
什么样的SQL
通过查询条件可以知道要查询的数据都在哪些分片上
travelrecord id travelrecord id ;(dn3执行)
4)分别在分片dn2,dn3上执行第 3)步改写的SQL,然后把从dn2,dn3上得到的结果进行拼装就是最终的结果了
visitor
确定分片的过程:首先看where
二、Mycat常用分片规则
1. 时间类:按天分片、自然月分片、单月小时分片
2. 哈希类:Hash固定分片、日期范围Hash分片、截取数字Hash求模范围分片、截取数字Hash分片、一致性Hash分片
3. 取模类:取模分片、取模范围分片、范围求模分片
4. 其他类:枚举分片、范围约定分片、应用指定分片、冷热数据分片
下面基于源码来介绍Mycat的常用分片规则,
三、Mycat常用分片规则介绍
分片规则都定义在rule.xml文件里面
id
func1
1,1,2,3,1
128,128,128
1. 自动范围分片
在rule.xml里面的配置:
id
rang-long
<span style="color: #0000ff"><
<span style="color: #800000">function <span style="color: #ff0000">name<span style="color: #0000ff">="rang-long"<span style="color: #ff0000">
class<span style="color: #0000ff">="io.mycat.route.function.AutoPartitionByLong"<span style="color: #0000ff">>
<span style="color: #0000ff"><<span style="color: #800000">property <span style="color: #ff0000">name<span style="color: #0000ff">="mapFile"<span style="color: #0000ff">>autopartition-long.txt<span style="color: #0000ff"></<span style="color: #800000">property<span style="color: #0000ff">>
<span style="color: #0000ff"></<span style="color: #800000">function<span style="color: #0000ff">>
说明:
有3个分片,第1个分片存储的是1-500000的数据,第2个分片存储的是500001-1000000的数据,第3个分片存储的是1000001-1500000的数据
insert into employee(id,name) value(1,Tom);在第1个分片
insert into employee(id,name) value(500002,Jack);在第2个分片
insert into employee(id,name) value(1000002,Lucy);在第3个分片
对应代码:
<span style="color: #0000ff">import<span style="color: #000000"> java.io.BufferedReader;
<span style="color: #0000ff">import<span style="color: #000000"> java.io.InputStream;
<span style="color: #0000ff">import<span style="color: #000000"> java.io.InputStreamReader;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.HashSet;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.LinkedList;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.Set;
<span style="color: #0000ff">import<span style="color: #000000"> io.mycat.config.model.rule.RuleAlgorithm;
<span style="color: #008000">/**<span style="color: #008000">
-
auto partition by Long,can be used in auto increment primary key partition
-
-
<span style="color: #808080">@author<span style="color: #008000"> wuzhi
<span style="color: #008000">*/
<span style="color: #0000ff">public <span style="color: #0000ff">class AutoPartitionByLong <span style="color: #0000ff">extends AbstractPartitionAlgorithm <span style="color: #0000ff">implements<span style="color: #000000"> RuleAlgorithm{
<span style="color: #0000ff">private<span style="color: #000000"> String mapFile;
<span style="color: #0000ff">private<span style="color: #000000"> LongRange[] longRongs;
<span style="color: #0000ff">private <span style="color: #0000ff">int defaultNode = -1<span style="color: #000000">;
@Override
<span style="color: #0000ff">public <span style="color: #0000ff">void<span style="color: #000000"> init() {
initialize();
}
<span style="color: #0000ff">public <span style="color: #0000ff">void<span style="color: #000000"> setMapFile(String mapFile) {
<span style="color: #0000ff">this.mapFile =<span style="color: #000000"> mapFile;
}
@Override
<span style="color: #0000ff">public<span style="color: #000000"> Integer calculate(String columnValue) {
<span style="color: #008000">//<span style="color: #008000"> columnValue = NumberParseUtil.eliminateQoute(columnValue);
<span style="color: #0000ff">try<span style="color: #000000"> {
<span style="color: #0000ff">long value =<span style="color: #000000"> Long.parseLong(columnValue);
Integer rst = <span style="color: #0000ff">null<span style="color: #000000">;
<span style="color: #0000ff">for (LongRange longRang : <span style="color: #0000ff">this<span style="color: #000000">.longRongs) {
<span style="color: #0000ff">if (value <= longRang.valueEnd && value >=<span style="color: #000000"> longRang.valueStart) {
<span style="color: #0000ff">return<span style="color: #000000"> longRang.nodeIndx;
}
}
<span style="color: #008000">//<span style="color: #008000">数据超过范围,暂时使用配置的默认节点
<span style="color: #0000ff">if (rst == <span style="color: #0000ff">null && defaultNode >= 0<span style="color: #000000">) {
<span style="color: #0000ff">return<span style="color: #000000"> defaultNode;
}
<span style="color: #0000ff">return<span style="color: #000000"> rst;
} <span style="color: #0000ff">catch<span style="color: #000000"> (NumberFormatException e){
<span style="color: #0000ff">throw <span style="color: #0000ff">new IllegalArgumentException(<span style="color: #0000ff">new StringBuilder().append("columnValue:").append(columnValue).append(" Please eliminate any quote and non number within it."<span style="color: #000000">).toString(),e);
}
}
@Override
<span style="color: #0000ff">public<span style="color: #000000"> Integer[] calculateRange(String beginValue,String endValue) {
<span style="color: #0000ff">return AbstractPartitionAlgorithm.calculateSequenceRange(<span style="color: #0000ff">this<span style="color: #000000">,beginValue,endValue);
}
@Override
<span style="color: #0000ff">public <span style="color: #0000ff">int<span style="color: #000000"> getPartitionNum() {
<span style="color: #008000">//<span style="color: #008000"> int nPartition = longRongs.length;
<span style="color: #008000">/*</span><span style="color: #008000">
* fix #1284 这里的统计应该统计Range的nodeIndex的distinct总数
</span><span style="color: #008000">*/</span><span style="color: #000000">
Set</span><Integer> distNodeIdxSet = <span style="color: #0000ff">new</span> HashSet<Integer><span style="color: #000000">();
</span><span style="color: #0000ff">for</span><span style="color: #000000">(LongRange range : longRongs) {
distNodeIdxSet.add(range.nodeIndx);
}
</span><span style="color: #0000ff">int</span> nPartition =<span style="color: #000000"> distNodeIdxSet.size();
</span><span style="color: #0000ff">return</span><span style="color: #000000"> nPartition;
}
<span style="color: #0000ff">private <span style="color: #0000ff">void<span style="color: #000000"> initialize() {
BufferedReader in = <span style="color: #0000ff">null<span style="color: #000000">;
<span style="color: #0000ff">try<span style="color: #000000"> {
<span style="color: #008000">//<span style="color: #008000"> FileInputStream fin = new FileInputStream(new File(fileMapPath));
InputStream fin = <span style="color: #0000ff">this<span style="color: #000000">.getClass().getClassLoader()
.getResourceAsStream(mapFile);
<span style="color: #0000ff">if (fin == <span style="color: #0000ff">null<span style="color: #000000">) {
<span style="color: #0000ff">throw <span style="color: #0000ff">new RuntimeException("can't find class resource file "
+<span style="color: #000000"> mapFile);
}
in = <span style="color: #0000ff">new BufferedReader(<span style="color: #0000ff">new<span style="color: #000000"> InputStreamReader(fin));
LinkedList longRangeList = <span style="color: #0000ff">new LinkedList<span style="color: #000000">();
</span><span style="color: #0000ff">for</span> (String line = <span style="color: #0000ff">null</span>; (line = in.readLine()) != <span style="color: #0000ff">null</span><span style="color: #000000">;) {
line </span>=<span style="color: #000000"> line.trim();
</span><span style="color: #0000ff">if</span> (line.startsWith("#") || line.startsWith("//"<span style="color: #000000">)) {
</span><span style="color: #0000ff">continue</span><span style="color: #000000">;
}
</span><span style="color: #0000ff">int</span> ind = line.indexOf('='<span style="color: #000000">);
</span><span style="color: #0000ff">if</span> (ind < 0<span style="color: #000000">) {
System.out.println(</span>" warn: bad line int " + mapFile + " :"
+<span style="color: #000000"> line);
</span><span style="color: #0000ff">continue</span><span style="color: #000000">;
}
String pairs[] </span>= line.substring(0,ind).trim().split("-"<span style="color: #000000">);
</span><span style="color: #0000ff">long</span> longStart = NumberParseUtil.parseLong(pairs[0<span style="color: #000000">].trim());
</span><span style="color: #0000ff">long</span> longEnd = NumberParseUtil.parseLong(pairs[1<span style="color: #000000">].trim());
</span><span style="color: #0000ff">int</span> nodeId = Integer.parseInt(line.substring(ind + 1<span style="color: #000000">)
.trim());
longRangeList
.add(</span><span style="color: #0000ff">new</span><span style="color: #000000"> LongRange(nodeId,longStart,longEnd));
}
longRongs </span>= longRangeList.toArray(<span style="color: #0000ff">new</span><span style="color: #000000"> LongRange[longRangeList
.size()]);
} </span><span style="color: #0000ff">catch</span><span style="color: #000000"> (Exception e) {
</span><span style="color: #0000ff">if</span> (e <span style="color: #0000ff">instanceof</span><span style="color: #000000"> RuntimeException) {
</span><span style="color: #0000ff">throw</span><span style="color: #000000"> (RuntimeException) e;
} </span><span style="color: #0000ff">else</span><span style="color: #000000"> {
</span><span style="color: #0000ff">throw</span> <span style="color: #0000ff">new</span><span style="color: #000000"> RuntimeException(e);
}
} </span><span style="color: #0000ff">finally</span><span style="color: #000000"> {
</span><span style="color: #0000ff">try</span><span style="color: #000000"> {
in.close();
} </span><span style="color: #0000ff">catch</span><span style="color: #000000"> (Exception e2) {
}
}
}
<span style="color: #0000ff">public <span style="color: #0000ff">int<span style="color: #000000"> getDefaultNode() {
<span style="color: #0000ff">return<span style="color: #000000"> defaultNode;
}
<span style="color: #0000ff">public <span style="color: #0000ff">void setDefaultNode(<span style="color: #0000ff">int<span style="color: #000000"> defaultNode) {
<span style="color: #0000ff">this.defaultNode =<span style="color: #000000"> defaultNode;
}
<span style="color: #0000ff">static <span style="color: #0000ff">class<span style="color: #000000"> LongRange {
<span style="color: #0000ff">public <span style="color: #0000ff">final <span style="color: #0000ff">int<span style="color: #000000"> nodeIndx;
<span style="color: #0000ff">public <span style="color: #0000ff">final <span style="color: #0000ff">long<span style="color: #000000"> valueStart;
<span style="color: #0000ff">public <span style="color: #0000ff">final <span style="color: #0000ff">long<span style="color: #000000"> valueEnd;
</span><span style="color: #0000ff">public</span> LongRange(<span style="color: #0000ff">int</span> nodeIndx,<span style="color: #0000ff">long</span> valueStart,<span style="color: #0000ff">long</span><span style="color: #000000"> valueEnd) {
</span><span style="color: #0000ff">super</span><span style="color: #000000">();
</span><span style="color: #0000ff">this</span>.nodeIndx =<span style="color: #000000"> nodeIndx;
</span><span style="color: #0000ff">this</span>.valueStart =<span style="color: #000000"> valueStart;
</span><span style="color: #0000ff">this</span>.valueEnd =<span style="color: #000000"> valueEnd;
}
}
}
<span style="color: #0000ff">import<span style="color: #000000"> java.io.BufferedReader;
<span style="color: #0000ff">import<span style="color: #000000"> java.io.InputStream;
<span style="color: #0000ff">import<span style="color: #000000"> java.io.InputStreamReader;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.HashMap;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.HashSet;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.Map;
<span style="color: #0000ff">import<span style="color: #000000"> java.util.Set;
<span style="color: #0000ff">import<span style="color: #000000"> io.mycat.config.model.rule.RuleAlgorithm;
<span style="color: #008000">/**<span style="color: #008000">
-
-
<span style="color: #808080">@author<span style="color: #008000"> mycat
<span style="color: #008000">*/
<span style="color: #0000ff">public <span style="color: #0000ff">class PartitionByFileMap <span style="color: #0000ff">extends AbstractPartitionAlgorithm <span style="color: #0000ff">implements<span style="color: #000000"> RuleAlgorithm {
<span style="color: #0000ff">private<span style="color: #000000"> String mapFile;
<span style="color: #0000ff">private Map<Object,Integer><span style="color: #000000"> app2Partition;
<span style="color: #008000">/**<span style="color: #008000">
- Map<Object,Integer> app2Partition中key值的类型:默认值为0,0表示Integer,非零表示String
<span style="color: #008000">*/
<span style="color: #0000ff">private <span style="color: #0000ff">int<span style="color: #000000"> type;
<span style="color: #008000">/**<span style="color: #008000">
- 默认节点在map中的key
<span style="color: #008000">*/
<span style="color: #0000ff">private <span style="color: #0000ff">static <span style="color: #0000ff">final String DEFAULT_NODE = "DEFAULT_NODE"<span style="color: #000000">;
<span style="color: #008000">/**<span style="color: #008000">
- 默认节点:小于0表示不设置默认节点,大于等于0表示设置默认节点
-
- 默认节点的作用:枚举分片时,如果碰到不识别的枚举值,就让它路由到默认节点
- 如果不配置默认节点(defaultNode值小于0表示不配置默认节点),碰到
- 不识别的枚举值就会报错,
- like this:can't find datanode for sharding column:column_name val:ffffffff
<span style="color: #008000">*/
<span style="color: #0000ff">private <span style="color: #0000ff">int defaultNode = -1<span style="color: #000000">;
@Override
<span style="color: #0000ff">public <span style="color: #0000ff">void<span style="color: #000000"> init() {
initialize();
}
<span style="color: #0000ff">public <span style="color: #0000ff">void<span style="color: #000000"> setMapFile(String mapFile) {
<span style="color: #0000ff">this.mapFile =<span style="color: #000000"> mapFile;
}
<span style="color: #0000ff">public <span style="color: #0000ff">void setType(<span style="color: #0000ff">int<span style="color: #000000"> type) {
<span style="color: #0000ff">this.type =<span style="color: #000000"> type;
}
<span style="color: #0000ff">public <span style="color: #0000ff">void setDefaultNode(<span style="color: #0000ff">int<span style="color: #000000"> defaultNode) {
<span style="color: #0000ff">this.defaultNode =<span style="color: #000000"> defaultNode;
}
@Override
<span style="color: #0000ff">public<span style="color: #000000"> Integer calculate(String columnValue) {
<span style="color: #0000ff">try<span style="color: #000000"> {
Object value =<span style="color: #000000"> columnValue;
<span style="color: #0000ff">if (type == 0<span style="color: #000000">) {
value =<span style="color: #000000"> Integer.valueOf(columnValue);
}
Integer rst = <span style="color: #0000ff">null<span style="color: #000000">;
Integer pid =<span style="color: #000000"> app2Partition.get(value);
<span style="color: #0000ff">if (pid != <span style="color: #0000ff">null<span style="color: #000000">) {
rst =<span style="color: #000000"> pid;
} <span style="color: #0000ff">else<span style="color: #000000"> {
rst =<span style="color: #000000"> app2Partition.get(DEFAULT_NODE);
}
<span style="color: #0000ff">return<span style="color: #000000"> rst;
} <span style="color: #0000ff">catch<span style="color: #000000"> (NumberFormatException e){
<span style="color: #0000ff">throw <span style="color: #0000ff">new IllegalArgumentException(<span style="color: #0000ff">new StringBuilder().append("columnValue:").append(columnValue).append(" Please check if the format satisfied."<span style="color: #000000">).toString(),e);
}
}
@Override
<span style="color: #0000ff">public <span style="color: #0000ff">int<span style="color: #000000"> getPartitionNum() {
Set set = <span style="color: #0000ff">new HashSet<span style="color: #000000">(app2Partition.values());
<span style="color: #0000ff">int count =<span style="color: #000000"> set.size();
<span style="color: #0000ff">return<span style="color: #000000"> count;
}
<span style="color: #0000ff">private <span style="color: #0000ff">void<span style="color: #000000"> initialize() {
BufferedReader in = <span style="color: #0000ff">null<span style="color: #000000">;
<span style="color: #0000ff">try<span style="color: #000000"> {
<span style="color: #008000">//<span style="color: #008000"> FileInputStream fin = new FileInputStream(new File(fileMapPath));
InputStream fin = <span style="color: #0000ff">this<span style="color: #000000">.getClass().getClassLoader()
.getResourceAsStream(mapFile);
<span style="color: #0000ff">if (fin == <span style="color: #0000ff">null<span style="color: #000000">) {
<span style="color: #0000ff">throw <span style="color: #0000ff">new RuntimeException("can't find class resource file "
+<span style="color: #000000"> mapFile);
}
in = <span style="color: #0000ff">new BufferedReader(<span style="color: #0000ff">new<span style="color: #000000"> InputStreamReader(fin));
app2Partition </span>= <span style="color: #0000ff">new</span> HashMap<Object,Integer><span style="color: #000000">();
</span><span style="color: #0000ff">for</span> (String line = <span style="color: #0000ff">null</span>; (line = in.readLine()) != <span style="color: #0000ff">null</span><span style="color: #000000">;) {
line </span>=<span style="color: #000000"> line.trim();
</span><span style="color: #0000ff">if</span> (line.startsWith("#") || line.startsWith("//"<span style="color: #000000">)) {
</span><span style="color: #0000ff">continue</span><span style="color: #000000">;
}
</span><span style="color: #0000ff">int</span> ind = line.indexOf('='<span style="color: #000000">);
</span><span style="color: #0000ff">if</span> (ind < 0<span style="color: #000000">) {
</span><span style="color: #0000ff">continue</span><span style="color: #000000">;
}
</span><span style="color: #0000ff">try</span><span style="color: #000000"> {
String key </span>= line.substring(0<span style="color: #000000">,ind).trim();
</span><span style="color: #0000ff">int</span> pid = Integer.parseInt(line.substring(ind + 1<span style="color: #000000">).trim());
</span><span style="color: #0000ff">if</span>(type == 0<span style="color: #000000">) {
app2Partition.put(Integer.parseInt(key),pid);
} </span><span style="color: #0000ff">else</span><span style="color: #000000"> {
app2Partition.put(key,pid);
}
} </span><span style="color: #0000ff">catch</span><span style="color: #000000"> (Exception e) {
}
}
</span><span style="color: #008000">//</span><span style="color: #008000">设置默认节点</span>
<span style="color: #0000ff">if</span>(defaultNode >= 0<span style="color: #000000">) {
app2Partition.put(DEFAULT_NODE,defaultNode);
}
} </span><span style="color: #0000ff">catch</span><span style="color: #000000"> (Exception e) {
</span><span style="color: #0000ff">if</span> (e <span style="color: #0000ff">instanceof</span><span style="color: #000000"> RuntimeException) {
</span><span style="color: #0000ff">throw</span><span style="color: #000000"> (RuntimeException) e;
} </span><span style="color: #0000ff">else</span><span style="color: #000000"> {
</span><span style="color: #0000ff">throw</span> <span style="color: #0000ff">new</span><span style="color: #000000"> RuntimeException(e);
}
} </span><span style="color: #0000ff">finally</span><span style="color: #000000"> {
</span><span style="color: #0000ff">try</span><span style="color: #000000"> {
in.close();
} </span><span style="color: #0000ff">catch</span><span style="color: #000000"> (Exception e2) {
}
}
}
}