將之前爬蟲抽取器用aardio實(shí)現(xiàn)。
import myplu.jpath;
import myplu.xpath;
namespace myplu
// 選擇器基類
// 繼承該類實(shí)現(xiàn)handle方法，handle每次處理一條數(shù)據(jù)
class selector{
ctor( ... ){
this.expression = "";
this.expType = "MIX";
if(type(...) == "table"){
..table.assign(this,...);
}
};
type="crawler.selector";
["select"] = function(data){
var res = owner.selectList(data);
if(#res and table.len(res)>0){
return res[1];
}
return res;
};
selectList = function(data){
// 空數(shù)據(jù)不用處理，直接退出
if(data == null){
return null;
}
var res = {};
// 將單個(gè)數(shù)據(jù)轉(zhuǎn)為數(shù)組來處理
if(!(type(data) == "table" and ..table.len(data)>0)){
data = {data};
}
// 遍歷數(shù)據(jù)，依次處理
for(k,v in data){
// 執(zhí)行handle方法,抽取數(shù)據(jù)。
var dt = owner.handle(v);
if(dt){
// 處理數(shù)據(jù)結(jié)果，數(shù)組合并，非數(shù)組放入數(shù)組。
if(type(dt) == "table" and ..table.len(dt)>0){
..table.assign(res,dt);
}else{
..table.push(res,dt);
}
}
}
// 如果結(jié)果就一條，直接返回?cái)?shù)據(jù)值。
if(#res == 1){
return res[1];
}
return res;
};
parse = function(...){
..table.assign(this,...);
};
category = {"MIX","XPATH","JPATH","REGEX","SPLIT","REPLACE"};
space = {"URL","BODY","BLOCK","PAGE"};
}
// 模式提取選擇器
class regexSelector{
ctor( expression, subStr){
this = ..myplu.selector(expression=expression; subStr=subStr; expType = "REGEX");
};
type="crawler.regexSelector";
handle = function(data){
var data = ..string.matches(data,this.expression);
if(type(subStr) == "number" and subStr > 0 and subStr < #data){
return data[subStr];
}
return data;
};
}
// 模式拆分選擇器
class splitSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "SPLIT");
};
type="crawler.splitSelector";
handle = function(data){
return ..string.split(data,this.expression);
};
}
// 模式替換選擇器
class replaceSelector{
ctor( expression, subStr){
this = ..myplu.selector(expression=expression; subStr=subStr; expType = "REPLACE");
};
type="crawler.replaceSelector";
handle = function(data){
return ..string.replace(data,this.expression,this.subStr);
};
}
// xpath選擇器
class xpathSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "XPATH");
};
type="crawler.xpathSelector";
handle = function(data){
data = ..myplu.xpath(data).selectList(this.expression);
if(#data==1){
return data[1];
}
return data;
};
}
// jpath選擇器
class jpathSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "JPATH");
};
type="crawler.jpathSelector";
handle = function(data){
data = ..myplu.jpath(data).selectList(this.expression);
if(#data==1){
return data[1];
}
return data;
};
}
// 混合選擇器
class mixSelector{
ctor( expression){
this = ..myplu.selector(expression=expression; expType = "MIX");
};
type="crawler.mixSelector";
handle = function(data){
if(!this["selectors"]){
var exp = ..string.replace(this.expression,"(\w+)(%())", "..myplu.\1Selector\2");
exp = ..string.matches(exp, "\.\.[^\.][\w\W]+?%()");
this.selectors = {};
for(k,v in exp){
var sel = eval(v[1]);
..table.push(this.selectors,sel);
}
}
for(k,v in this.selectors){
data = v.handle(data);
}
return data;
};
}
// 鏈?zhǔn)匠槿∑?class selectors{
ctor(){
this.selectors = {};
};
type="crawler.selectors";
jpath = function(expression){
..table.push(this.selectors, ..myplu.jpathSelector(expression,araeSpace,mutil));
return this;
};
xpath = function(expression){
..table.push(this.selectors, ..myplu.xpathSelector(expression,araeSpace,mutil));
return this;
};
regex = function(expression, subStr){
..table.push(this.selectors, ..myplu.regexSelector(expression,subStr,araeSpace,mutil));
return this;
};
replace = function(expression, subStr){
..table.push(this.selectors, ..myplu.replaceSelector(expression,subStr,araeSpace,mutil));
return this;
};
split = function(expression){
..table.push(this.selectors, ..myplu.splitSelector(expression,araeSpace,mutil));
return this;
};
mix = function(expression){
..table.push(this.selectors, ..myplu.mixSelector(expression,araeSpace,mutil));
return this;
};
more = function(...){
var data = ...;
if(type(data)!="table"){
data = {...};
}elseif(data["type"]) {
data = {...};
}
..table.push(this.selectors,data);
return this;
};
["select"] = function(data){
data = this.selectList(data);
if(#data){
return data[1];
}
};
selectList = function(data){
for(k,v in this.selectors){
if(v["type"] and ..string.find(v["type"],"crawler\.\w+Selector")){
data = v.selectList(data);
}else {
var res = {};
//..console.dump(data);
if(!(type(data) == "table" and ..table.len(data)>0)){
data = {data};
}
//..console.dump(data);
for(k1, v1 in data){
var re = {};
for(k2, v2 in v){
//..console.dump(v2);
if(v2["type"] and v2["type"]=="crawler.selectors"){
re[k2] = v2.selectList(v1);
}else{
// 附加的直接傳值
re[k2] = v2;
}
}
..table.push(res,re);
}
data = res;
}
if(data == null){
return null;
}
}
return data;
};
}
namespace crawler.selector
/*****intellisense(myplu)
crawler.selector = 導(dǎo)入數(shù)據(jù)抽取器
selector(__) = 基礎(chǔ)選擇器。需要自己實(shí)現(xiàn)handle方法，執(zhí)行數(shù)據(jù)在處理函數(shù)。\n!cwSelector.
regexSelector(.(表達(dá)式, 抽取結(jié)果下標(biāo)) = 模式匹配選擇器。\n!cwSelector.
replaceSelector(.(表達(dá)式, 替換內(nèi)容) = 模式替換選擇器。\n!cwSelector.
splitSelector(.(表達(dá)式) = 模式拆分選擇器。\n!cwSelector.
xpathSelector(.(表達(dá)式) = xpath選擇器。\n!cwSelector.
jpathSelector(.(表達(dá)式) = jsonPath選擇器。\n!cwSelector.
mixSelector(.(表達(dá)式) = 混合選擇器。\n!cwSelector.
selectors() = 串聯(lián)選擇器。\n!cwSelectors.
end intellisense*****/
/*****intellisense(!cwSelector)
select(__) = 執(zhí)行選擇器，抽取單條數(shù)據(jù)。
selectList(__) = 執(zhí)行選擇器，抽取數(shù)據(jù)列表。
handle(__) = 選擇器處理函數(shù)。
parse(__) = 將table轉(zhuǎn)為對(duì)象。
category.? = 選擇器類型
space.? = 抽取數(shù)據(jù)區(qū)域
subStr = 替換選擇器，替換內(nèi)容。
expression = 規(guī)則表達(dá)式。
expType = 選擇器類型category.?。\n"MIX":混合選擇器,\n"XPATH":xpath選擇器,\n"JPATH":jsonPath選擇器,\n"REGEX":模式抽取選擇器,\n"SPLIT":模式拆分選擇器,\n"REPLACE":模式替換選擇器。
setExpression(__) = 設(shè)置規(guī)則表達(dá)式。
setExpType(__) = 選擇器類型category.?。\n"MIX":混合選擇器,\n"XPATH":xpath選擇器,\n"JPATH":jsonPath選擇器,\n"REGEX":模式抽取選擇器,\n"SPLIT":模式拆分選擇器,\n"REPLACE":模式替換選擇器。
setSubStr(__) = 設(shè)置替換內(nèi)容。
getExpression() = 獲取規(guī)則表達(dá)式。
getExpType() = 選擇器類型。
getSubStr() = 獲取替換內(nèi)容。
end intellisense*****/
/*****intellisense(!cwSelectors)
regex(.(表達(dá)式,抽取結(jié)果下標(biāo)) = 模式匹配選擇器。\n!cwSelectors.
split(.(表達(dá)式) = 模式拆分選擇器。\n!cwSelectors.
replace(.(表達(dá)式,替換內(nèi)容) = 模式替換選擇器。\n!cwSelectors.
xpath(.(表達(dá)式) = xpath選擇器。\n!cwSelectors.
jpath(.(表達(dá)式) = jsonPath選擇器。\n!cwSelectors.
mix(.(表達(dá)式) = 混合選擇器。\n!cwSelectors.
more(.(__) = 列表抽取。@1 可以是myplu.selectors數(shù)組，或myplu.selectors為value的表
select(__) = 執(zhí)行選擇器，抽取單條數(shù)據(jù)。
selectList(__) = 執(zhí)行選擇器，抽取數(shù)據(jù)列表。
end intellisense*****/
